Optimize account history fetch

[nounderline]

Some calls to getAccountHistory may take even to ~10s in some circumstances. The function itself is fetching data from:

• Internal indexers;
• Multiple Postgres queries through TRPC router or pg pool directly;
• external (!) RPC call through Viem.

Code, measurements: https://github.com/nounderline/240620-daimo-perf

[nounderline]

Finalized block and exchange rates are bottlenecks:

┌───────────────────┬───────────────┬───────────────┬───────────────┬───────────────┐
│       event       │      min      │      max      │      avg      │    stddev     │
│      varchar      │ decimal(10,2) │ decimal(10,2) │ decimal(10,2) │ decimal(10,2) │
├───────────────────┼───────────────┼───────────────┼───────────────┼───────────────┤
│ finBlock          │         26.51 │      20343.94 │        703.98 │       3177.45 │
│ linkedAccounts    │          0.00 │          0.12 │          0.00 │          0.01 │
│ landlineSessions  │          0.00 │          0.12 │          0.01 │          0.01 │
│ namedAccounts     │          0.01 │          0.65 │          0.02 │          0.04 │
│ chainGasConstants │          0.01 │          0.25 │          0.02 │          0.02 │
│ exchangeRates     │          1.43 │         51.36 │         31.93 │         17.77 │
│ balance           │          0.01 │          0.14 │          0.02 │          0.01 │
│ filterTransfers   │          0.13 │          2.80 │          0.21 │          0.17 │
└───────────────────┴───────────────┴───────────────┴───────────────┴───────────────┘

Above is using publicnode.com which has unpredictable response time.

For QuickNode L2 URL we get much better response:

┌───────────────────┬───────────────┬───────────────┬───────────────┬───────────────┐
│       event       │      min      │      max      │      avg      │    stddev     │
│      varchar      │ decimal(10,2) │ decimal(10,2) │ decimal(10,2) │ decimal(10,2) │
├───────────────────┼───────────────┼───────────────┼───────────────┼───────────────┤
│ balance           │          0.01 │          0.17 │          0.02 │          0.02 │
│ filterTransfers   │          0.15 │          0.94 │          0.24 │          0.12 │
│ linkedAccounts    │          0.00 │          0.03 │          0.01 │          0.00 │
│ landlineSessions  │          0.01 │          0.08 │          0.01 │          0.01 │
│ finBlock          │          4.13 │         21.45 │          5.53 │          1.91 │
│ namedAccounts     │          0.01 │          0.13 │          0.02 │          0.01 │
│ chainGasConstants │          0.01 │          0.22 │          0.02 │          0.03 │
│ exchangeRates     │          1.54 │         44.02 │         34.44 │         15.97 │
└───────────────────┴───────────────┴───────────────┴───────────────┴───────────────┘

Orchestrated log code

```diff diff --git a/packages/daimo-api/src/api/getAccountHistory.ts b/packages/daimo-api/src/api/getAccountHistory.ts index 2ba13409..7a28890b 100644 --- a/packages/daimo-api/src/api/getAccountHistory.ts +++ b/packages/daimo-api/src/api/getAccountHistory.ts @@ -110,12 +110,15 @@ export async function getAccountHistory( extApiCache: ExternalApiCache, blockNumber: number ): Promise { + let t0; + const eAcc = nameReg.getDaimoAccount(address); assert(eAcc != null && eAcc.name != null, "Not a Daimo account"); const startMs = Date.now(); const log = `[API] start getAccountHist: ${eAcc.name} ${address} since ${sinceBlockNum}`; console.log(log); + t0 = performance.now(); // Get latest finalized block. Next account sync, fetch since this block. const finBlock = await vc.publicClient.getBlock({ blockTag: "finalized", @@ -124,6 +127,9 @@ export async function getAccountHistory( if (finBlock.number < sinceBlockNum) { console.log(`${log}: sinceBlockNum > finalized block ${finBlock.number}`); } + console.log(`\ttime\tfinBlock\t${performance.now() - t0}`); + + t0 = performance.now(); // Get the latest block + current balance. assert( @@ -137,16 +143,22 @@ export async function getAccountHistory( ); const lastBalance = homeCoinIndexer.getCurrentBalance(address); + console.log(`\ttime\tbalance\t${performance.now() - t0}`); + // TODO: get userops, including reverted ones. Show failed sends. + t0 = performance.now(); + // Get successful transfers since sinceBlockNum const transferLogs = homeCoinIndexer.filterTransfers({ addr: address, sinceBlockNum: BigInt(sinceBlockNum), }); let elapsedMs = Date.now() - startMs; + console.log(`\ttime\tfilterTransfers\t${performance.now() - t0}`); console.log(`${log}: ${elapsedMs}ms ${transferLogs.length} logs`); + t0 = performance.now(); // Get named accounts const addrs = new Set

(); transferLogs.forEach((log) => { @@ -163,16 +175,21 @@ export async function getAccountHistory( await Promise.all([...addrs].map((addr) => nameReg.getEAccount(addr))) ).filter((acc) => hasAccountName(acc)); + console.log(`\ttime\tnamedAccounts\t${performance.now() - t0}`); + // Get account keys const accountKeys = keyReg.resolveAddressKeys(address); assert(accountKeys != null); + t0 = performance.now(); // Prefetch info required to send operations > fast at time of sending. const chainGasConstants = await paymaster.calculateChainGasConstants(eAcc); + console.log(`\ttime\tchainGasConstants\t${performance.now() - t0}`); // Prefetch info required to deposit to your Daimo account. const recommendedExchanges = fetchRecommendedExchanges(eAcc); + t0 = performance.now(); // Get linked accounts const linkedAccounts = profileCache.getLinkedAccounts(address); const inviteLinkStatus = inviteCode @@ -185,6 +202,7 @@ export async function getAccountHistory( db )) as DaimoInviteCodeStatus) : null; + console.log(`\ttime\tlinkedAccounts\t${performance.now() - t0}`); const inviteeAddrs = inviteGraph.getInvitees(address); const invitees = inviteeAddrs @@ -209,12 +227,15 @@ export async function getAccountHistory( console.log(`${log}: ${elapsedMs}: ${proposedSwaps.length} swaps`); // Get exchange rates + t0 = performance.now(); const exchangeRates = await getExchangeRates(extApiCache); + console.log(`\ttime\texchangeRates\t${performance.now() - t0}`); // Get landline session key and accounts let landlineSessionKey = ""; let landlineAccounts: LandlineAccount[] = []; + t0 = performance.now(); const username = eAcc.name; const isUserWhitelisted = getEnvApi().LANDLINE_WHITELIST_USERNAMES.includes(username); @@ -222,6 +243,7 @@ export async function getAccountHistory( landlineSessionKey = await getLandlineSession(address); landlineAccounts = await getLandlineAccounts(address); } + console.log(`\ttime\tlandlineSessions\t${performance.now() - t0}`); const ret: AccountHistoryResult = { address, ```