api: indexer debug health check (#1347)

* api: add debug health * api: efficient getLatency() --------- Co-authored-by: DC <[email protected]>
daimo-eth · Sep 27, 2024 · 3bbb0aa · 3bbb0aa
1 parent 348bc77
commit 3bbb0aa
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 1 deletion.
diff --git a/packages/daimo-api/src/api/healthCheck.ts b/packages/daimo-api/src/api/healthCheck.ts
@@ -64,10 +64,12 @@ async function healthCheckInner(
 
   if (showDetailedDebug) {
     const nPromises = await countPromises();
+    const latencies = await watcher.getLatency();
     ret = {
       ...ret,
       nPromises,
       trpcReqsInFlight: trpcReqsInFlight.slice(),
+      ...latencies,
     } as any;
   }
 

diff --git a/packages/daimo-api/src/codegen/dbIndex.ts b/packages/daimo-api/src/codegen/dbIndex.ts
@@ -1,5 +1,11 @@
 import type { ColumnType } from "kysely";
 
+export type Int8 = ColumnType<
+  string,
+  bigint | number | string,
+  bigint | number | string
+>;
+
 export type Numeric = ColumnType<string, number | string, number | string>;
 
 export interface IndexDaimoAcct {
@@ -39,13 +45,22 @@ export interface IndexDaimoAcctUpdate {
   tx_idx: Numeric;
 }
 
+export interface IndexDaimoBlock {
+  chain_id: Int8;
+  hash: Buffer;
+  inserted_at: Int8 | null;
+  number: Int8;
+  parent_hash: Buffer;
+  timestamp: Int8;
+}
+
 export interface IndexDaimoFastCctp {
   block_hash: Buffer;
   block_num: Numeric;
   block_ts: Numeric;
   chain_id: Numeric;
   final_recipient: Buffer | null;
-  from_addr: Buffer | null;
+  from_addr: Buffer;
   from_amount: Numeric;
   from_chain_id: Numeric;
   from_token: Buffer | null;
@@ -162,6 +177,7 @@ export interface IndexDaimoTransferTx {
 export interface DB {
   "index.daimo_acct": IndexDaimoAcct;
   "index.daimo_acct_update": IndexDaimoAcctUpdate;
+  "index.daimo_block": IndexDaimoBlock;
   "index.daimo_fast_cctp": IndexDaimoFastCctp;
   "index.daimo_index": IndexDaimoIndex;
   "index.daimo_name": IndexDaimoName;

diff --git a/packages/daimo-api/src/db/indexWatcher.ts b/packages/daimo-api/src/db/indexWatcher.ts
@@ -179,6 +179,39 @@ export class IndexWatcher {
     return Number(result?.latest_block_num || 0);
   }
 
+  // Get the block write to postgres latency over the last numBlocks
+  async getLatency() {
+    // Get the block times and latencies for the last 1000 blocks
+    const blockTimes = await retryBackoff(`get-block-db-latency`, () =>
+      this.kdb
+        .selectFrom("index.daimo_block")
+        .select(["timestamp", "inserted_at"])
+        .where("chain_id", "=", "" + chainConfig.chainL2.id)
+        .where("inserted_at", "is not", null)
+        .orderBy("number", "desc") // Primary key scan: (chain_id, number)
+        .limit(100)
+        .execute()
+    );
+    if (blockTimes.length === 0) return {};
+
+    const latencies = blockTimes.map((blockTime) =>
+      Math.abs(
+        Number(BigInt(blockTime.timestamp) - BigInt(blockTime.inserted_at!))
+      )
+    );
+
+    // Latency stats
+    const avgLatency =
+      latencies.reduce((sum, latency) => sum + latency, 0) / latencies.length;
+    const maxLatency = Math.max(...latencies);
+    const minLatency = Math.min(...latencies);
+    return {
+      avgLatency,
+      maxLatency,
+      minLatency,
+    };
+  }
+
   getStatus() {
     const { lastGoodTickS, indexLatest, rpcLatest } = this;
     const { idleCount, totalCount, waitingCount } = this.pg;
@@ -222,6 +255,7 @@ export class IndexWatcher {
         hash BYTEA NOT NULL,
         parent_hash BYTEA NOT NULL,
         timestamp NUMERIC NOT NULL,
+        inserted_at BYTEA NOT NULL,
         PRIMARY KEY (chain_id, number, hash)
       );