feat(store-indexer,store-sync): allow filtering by table IDs (#1572)

latticexyz · Sep 29, 2023 · 4998d69 · 4998d69
1 parent da6e0de
commit 4998d69
Show file tree

Hide file tree

Showing 8 changed files with 47 additions and 15 deletions.
diff --git a/packages/store-indexer/src/postgres/createQueryAdapter.ts b/packages/store-indexer/src/postgres/createQueryAdapter.ts
@@ -13,11 +13,10 @@ import { getAddress } from "viem";
  */
 export async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {
   const adapter: QueryAdapter = {
-    async findAll(chainId, address) {
-      const internalTables = buildInternalTables();
-      const tables = (await getTables(database)).filter(
-        (table) => address != null && getAddress(address) === getAddress(table.address)
-      );
+    async findAll({ chainId, address, tableIds = [] }) {
+      const tables = (await getTables(database))
+        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
+        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
 
       const tablesWithRecords = await Promise.all(
         tables.map(async (table) => {
@@ -33,6 +32,7 @@ export async function createQueryAdapter(database: PgDatabase<any>): Promise<Que
         })
       );
 
+      const internalTables = buildInternalTables();
       const metadata = await database
         .select()
         .from(internalTables.chain)

diff --git a/packages/store-indexer/src/sqlite/createQueryAdapter.ts b/packages/store-indexer/src/sqlite/createQueryAdapter.ts
@@ -3,6 +3,7 @@ import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
 import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
 import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
 import { debug } from "../debug";
+import { getAddress } from "viem";
 
 /**
  * Creates a storage adapter for the tRPC server/client to query data from SQLite.
@@ -12,8 +13,10 @@ import { debug } from "../debug";
  */
 export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<QueryAdapter> {
   const adapter: QueryAdapter = {
-    async findAll(chainId, address) {
-      const tables = getTables(database).filter((table) => table.address === address);
+    async findAll({ chainId, address, tableIds = [] }) {
+      const tables = getTables(database)
+        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
+        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
 
       const tablesWithRecords = tables.map((table) => {
         const sqliteTable = buildTable(table);

diff --git a/packages/store-sync/src/common.ts b/packages/store-sync/src/common.ts
@@ -41,6 +41,10 @@ export type SyncOptions<TConfig extends StoreConfig = StoreConfig> = {
    * MUD Store/World contract address
    */
   address?: Address;
+  /**
+   * Optional table IDs to filter indexer state and RPC state.
+   */
+  tableIds?: Hex[];
   /**
    * Optional block number to start indexing from. Useful for resuming the indexer from a particular point in time or starting after a particular contract deployment.
    */

diff --git a/packages/store-sync/src/createStoreSync.ts b/packages/store-sync/src/createStoreSync.ts
@@ -32,6 +32,7 @@ import { createIndexerClient } from "./trpc-indexer";
 import { SyncStep } from "./SyncStep";
 import { chunk, isDefined } from "@latticexyz/common/utils";
 import { encodeKey, encodeValueArgs } from "@latticexyz/protocol-parser";
+import { internalTableIds } from "./internalTableIds";
 
 const debug = parentDebug.extend("createStoreSync");
 
@@ -49,13 +50,15 @@ type CreateStoreSyncOptions<TConfig extends StoreConfig = StoreConfig> = SyncOpt
 export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>({
   storageAdapter,
   onProgress,
-  address,
   publicClient,
+  address,
+  tableIds = [],
   startBlock: initialStartBlock = 0n,
   maxBlockRange,
   initialState,
   indexerUrl,
 }: CreateStoreSyncOptions<TConfig>): Promise<SyncResult> {
+  const includedTableIds = new Set(tableIds.length ? [...internalTableIds, ...tableIds] : []);
   const initialState$ = defer(
     async (): Promise<
       | {
@@ -79,7 +82,7 @@ export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>
 
       const indexer = createIndexerClient({ url: indexerUrl });
       const chainId = publicClient.chain?.id ?? (await publicClient.getChainId());
-      const result = await indexer.findAll.query({ chainId, address });
+      const result = await indexer.findAll.query({ chainId, address, tableIds: Array.from(includedTableIds) });
 
       onProgress?.({
         step: SyncStep.SNAPSHOT,

diff --git a/packages/store-sync/src/index.ts b/packages/store-sync/src/index.ts
@@ -1,3 +1,4 @@
 export * from "./common";
 export * from "./createStoreSync";
+export * from "./internalTableIds";
 export * from "./SyncStep";
diff --git a/packages/store-sync/src/internalTableIds.ts b/packages/store-sync/src/internalTableIds.ts
@@ -0,0 +1,23 @@
+import { resourceIdToHex } from "@latticexyz/common";
+import storeConfig from "@latticexyz/store/mud.config";
+import worldConfig from "@latticexyz/world/mud.config";
+
+// TODO: refactor config to include table IDs (https://github.com/latticexyz/mud/pull/1561)
+
+export const storeTableIds = Object.keys(storeConfig.tables).map((name) =>
+  resourceIdToHex({
+    type: storeConfig.tables[name as keyof typeof storeConfig.tables].offchainOnly ? "offchainTable" : "table",
+    namespace: storeConfig.namespace,
+    name,
+  })
+);
+
+const worldTableIds = Object.keys(worldConfig.tables).map((name) =>
+  resourceIdToHex({
+    type: worldConfig.tables[name as keyof typeof worldConfig.tables].offchainOnly ? "offchainTable" : "table",
+    namespace: worldConfig.namespace,
+    name,
+  })
+);
+
+export const internalTableIds = [...storeTableIds, ...worldTableIds];
diff --git a/packages/store-sync/src/trpc-indexer/common.ts b/packages/store-sync/src/trpc-indexer/common.ts
@@ -2,10 +2,7 @@ import { Hex } from "viem";
 import { TableWithRecords } from "../common";
 
 export type QueryAdapter = {
-  findAll: (
-    chainId: number,
-    address?: Hex
-  ) => Promise<{
+  findAll: (opts: { chainId: number; address?: Hex; tableIds?: Hex[] }) => Promise<{
     blockNumber: bigint | null;
     tables: TableWithRecords[];
   }>;

diff --git a/packages/store-sync/src/trpc-indexer/createAppRouter.ts b/packages/store-sync/src/trpc-indexer/createAppRouter.ts
@@ -16,12 +16,13 @@ export function createAppRouter() {
         z.object({
           chainId: z.number(),
           address: z.string().refine(isHex).optional(),
+          tableIds: z.array(z.string().refine(isHex)).optional(),
         })
       )
       .query(async (opts): ReturnType<QueryAdapter["findAll"]> => {
         const { queryAdapter } = opts.ctx;
-        const { chainId, address } = opts.input;
-        return queryAdapter.findAll(chainId, address);
+        const { chainId, address, tableIds } = opts.input;
+        return queryAdapter.findAll({ chainId, address, tableIds });
       }),
   });
 }