feat: add getRecords util to fetch records from indexer, update dep…

…loyer and explorer to use indexer (#3385)

.changeset/big-cooks-fetch.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/explorer": patch
+---
+
+Improved the performance of the explorer's `Interact` tab by fetching the ABI from an indexer instead of from an Ethereum RPC if available.

.changeset/shy-eels-hide.md

@@ -0,0 +1,8 @@
+---
+"@latticexyz/store-sync": patch
+"@latticexyz/world": patch
+---
+
+Added a `getRecords` util to fetch table records from an indexer or RPC.
+
+Migrated the `getFunctions` and `getWorldAbi` utils from `@latticexyz/world` to `@latticexyz/store-sync/world` to allow `getFunctions` and `getWorldAbi` to use `getRecords` internally without circular dependencies.

.changeset/slimy-cheetahs-vanish.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/cli": patch
+---
+
+Added an `indexerUrl` option to the `mud deploy` and `mud pull` CLI commands to read table records from an indexer instead of fetching logs from an Ethereum RPC.

packages/cli/package.json

@@ -49,6 +49,7 @@
     "@latticexyz/protocol-parser": "workspace:*",
     "@latticexyz/schema-type": "workspace:*",
     "@latticexyz/store": "workspace:*",
+    "@latticexyz/store-sync": "workspace:*",
     "@latticexyz/utils": "workspace:*",
     "@latticexyz/world": "workspace:*",
     "@latticexyz/world-module-metadata": "workspace:*",

packages/cli/src/commands/dev-contracts.ts

@@ -92,6 +92,7 @@ const commandModule: CommandModule<typeof devOptions, InferredOptionTypes<typeof
             worldAddress,
             salt: "0x",
             kms: undefined,
+            indexerUrl: undefined,
           });
           worldAddress = deploy.address;
           // if there were changes while we were deploying, trigger it again

packages/cli/src/commands/pull.ts

@@ -5,6 +5,8 @@ import chalk from "chalk";
 import { WriteFileExistsError, pull } from "../pull/pull";
 import path from "node:path";
 import { build } from "../build";
+import { getChainId } from "viem/actions";
+import { defaultChains } from "../defaultChains";
 
 const options = {
   worldAddress: { type: "string", required: true, desc: "Remote world address" },
@@ -18,6 +20,11 @@ const options = {
     type: "boolean",
     desc: "Replace existing files and directories with data from remote world.",
   },
+  indexerUrl: {
+    type: "string",
+    desc: "The indexer URL to pull from.",
+    required: false,
+  },
 } as const;
 
 type Options = InferredOptionTypes<typeof options>;
@@ -44,6 +51,8 @@ const commandModule: CommandModule<Options, Options> = {
           : undefined,
       }),
     });
+    const chainId = await getChainId(client);
+    const indexerUrl = opts.indexerUrl ?? defaultChains.find((chain) => chain.id === chainId)?.indexerUrl;
 
     console.log(chalk.bgBlue(chalk.whiteBright(`\n Pulling MUD config from world at ${opts.worldAddress} \n`)));
     const rootDir = process.cwd();
@@ -53,6 +62,8 @@ const commandModule: CommandModule<Options, Options> = {
         rootDir,
         client,
         worldAddress: opts.worldAddress as Address,
+        indexerUrl,
+        chainId,
         replace: opts.replace,
       });
       await build({ rootDir, config, foundryProfile: profile });

packages/cli/src/defaultChains.ts

@@ -0,0 +1,3 @@
+import { redstone, garnet, rhodolite } from "@latticexyz/common/chains";
+
+export const defaultChains = [redstone, garnet, rhodolite];

packages/cli/src/deploy/common.ts

@@ -1,4 +1,4 @@
-import { Abi, Address, Hex, padHex } from "viem";
+import { Abi, Account, Address, Chain, Client, Hex, Transport, padHex } from "viem";
 import IBaseWorldAbi from "@latticexyz/world/out/IBaseWorld.sol/IBaseWorld.abi.json" assert { type: "json" };
 import { helloStoreEvent } from "@latticexyz/store";
 import { helloWorldEvent } from "@latticexyz/world";
@@ -119,3 +119,10 @@ export type Module = DeterministicContract & {
    */
   readonly optional?: boolean;
 };
+
+export type CommonDeployOptions = {
+  readonly client: Client<Transport, Chain | undefined, Account>;
+  readonly worldDeploy: WorldDeploy;
+  readonly indexerUrl?: string;
+  readonly chainId?: number;
+};

packages/cli/src/deploy/deploy.ts

@@ -1,8 +1,16 @@
-import { Account, Address, Chain, Client, Hex, Transport, stringToHex } from "viem";
+import { Address, Hex, stringToHex } from "viem";
 import { ensureDeployer } from "./ensureDeployer";
 import { deployWorld } from "./deployWorld";
 import { ensureTables } from "./ensureTables";
-import { Library, Module, System, WorldDeploy, supportedStoreVersions, supportedWorldVersions } from "./common";
+import {
+  CommonDeployOptions,
+  Library,
+  Module,
+  System,
+  WorldDeploy,
+  supportedStoreVersions,
+  supportedWorldVersions,
+} from "./common";
 import { ensureSystems } from "./ensureSystems";
 import { getWorldDeploy } from "./getWorldDeploy";
 import { ensureFunctions } from "./ensureFunctions";
@@ -23,7 +31,6 @@ import { getLibraryMap } from "./getLibraryMap";
 
 type DeployOptions = {
   config: World;
-  client: Client<Transport, Chain | undefined, Account>;
   tables: readonly Table[];
   systems: readonly System[];
   libraries: readonly Library[];
@@ -39,7 +46,7 @@ type DeployOptions = {
    */
   deployerAddress?: Hex;
   withWorldProxy?: boolean;
-};
+} & Omit<CommonDeployOptions, "worldDeploy">;
 
 /**
  * Given a viem client and MUD config, we attempt to introspect the world
@@ -58,6 +65,8 @@ export async function deploy({
   salt,
   worldAddress: existingWorldAddress,
   deployerAddress: initialDeployerAddress,
+  indexerUrl,
+  chainId,
 }: DeployOptions): Promise<WorldDeploy> {
   const deployerAddress = initialDeployerAddress ?? (await ensureDeployer(client));
 
@@ -77,6 +86,13 @@ export async function deploy({
           config.deploy.upgradeableWorldImplementation,
         );
 
+  const commonDeployOptions = {
+    client,
+    indexerUrl,
+    chainId,
+    worldDeploy,
+  } satisfies CommonDeployOptions;
+
   if (!supportedStoreVersions.includes(worldDeploy.storeVersion)) {
     throw new Error(`Unsupported Store version: ${worldDeploy.storeVersion}`);
   }
@@ -86,7 +102,7 @@ export async function deploy({
 
   const libraryMap = getLibraryMap(libraries);
   await ensureContractsDeployed({
-    client,
+    ...commonDeployOptions,
     deployerAddress,
     contracts: [
       ...libraries.map((library) => ({
@@ -108,24 +124,21 @@ export async function deploy({
   });
 
   const namespaceTxs = await ensureNamespaceOwner({
-    client,
-    worldDeploy,
+    ...commonDeployOptions,
     resourceIds: [...tables.map(({ tableId }) => tableId), ...systems.map(({ systemId }) => systemId)],
   });
   // Wait for namespaces to be available, otherwise referencing them below may fail.
   // This is only here because OPStack chains don't let us estimate gas with pending block tag.
   await waitForTransactions({ client, hashes: namespaceTxs, debugLabel: "namespace registrations" });
 
   const tableTxs = await ensureTables({
-    client,
-    worldDeploy,
+    ...commonDeployOptions,
     tables,
   });
   const systemTxs = await ensureSystems({
-    client,
+    ...commonDeployOptions,
     deployerAddress,
     libraryMap,
-    worldDeploy,
     systems,
   });
   // Wait for tables and systems to be available, otherwise referencing their resource IDs below may fail.
@@ -137,15 +150,13 @@ export async function deploy({
   });
 
   const functionTxs = await ensureFunctions({
-    client,
-    worldDeploy,
+    ...commonDeployOptions,
     functions: systems.flatMap((system) => system.worldFunctions),
   });
   const moduleTxs = await ensureModules({
-    client,
+    ...commonDeployOptions,
     deployerAddress,
     libraryMap,
-    worldDeploy,
     modules,
   });
 
@@ -174,10 +185,9 @@ export async function deploy({
   ]);
 
   const tagTxs = await ensureResourceTags({
-    client,
+    ...commonDeployOptions,
     deployerAddress,
     libraryMap,
-    worldDeploy,
     tags: [...namespaceTags, ...tableTags, ...systemTags],
     valueToHex: stringToHex,
   });

packages/cli/src/deploy/ensureFunctions.ts

@@ -1,24 +1,26 @@
-import { Client, Transport, Chain, Account, Hex } from "viem";
+import { Hex } from "viem";
 import { hexToResource, writeContract } from "@latticexyz/common";
-import { getFunctions } from "@latticexyz/world/internal";
-import { WorldDeploy, WorldFunction, worldAbi } from "./common";
+import { getFunctions } from "@latticexyz/store-sync/world";
+import { CommonDeployOptions, WorldFunction, worldAbi } from "./common";
 import { debug } from "./debug";
 import pRetry from "p-retry";
 
 export async function ensureFunctions({
   client,
   worldDeploy,
   functions,
-}: {
-  readonly client: Client<Transport, Chain | undefined, Account>;
-  readonly worldDeploy: WorldDeploy;
+  indexerUrl,
+  chainId,
+}: CommonDeployOptions & {
   readonly functions: readonly WorldFunction[];
 }): Promise<readonly Hex[]> {
   const worldFunctions = await getFunctions({
     client,
     worldAddress: worldDeploy.address,
     fromBlock: worldDeploy.deployBlock,
     toBlock: worldDeploy.stateBlock,
+    indexerUrl,
+    chainId,
   });
   const worldSelectorToFunction = Object.fromEntries(worldFunctions.map((func) => [func.selector, func]));
 

packages/cli/src/deploy/ensureNamespaceOwner.ts

@@ -1,5 +1,5 @@
-import { Account, Chain, Client, Hex, Transport, getAddress } from "viem";
-import { WorldDeploy, worldAbi } from "./common";
+import { Hex, getAddress } from "viem";
+import { CommonDeployOptions, worldAbi } from "./common";
 import { hexToResource, resourceToHex, writeContract } from "@latticexyz/common";
 import { getResourceIds } from "./getResourceIds";
 import { getTableValue } from "./getTableValue";
@@ -10,13 +10,13 @@ export async function ensureNamespaceOwner({
   client,
   worldDeploy,
   resourceIds,
-}: {
-  readonly client: Client<Transport, Chain | undefined, Account>;
-  readonly worldDeploy: WorldDeploy;
+  indexerUrl,
+  chainId,
+}: CommonDeployOptions & {
   readonly resourceIds: readonly Hex[];
 }): Promise<readonly Hex[]> {
   const desiredNamespaces = Array.from(new Set(resourceIds.map((resourceId) => hexToResource(resourceId).namespace)));
-  const existingResourceIds = await getResourceIds({ client, worldDeploy });
+  const existingResourceIds = await getResourceIds({ client, worldDeploy, indexerUrl, chainId });
   const existingNamespaces = new Set(existingResourceIds.map((resourceId) => hexToResource(resourceId).namespace));
   if (existingNamespaces.size) {
     debug(

packages/cli/src/deploy/ensureResourceTags.ts

@@ -1,5 +1,4 @@
-import { Hex, Client, Transport, Chain, Account, stringToHex, BaseError, concatHex } from "viem";
-import { WorldDeploy } from "./common";
+import { Hex, stringToHex, BaseError, concatHex } from "viem";
 import { debug } from "./debug";
 import { hexToResource, writeContract } from "@latticexyz/common";
 import { identity, isDefined } from "@latticexyz/common/utils";
@@ -11,9 +10,9 @@ import { getContractArtifact } from "../utils/getContractArtifact";
 import { createPrepareDeploy } from "./createPrepareDeploy";
 import { waitForTransactions } from "./waitForTransactions";
 import { LibraryMap } from "./getLibraryMap";
-import { fetchBlockLogs } from "@latticexyz/block-logs-stream";
-import { getStoreLogs, flattenStoreLogs, logToRecord } from "@latticexyz/store/internal";
 import { getKeyTuple, getSchemaPrimitives } from "@latticexyz/protocol-parser/internal";
+import { getRecords } from "@latticexyz/store-sync";
+import { CommonDeployOptions } from "./common";
 
 const metadataModuleArtifact = getContractArtifact(metadataModule);
 
@@ -30,33 +29,28 @@ export async function ensureResourceTags<const value>({
   worldDeploy,
   tags,
   valueToHex = identity,
-}: {
-  readonly client: Client<Transport, Chain | undefined, Account>;
+  indexerUrl,
+  chainId,
+}: CommonDeployOptions & {
   readonly deployerAddress: Hex;
   readonly libraryMap: LibraryMap;
-  readonly worldDeploy: WorldDeploy;
   readonly tags: readonly ResourceTag<value>[];
 } & (value extends Hex
-  ? { readonly valueToHex?: (value: value) => Hex }
-  : { readonly valueToHex: (value: value) => Hex })): Promise<readonly Hex[]> {
+    ? { readonly valueToHex?: (value: value) => Hex }
+    : { readonly valueToHex: (value: value) => Hex })): Promise<readonly Hex[]> {
   debug("ensuring", tags.length, "resource tags");
-
   debug("looking up existing resource tags");
-  const blockLogs = await fetchBlockLogs({
+
+  const { records } = await getRecords({
+    table: metadataConfig.tables.metadata__ResourceTag,
+    worldAddress: worldDeploy.address,
+    chainId,
+    indexerUrl,
+    client,
     fromBlock: worldDeploy.deployBlock,
     toBlock: worldDeploy.stateBlock,
-    maxBlockRange: 100_000n,
-    async getLogs({ fromBlock, toBlock }) {
-      return getStoreLogs(client, {
-        address: worldDeploy.address,
-        fromBlock,
-        toBlock,
-        tableId: metadataConfig.tables.metadata__ResourceTag.tableId,
-      });
-    },
   });
-  const logs = flattenStoreLogs(blockLogs.flatMap((block) => block.logs));
-  const records = logs.map((log) => logToRecord({ log, table: metadataConfig.tables.metadata__ResourceTag }));
+
   debug("found", records.length, "resource tags");
 
   const existingTags = new Map(

packages/cli/src/deploy/ensureSystems.ts

@@ -1,6 +1,6 @@
-import { Client, Transport, Chain, Account, Hex, getAddress, Address } from "viem";
+import { Hex, getAddress, Address } from "viem";
 import { writeContract, resourceToLabel } from "@latticexyz/common";
-import { System, WorldDeploy, worldAbi } from "./common";
+import { CommonDeployOptions, System, worldAbi } from "./common";
 import { debug } from "./debug";
 import { getSystems } from "./getSystems";
 import { getResourceAccess } from "./getResourceAccess";
@@ -16,16 +16,16 @@ export async function ensureSystems({
   libraryMap,
   worldDeploy,
   systems,
-}: {
-  readonly client: Client<Transport, Chain | undefined, Account>;
+  indexerUrl,
+  chainId,
+}: CommonDeployOptions & {
   readonly deployerAddress: Hex;
   readonly libraryMap: LibraryMap;
-  readonly worldDeploy: WorldDeploy;
   readonly systems: readonly System[];
 }): Promise<readonly Hex[]> {
   const [worldSystems, worldAccess] = await Promise.all([
-    getSystems({ client, worldDeploy }),
-    getResourceAccess({ client, worldDeploy }),
+    getSystems({ client, worldDeploy, indexerUrl, chainId }),
+    getResourceAccess({ client, worldDeploy, indexerUrl, chainId }),
   ]);
 
   // Register or replace systems