Merge branch 'main' into holic/codegen-index

latticexyz · Sep 17, 2023 · 3a94da7 · 3a94da7
2 parents b022ca3 + 498d05e
commit 3a94da7
Show file tree

Hide file tree

Showing 25 changed files with 358 additions and 630 deletions.
diff --git a/.changeset/few-jars-turn.md b/.changeset/few-jars-turn.md
@@ -0,0 +1,42 @@
+---
+"@latticexyz/store": major
+"@latticexyz/world": major
+---
+
+Moved the registration of store hooks and systems hooks to bitmaps with bitwise operator instead of a struct.
+
+```diff
+- import { StoreHookLib } from "@latticexyz/src/StoreHook.sol";
++ import {
++   BEFORE_SET_RECORD,
++   BEFORE_SET_FIELD,
++   BEFORE_DELETE_RECORD
++ } from "@latticexyz/store/storeHookTypes.sol";
+
+  StoreCore.registerStoreHook(
+    tableId,
+    subscriber,
+-   StoreHookLib.encodeBitmap({
+-     onBeforeSetRecord: true,
+-     onAfterSetRecord: false,
+-     onBeforeSetField: true,
+-     onAfterSetField: false,
+-     onBeforeDeleteRecord: true,
+-     onAfterDeleteRecord: false
+-   })
++   BEFORE_SET_RECORD | BEFORE_SET_FIELD | BEFORE_DELETE_RECORD
+  );
+```
+
+```diff
+- import { SystemHookLib } from "../src/SystemHook.sol";
++ import { BEFORE_CALL_SYSTEM, AFTER_CALL_SYSTEM } from "../src/systemHookTypes.sol";
+
+  world.registerSystemHook(
+    systemId,
+    subscriber,
+-   SystemHookLib.encodeBitmap({ onBeforeCallSystem: true, onAfterCallSystem: true })
++   BEFORE_CALL_SYSTEM | AFTER_CALL_SYSTEM
+  );
+
+```
diff --git a/.changeset/proud-insects-perform.md b/.changeset/proud-insects-perform.md
@@ -0,0 +1,25 @@
+---
+"@latticexyz/store-indexer": minor
+---
+
+You can now install and run `@latticexyz/store-indexer` from the npm package itself, without having to clone/build the MUD repo:
+
+```sh
+npm install @latticexyz/store-indexer
+
+npm sqlite-indexer
+# or
+npm postgres-indexer
+```
+
+or
+
+```sh
+npx -p @latticexyz/store-indexer sqlite-indexer
+# or
+npx -p @latticexyz/store-indexer postgres-indexer
+```
+
+The binary will also load the nearby `.env` file for easier local configuration.
+
+We've removed the `CHAIN_ID` requirement and instead require just a `RPC_HTTP_URL` or `RPC_WS_URL` or both. You can now also adjust the polling interval with `POLLING_INTERVAL` (defaults to 1000ms, which corresponds to MUD's default block time).
diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts
@@ -1,74 +1,64 @@
+#!/usr/bin/env node
+import "dotenv/config";
 import { z } from "zod";
 import { DefaultLogger, eq } from "drizzle-orm";
 import { createPublicClient, fallback, webSocket, http, Transport } from "viem";
 import fastify from "fastify";
 import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify";
 import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer";
 import { createQueryAdapter } from "../src/postgres/createQueryAdapter";
-import type { Chain } from "viem/chains";
-import * as mudChains from "@latticexyz/common/chains";
-import * as chains from "viem/chains";
-import { isNotNull } from "@latticexyz/common/utils";
+import { isDefined } from "@latticexyz/common/utils";
 import { combineLatest, filter, first } from "rxjs";
 import { drizzle } from "drizzle-orm/postgres-js";
 import postgres from "postgres";
 import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres";
 import { createStoreSync } from "@latticexyz/store-sync";
 
-const possibleChains = Object.values({ ...mudChains, ...chains }) as Chain[];
-
-// TODO: refine zod type to be either CHAIN_ID or RPC_HTTP_URL/RPC_WS_URL
 const env = z
-  .object({
-    CHAIN_ID: z.coerce.number().positive().optional(),
-    RPC_HTTP_URL: z.string().optional(),
-    RPC_WS_URL: z.string().optional(),
-    START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
-    MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
-    HOST: z.string().default("0.0.0.0"),
-    PORT: z.coerce.number().positive().default(3001),
-    DATABASE_URL: z.string(),
-  })
+  .intersection(
+    z.object({
+      HOST: z.string().default("0.0.0.0"),
+      PORT: z.coerce.number().positive().default(3001),
+      DATABASE_URL: z.string(),
+      START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
+      MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
+      POLLING_INTERVAL: z.coerce.number().positive().default(1000),
+    }),
+    z
+      .object({
+        RPC_HTTP_URL: z.string(),
+        RPC_WS_URL: z.string(),
+      })
+      .partial()
+      .refine((values) => Object.values(values).some(isDefined))
+  )
   .parse(process.env, {
     errorMap: (issue) => ({
       message: `Missing or invalid environment variable: ${issue.path.join(".")}`,
     }),
   });
 
-const chain = env.CHAIN_ID != null ? possibleChains.find((c) => c.id === env.CHAIN_ID) : undefined;
-if (env.CHAIN_ID != null && !chain) {
-  console.warn(`No chain found for chain ID ${env.CHAIN_ID}`);
-}
-
 const transports: Transport[] = [
-  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : null,
-  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : null,
-].filter(isNotNull);
+  // prefer WS when specified
+  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,
+  // otherwise use or fallback to HTTP
+  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,
+].filter(isDefined);
 
 const publicClient = createPublicClient({
-  chain,
-  transport: fallback(
-    // If one or more RPC URLs are provided, we'll configure the transport with only those RPC URLs
-    transports.length > 0
-      ? transports
-      : // Otherwise use the chain defaults
-        [webSocket(), http()]
-  ),
-  pollingInterval: 1000,
+  transport: fallback(transports),
+  pollingInterval: env.POLLING_INTERVAL,
 });
 
-// Fetch the chain ID from the RPC if no chain object was found for the provided chain ID.
-// We do this to match the downstream logic, which also attempts to find the chain ID.
-const chainId = chain?.id ?? (await publicClient.getChainId());
-
+const chainId = await publicClient.getChainId();
 const database = drizzle(postgres(env.DATABASE_URL), {
   logger: new DefaultLogger(),
 });
 
-let startBlock = env.START_BLOCK;
-
 const { storageAdapter, internalTables } = await postgresStorage({ database, publicClient });
 
+let startBlock = env.START_BLOCK;
+
 // Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.
 try {
   const currentChainStates = await database

diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts
@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+import "dotenv/config";
 import fs from "node:fs";
 import { z } from "zod";
 import { eq } from "drizzle-orm";
@@ -9,58 +11,46 @@ import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify";
 import { AppRouter, createAppRouter } from "@latticexyz/store-sync/trpc-indexer";
 import { chainState, schemaVersion, syncToSqlite } from "@latticexyz/store-sync/sqlite";
 import { createQueryAdapter } from "../src/sqlite/createQueryAdapter";
-import type { Chain } from "viem/chains";
-import * as mudChains from "@latticexyz/common/chains";
-import * as chains from "viem/chains";
-import { isNotNull } from "@latticexyz/common/utils";
+import { isDefined } from "@latticexyz/common/utils";
 import { combineLatest, filter, first } from "rxjs";
 
-const possibleChains = Object.values({ ...mudChains, ...chains }) as Chain[];
-
-// TODO: refine zod type to be either CHAIN_ID or RPC_HTTP_URL/RPC_WS_URL
 const env = z
-  .object({
-    CHAIN_ID: z.coerce.number().positive().optional(),
-    RPC_HTTP_URL: z.string().optional(),
-    RPC_WS_URL: z.string().optional(),
-    START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
-    MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
-    HOST: z.string().default("0.0.0.0"),
-    PORT: z.coerce.number().positive().default(3001),
-    SQLITE_FILENAME: z.string().default("indexer.db"),
-  })
+  .intersection(
+    z.object({
+      HOST: z.string().default("0.0.0.0"),
+      PORT: z.coerce.number().positive().default(3001),
+      SQLITE_FILENAME: z.string().default("indexer.db"),
+      START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
+      MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
+      POLLING_INTERVAL: z.coerce.number().positive().default(1000),
+    }),
+    z
+      .object({
+        RPC_HTTP_URL: z.string(),
+        RPC_WS_URL: z.string(),
+      })
+      .partial()
+      .refine((values) => Object.values(values).some(isDefined))
+  )
   .parse(process.env, {
     errorMap: (issue) => ({
       message: `Missing or invalid environment variable: ${issue.path.join(".")}`,
     }),
   });
 
-const chain = env.CHAIN_ID != null ? possibleChains.find((c) => c.id === env.CHAIN_ID) : undefined;
-if (env.CHAIN_ID != null && !chain) {
-  console.warn(`No chain found for chain ID ${env.CHAIN_ID}`);
-}
-
 const transports: Transport[] = [
-  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : null,
-  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : null,
-].filter(isNotNull);
+  // prefer WS when specified
+  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,
+  // otherwise use or fallback to HTTP
+  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,
+].filter(isDefined);
 
 const publicClient = createPublicClient({
-  chain,
-  transport: fallback(
-    // If one or more RPC URLs are provided, we'll configure the transport with only those RPC URLs
-    transports.length > 0
-      ? transports
-      : // Otherwise use the chain defaults
-        [webSocket(), http()]
-  ),
-  pollingInterval: 1000,
+  transport: fallback(transports),
+  pollingInterval: env.POLLING_INTERVAL,
 });
 
-// Fetch the chain ID from the RPC if no chain object was found for the provided chain ID.
-// We do this to match the downstream logic, which also attempts to find the chain ID.
-const chainId = chain?.id ?? (await publicClient.getChainId());
-
+const chainId = await publicClient.getChainId();
 const database = drizzle(new Database(env.SQLITE_FILENAME));
 
 let startBlock = env.START_BLOCK;

diff --git a/packages/store-indexer/package.json b/packages/store-indexer/package.json
@@ -13,6 +13,10 @@
     ".": "./dist/index.js"
   },
   "types": "src/index.ts",
+  "bin": {
+    "postgres-indexer": "./dist/bin/postgres-indexer.js",
+    "sqlite-indexer": "./dist/bin/sqlite-indexer.js"
+  },
   "scripts": {
     "build": "pnpm run build:js",
     "build:js": "tsup",
@@ -42,6 +46,7 @@
     "@wagmi/chains": "^0.2.22",
     "better-sqlite3": "^8.6.0",
     "debug": "^4.3.4",
+    "dotenv": "^16.0.3",
     "drizzle-orm": "^0.28.5",
     "fastify": "^4.21.0",
     "postgres": "^3.3.5",

diff --git a/packages/store-indexer/tsup.config.ts b/packages/store-indexer/tsup.config.ts
@@ -1,7 +1,7 @@
 import { defineConfig } from "tsup";
 
 export default defineConfig({
-  entry: ["src/index.ts"],
+  entry: ["src/index.ts", "bin/postgres-indexer.ts", "bin/sqlite-indexer.ts"],
   target: "esnext",
   format: ["esm"],
   dts: false,

diff --git a/packages/store/gas-report.json b/packages/store/gas-report.json
@@ -345,7 +345,7 @@
     "file": "test/Hook.t.sol",
     "test": "testIsEnabled",
     "name": "check if hook is enabled",
-    "gasUsed": 114
+    "gasUsed": 108
   },
   {
     "file": "test/KeyEncoding.t.sol",
@@ -681,49 +681,49 @@
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooks",
     "name": "register subscriber",
-    "gasUsed": 60413
+    "gasUsed": 60150
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooks",
     "name": "set record on table with subscriber",
-    "gasUsed": 71044
+    "gasUsed": 71004
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooks",
     "name": "set static field on table with subscriber",
-    "gasUsed": 20820
+    "gasUsed": 20780
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooks",
     "name": "delete record on table with subscriber",
-    "gasUsed": 16438
+    "gasUsed": 16398
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooksDynamicData",
     "name": "register subscriber",
-    "gasUsed": 60413
+    "gasUsed": 60150
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooksDynamicData",
     "name": "set (dynamic) record on table with subscriber",
-    "gasUsed": 164166
+    "gasUsed": 164126
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooksDynamicData",
     "name": "set (dynamic) field on table with subscriber",
-    "gasUsed": 24020
+    "gasUsed": 23980
   },
   {
     "file": "test/StoreCoreGas.t.sol",
     "test": "testHooksDynamicData",
     "name": "delete (dynamic) record on table with subscriber",
-    "gasUsed": 17423
+    "gasUsed": 17383
   },
   {
     "file": "test/StoreCoreGas.t.sol",
@@ -891,31 +891,25 @@
     "file": "test/StoreHook.t.sol",
     "test": "testCallHook",
     "name": "call an enabled hook",
-    "gasUsed": 14598
+    "gasUsed": 14588
   },
   {
     "file": "test/StoreHook.t.sol",
     "test": "testCallHook",
     "name": "call a disabled hook",
-    "gasUsed": 133
+    "gasUsed": 123
   },
   {
     "file": "test/StoreHook.t.sol",
     "test": "testGetAddress",
     "name": "get store hook address",
     "gasUsed": 1
   },
-  {
-    "file": "test/StoreHook.t.sol",
-    "test": "testGetBitmap",
-    "name": "get store hook bitmap",
-    "gasUsed": 1
-  },
   {
     "file": "test/StoreHook.t.sol",
     "test": "testIsEnabled",
     "name": "check if store hook is enabled",
-    "gasUsed": 129
+    "gasUsed": 108
   },
   {
     "file": "test/StoreSwitch.t.sol",