Skip to content

Commit

Permalink
update store-sync
Browse files Browse the repository at this point in the history
  • Loading branch information
holic committed Mar 20, 2024
1 parent 994f024 commit 57cb281
Show file tree
Hide file tree
Showing 16 changed files with 56 additions and 136 deletions.
11 changes: 6 additions & 5 deletions packages/store-sync/src/common.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import { Address, Block, Hex, Log, PublicClient } from "viem";
import { StoreConfig, StoreEventsAbiItem, StoreEventsAbi, resolveConfig } from "@latticexyz/store";
import { StoreEventsAbiItem, StoreEventsAbi, resolveConfig } from "@latticexyz/store";
import { Observable } from "rxjs";
import { UnionPick } from "@latticexyz/common/type-utils";
import { KeySchema, TableRecord, ValueSchema } from "@latticexyz/protocol-parser/internal";
import storeConfig from "@latticexyz/store/mud.config";
import worldConfig from "@latticexyz/world/mud.config";
import { flattenSchema } from "./flattenSchema";
import { Store as StoreConfig, storeToV1 } from "@latticexyz/store/config/v2";

/** @internal Temporary workaround until we redo our config parsing and can pull this directly from the config (https://github.com/latticexyz/mud/issues/1668) */
export const storeTables = resolveConfig(storeConfig).tables;
export const storeTables = resolveConfig(storeToV1(storeConfig)).tables;
/** @internal Temporary workaround until we redo our config parsing and can pull this directly from the config (https://github.com/latticexyz/mud/issues/1668) */
export const worldTables = resolveConfig(worldConfig).tables;
export const worldTables = resolveConfig(storeToV1(worldConfig)).tables;

export const internalTableIds = [...Object.values(storeTables), ...Object.values(worldTables)].map(
(table) => table.tableId,
Expand Down Expand Up @@ -54,11 +55,11 @@ export type SyncFilter = {
key1?: Hex;
};

export type SyncOptions<TConfig extends StoreConfig = StoreConfig> = {
export type SyncOptions<config extends StoreConfig = StoreConfig> = {
/**
* MUD config
*/
config?: TConfig;
config?: config;
/**
* [viem `PublicClient`][0] used for fetching logs from the RPC.
*
Expand Down
9 changes: 5 additions & 4 deletions packages/store-sync/src/createStoreSync.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { StoreConfig, storeEventsAbi } from "@latticexyz/store";
import { storeEventsAbi } from "@latticexyz/store";
import { Hex, TransactionReceiptNotFoundError } from "viem";
import {
StorageAdapter,
Expand Down Expand Up @@ -32,12 +32,13 @@ import { SyncStep } from "./SyncStep";
import { bigIntMax, chunk, isDefined, waitForIdle } from "@latticexyz/common/utils";
import { getSnapshot } from "./getSnapshot";
import { fetchAndStoreLogs } from "./fetchAndStoreLogs";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";

const debug = parentDebug.extend("createStoreSync");

const defaultFilters: SyncFilter[] = internalTableIds.map((tableId) => ({ tableId }));

type CreateStoreSyncOptions<TConfig extends StoreConfig = StoreConfig> = SyncOptions<TConfig> & {
type CreateStoreSyncOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
storageAdapter: StorageAdapter;
onProgress?: (opts: {
step: SyncStep;
Expand All @@ -48,7 +49,7 @@ type CreateStoreSyncOptions<TConfig extends StoreConfig = StoreConfig> = SyncOpt
}) => void;
};

export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>({
export async function createStoreSync<config extends StoreConfig = StoreConfig>({
storageAdapter,
onProgress,
publicClient,
Expand All @@ -61,7 +62,7 @@ export async function createStoreSync<TConfig extends StoreConfig = StoreConfig>
initialState,
initialBlockLogs,
indexerUrl,
}: CreateStoreSyncOptions<TConfig>): Promise<SyncResult> {
}: CreateStoreSyncOptions<config>): Promise<SyncResult> {
const filters: SyncFilter[] =
initialFilters.length || tableIds.length
? [...initialFilters, ...tableIds.map((tableId) => ({ tableId })), ...defaultFilters]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Hex, PublicClient, concatHex, getAddress } from "viem";
import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core";
import { and, eq } from "drizzle-orm";
import { buildTable } from "./buildTable";
import { StoreConfig } from "@latticexyz/store";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";
import { debug } from "./debug";
import { StorageAdapter, StorageAdapterBlock } from "../common";
import { isTableRegistrationLog } from "../isTableRegistrationLog";
Expand All @@ -22,14 +22,14 @@ export type PostgresStorageAdapter = {
cleanUp: () => Promise<void>;
};

export async function createStorageAdapter<TConfig extends StoreConfig = StoreConfig>({
export async function createStorageAdapter<config extends StoreConfig = StoreConfig>({
database,
publicClient,
config,
}: {
database: PgDatabase<QueryResultHKT>;
publicClient: PublicClient;
config?: TConfig;
config?: config;
}): Promise<PostgresStorageAdapter> {
const bytesStorageAdapter = await createBytesStorageAdapter({ database, publicClient, config });
const cleanUp: (() => Promise<void>)[] = [];
Expand Down
8 changes: 4 additions & 4 deletions packages/store-sync/src/postgres-decoded/syncToPostgres.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { StoreConfig } from "@latticexyz/store";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";
import { PgDatabase } from "drizzle-orm/pg-core";
import { SyncOptions, SyncResult } from "../common";
import { createStorageAdapter } from "./createStorageAdapter";
import { createStoreSync } from "../createStoreSync";

type SyncToPostgresOptions<TConfig extends StoreConfig = StoreConfig> = SyncOptions<TConfig> & {
type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
/**
* [Postgres database object from Drizzle][0].
*
Expand All @@ -25,13 +25,13 @@ type SyncToPostgresResult = SyncResult & {
* @param {CreateIndexerOptions} options See `CreateIndexerOptions`.
* @returns A function to unsubscribe from the block stream, effectively stopping the indexer.
*/
export async function syncToPostgres<TConfig extends StoreConfig = StoreConfig>({
export async function syncToPostgres<config extends StoreConfig = StoreConfig>({
config,
database,
publicClient,
startSync = true,
...syncOptions
}: SyncToPostgresOptions<TConfig>): Promise<SyncToPostgresResult> {
}: SyncToPostgresOptions<config>): Promise<SyncToPostgresResult> {
const { storageAdapter } = await createStorageAdapter({ database, publicClient, config });
const storeSync = await createStoreSync({
storageAdapter,
Expand Down
6 changes: 3 additions & 3 deletions packages/store-sync/src/postgres/createStorageAdapter.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { PublicClient, encodePacked, size } from "viem";
import { PgDatabase, QueryResultHKT } from "drizzle-orm/pg-core";
import { and, eq } from "drizzle-orm";
import { StoreConfig } from "@latticexyz/store";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";
import { debug } from "./debug";
import { tables } from "./tables";
import { spliceHex } from "@latticexyz/common";
Expand All @@ -17,13 +17,13 @@ export type PostgresStorageAdapter = {
cleanUp: () => Promise<void>;
};

export async function createStorageAdapter<TConfig extends StoreConfig = StoreConfig>({
export async function createStorageAdapter<config extends StoreConfig = StoreConfig>({
database,
publicClient,
}: {
database: PgDatabase<QueryResultHKT>;
publicClient: PublicClient;
config?: TConfig;
config?: config;
}): Promise<PostgresStorageAdapter> {
const cleanUp: (() => Promise<void>)[] = [];

Expand Down
8 changes: 4 additions & 4 deletions packages/store-sync/src/postgres/syncToPostgres.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { StoreConfig } from "@latticexyz/store";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";
import { PgDatabase } from "drizzle-orm/pg-core";
import { SyncOptions, SyncResult } from "../common";
import { createStorageAdapter } from "./createStorageAdapter";
import { createStoreSync } from "../createStoreSync";

type SyncToPostgresOptions<TConfig extends StoreConfig = StoreConfig> = SyncOptions<TConfig> & {
type SyncToPostgresOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
/**
* [Postgres database object from Drizzle][0].
*
Expand All @@ -25,13 +25,13 @@ type SyncToPostgresResult = SyncResult & {
* @param {CreateIndexerOptions} options See `CreateIndexerOptions`.
* @returns A function to unsubscribe from the block stream, effectively stopping the indexer.
*/
export async function syncToPostgres<TConfig extends StoreConfig = StoreConfig>({
export async function syncToPostgres<config extends StoreConfig = StoreConfig>({
config,
database,
publicClient,
startSync = true,
...syncOptions
}: SyncToPostgresOptions<TConfig>): Promise<SyncToPostgresResult> {
}: SyncToPostgresOptions<config>): Promise<SyncToPostgresResult> {
const { storageAdapter } = await createStorageAdapter({ database, publicClient, config });
const storeSync = await createStoreSync({
storageAdapter,
Expand Down
3 changes: 2 additions & 1 deletion packages/store-sync/src/recs/recsStorage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import { StoreEventsLog } from "../common";
import { singletonEntity } from "./singletonEntity";
import { RpcLog, formatLog, decodeEventLog, Hex } from "viem";
import { resolveConfig, storeEventsAbi } from "@latticexyz/store";
import { storeToV1 } from "@latticexyz/store/config/v2";

const tables = resolveConfig(mudConfig).tables;
const tables = resolveConfig(storeToV1(mudConfig)).tables;

// TODO: make test-data a proper package and export this
const blocks = groupLogsByBlockNumber(
Expand Down
5 changes: 3 additions & 2 deletions packages/store-sync/src/recs/recsStorage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@ import { singletonEntity } from "./singletonEntity";
import storeConfig from "@latticexyz/store/mud.config";
import worldConfig from "@latticexyz/world/mud.config";
import { TablesToComponents, tablesToComponents } from "./tablesToComponents";
import { storeToV1 } from "@latticexyz/store/config/v2";

const storeTables = resolveConfig(storeConfig).tables;
const worldTables = resolveConfig(worldConfig).tables;
const storeTables = resolveConfig(storeToV1(storeConfig)).tables;
const worldTables = resolveConfig(storeToV1(worldConfig)).tables;

export type RecsStorageOptions<tables extends Record<string, Table>> = {
world: RecsWorld;
Expand Down
14 changes: 9 additions & 5 deletions packages/store-sync/src/recs/syncToRecs.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,24 @@
import { StoreConfig, Table, ResolvedStoreConfig, resolveConfig } from "@latticexyz/store";
import { Table, ResolvedStoreConfig, resolveConfig } from "@latticexyz/store";
import { Store as StoreConfig, storeToV1 } from "@latticexyz/store/config/v2";
import { Component as RecsComponent, World as RecsWorld, getComponentValue, setComponent } from "@latticexyz/recs";
import { SyncOptions, SyncResult } from "../common";
import { RecsStorageAdapter, recsStorage } from "./recsStorage";
import { createStoreSync } from "../createStoreSync";
import { singletonEntity } from "./singletonEntity";
import { SyncStep } from "../SyncStep";

type SyncToRecsOptions<config extends StoreConfig, extraTables extends Record<string, Table>> = SyncOptions<config> & {
type SyncToRecsOptions<config extends StoreConfig, extraTables extends Record<string, Table>> = Omit<
SyncOptions<config>,
"config"
> & {
world: RecsWorld;
config: config;
tables?: extraTables;
startSync?: boolean;
};

type SyncToRecsResult<config extends StoreConfig, extraTables extends Record<string, Table>> = SyncResult & {
components: RecsStorageAdapter<ResolvedStoreConfig<config>["tables"] & extraTables>["components"];
components: RecsStorageAdapter<ResolvedStoreConfig<storeToV1<config>>["tables"] & extraTables>["components"];
stopSync: () => void;
};

Expand All @@ -26,9 +30,9 @@ export async function syncToRecs<config extends StoreConfig, extraTables extends
...syncOptions
}: SyncToRecsOptions<config, extraTables>): Promise<SyncToRecsResult<config, extraTables>> {
const tables = {
...resolveConfig(config).tables,
...resolveConfig(storeToV1(config as StoreConfig)).tables,
...extraTables,
} as ResolvedStoreConfig<config>["tables"] & extraTables;
} as ResolvedStoreConfig<storeToV1<config>>["tables"] & extraTables;

const { storageAdapter, components } = recsStorage({
world,
Expand Down
6 changes: 3 additions & 3 deletions packages/store-sync/src/sqlite/sqliteStorage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
import { and, eq, sql } from "drizzle-orm";
import { sqliteTableToSql } from "./sqliteTableToSql";
import { buildTable } from "./buildTable";
import { StoreConfig } from "@latticexyz/store";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";
import { debug } from "./debug";
import { getTableName } from "./getTableName";
import { chainState, mudStoreTables } from "./internalTables";
Expand All @@ -17,13 +17,13 @@ import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser/internal

// TODO: upgrade drizzle and use async sqlite interface for consistency

export async function sqliteStorage<TConfig extends StoreConfig = StoreConfig>({
export async function sqliteStorage<config extends StoreConfig = StoreConfig>({
database,
publicClient,
}: {
database: BaseSQLiteDatabase<"sync", void>;
publicClient: PublicClient;
config?: TConfig;
config?: config;
}): Promise<StorageAdapter> {
const chainId = publicClient.chain?.id ?? (await publicClient.getChainId());

Expand Down
8 changes: 4 additions & 4 deletions packages/store-sync/src/sqlite/syncToSqlite.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { StoreConfig } from "@latticexyz/store";
import { Store as StoreConfig } from "@latticexyz/store/config/v2";
import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
import { SyncOptions, SyncResult } from "../common";
import { sqliteStorage } from "./sqliteStorage";
import { createStoreSync } from "../createStoreSync";

type SyncToSqliteOptions<TConfig extends StoreConfig = StoreConfig> = SyncOptions<TConfig> & {
type SyncToSqliteOptions<config extends StoreConfig = StoreConfig> = SyncOptions<config> & {
/**
* [SQLite database object from Drizzle][0].
*
Expand All @@ -25,13 +25,13 @@ type SyncToSqliteResult = SyncResult & {
* @param {SyncToSqliteOptions} options See `SyncToSqliteOptions`.
* @returns A function to unsubscribe from the block stream, effectively stopping the indexer.
*/
export async function syncToSqlite<TConfig extends StoreConfig = StoreConfig>({
export async function syncToSqlite<config extends StoreConfig = StoreConfig>({
config,
database,
publicClient,
startSync = true,
...syncOptions
}: SyncToSqliteOptions<TConfig>): Promise<SyncToSqliteResult> {
}: SyncToSqliteOptions<config>): Promise<SyncToSqliteResult> {
const storeSync = await createStoreSync({
storageAdapter: await sqliteStorage({ database, publicClient, config }),
config,
Expand Down
1 change: 0 additions & 1 deletion packages/store-sync/src/zustand/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,4 @@ export * from "./common";
export * from "./createStorageAdapter";
export * from "./createStore";
export * from "./getId";
export * from "./logToTable";
export * from "./syncToZustand";
52 changes: 0 additions & 52 deletions packages/store-sync/src/zustand/logToTable.test.ts

This file was deleted.

35 changes: 0 additions & 35 deletions packages/store-sync/src/zustand/logToTable.ts

This file was deleted.

Loading

0 comments on commit 57cb281

Please sign in to comment.