feat: support partial indexing for rich-indexer

.changeset/light-items-own.md

@@ -0,0 +1,5 @@
+---
+"@ckb-lumos/codec": minor
+---
+
+feat: added indexOf for searching bytes subset

.changeset/red-bulldogs-punch.md

@@ -0,0 +1,7 @@
+---
+"@ckb-lumos/ckb-indexer": minor
+"@ckb-lumos/base": minor
+"@ckb-lumos/rpc": minor
+---
+
+feat: support partial indexing for rich-indexer

.eslintrc.next.js

@@ -23,6 +23,7 @@ module.exports = {
               -1, // index -1 is not found
               0, // first element of an array
               1, // common for i + 1 in a loop
+              2, // slice(2) for string that starts with "0x" is common to work with 3rd-party libs
               16, // toString(16)
               1000, // second to millisecond
             ],

packages/base/src/indexer.ts

@@ -6,7 +6,7 @@ import { Hexadecimal, HexString } from "./primitive";
 import { Logger } from "./logger";
 import { isScriptWrapper } from "./helpers";
 
-export type SearchMode = "exact" | "prefix";
+export type SearchMode = "exact" | "prefix" | "partial";
 
 export type DataWithSearchMode = {
   searchMode: SearchMode;

packages/ckb-indexer/src/ckbIndexerFilter.ts

@@ -26,6 +26,7 @@
   | "outputDataLenRange"
   | "outputCapacityRange"
   | "scriptLenRange"
+  | "scriptSearchMode"
 >;
 
 /**
@@ -71,7 +72,7 @@
       scriptType: "lock",
       scriptSearchMode: instanceOfScriptWrapper(queryLock)
         ? queryLock.searchMode || "prefix"
-        : "prefix",
+        : queryOptions.scriptSearchMode || "prefix",
       filter: {},
     };
     searchKeyType && (searchKey.filter.script = searchKeyType);
@@ -81,7 +82,7 @@
       scriptType: "type",
       scriptSearchMode: instanceOfScriptWrapper(queryType)
         ? queryType.searchMode || "prefix"
-        : "prefix",
+        : queryOptions.scriptSearchMode || "prefix",
       filter: {},
     };
   } else {
@@ -139,9 +140,21 @@
           .slice(0, expectPrefix.length);
         return bytes.equal(expectPrefix, actualPrefix);
       });
-    } else {
+    } else if (
+      instanceOfDataWithSearchMode(options.data) &&
+      options.data.searchMode === "partial"
+    ) {
+      const search: DataWithSearchMode = options.data;
+      filteredCells = filteredCells.filter((cell) =>
+        bytes.indexOf(cell.data, search.data)
+      );
+    } else if (
+      typeof options.data === "string" &&
+      options.data.startsWith("0x")
+    ) {
+      const searchBytes = options.data;
       filteredCells = filteredCells.filter((cell) => {
-        const expectPrefix = bytes.bytify(options.data as string);
+        const expectPrefix = bytes.bytify(searchBytes);
         const actualPrefix = bytes
           .bytify(cell.data)
           .slice(0, expectPrefix.length);
@@ -160,12 +173,11 @@
   cell: Cell,
   searchKey: LumosSearchKey
 ): boolean {
-  const isExactMode = searchKey.scriptSearchMode === "exact";
   const { cellOutput } = cell;
   const { scriptType, script, filter } = searchKey;
 
   // Search mode
-  if (isExactMode) {
+  if (searchKey.scriptSearchMode === "exact") {
     if (scriptType === "lock") {
       if (
         !bytes.equal(
@@ -186,8 +198,19 @@
         return false;
       }
     }
-    // Prefix mode
+    // partial mode
+  } else if (searchKey.scriptSearchMode === "partial") {
+    if (scriptType === "lock") {
+      if (!checkScriptWithPartialMode(cellOutput.lock, script)) {
+        return false;
+      }
+    } else {
+      if (!checkScriptWithPartialMode(cellOutput.type, script)) {
+        return false;
+      }
+    }
   } else {
+    // Prefix mode
     if (scriptType === "lock") {
       if (!checkScriptWithPrefixMode(cellOutput.lock, script)) {
         return false;
@@ -284,6 +307,24 @@
   return true;
 }
 
+function checkScriptWithPartialMode(
+  script: Script | undefined,
+  filterScript: Script
+): boolean {
+  if (!script) return false;
+
+  // codeHash should always be 32 bytes, so it only supports exact match mode
+  if (!bytes.equal(filterScript.codeHash, script.codeHash)) {
+    return false;
+  }
+
+  if (script.hashType !== filterScript.hashType) {
+    return false;
+  }
+
+  return bytes.indexOf(script.args, filterScript.args) > -1;
+}
+
 function checkScriptLenRange(
   script: Script | undefined,
   scriptLenRange: HexadecimalRange

packages/ckb-indexer/src/collector.ts

@@ -141,6 +141,7 @@
       if (queryLock) {
         if (instanceOfScriptWrapper(queryLock)) {
           validators.ValidateScript(queryLock.script);
+          query.scriptSearchMode = queryLock.searchMode;
           query.lock = queryLock.script;
         }
       }
@@ -152,18 +153,24 @@
           instanceOfScriptWrapper(query.type)
         ) {
           validators.ValidateScript(query.type.script);
+          query.scriptSearchMode = query.type.searchMode;
           query.type = query.type.script;
         }
       }
 
-      if (!query.outputDataLenRange) {
-        if (query.data && query.data !== "any") {
-          const dataLenRange = getHexStringBytes(unwrapDataWrapper(query.data));
-          query.outputDataLenRange = [
-            "0x" + dataLenRange.toString(16),
-            "0x" + (dataLenRange + 1).toString(16),
-          ];
-        }
+      // set data range to narrow the search result if the data is set
+      if (
+        !query.outputDataLenRange &&
+        query.data &&
+        query.data !== "any" &&
+        // the default data search filter mode is exact search that is the same as ckb-indexer
+        (typeof query.data === "string" || query.data.searchMode === "exact")
+      ) {
+        const dataLenRange = getHexStringBytes(unwrapDataWrapper(query.data));
+        query.outputDataLenRange = [
+          "0x" + dataLenRange.toString(16),
+          "0x" + (dataLenRange + 1).toString(16),
+        ];
       }
 
       if (!query.scriptLenRange && query.type === "empty") {
@@ -207,7 +214,10 @@
         "Content-Type": "application/json",
       },
     });
-    if (res.status !== 200) {
+
+    const HTTP_SUCCESS_STATUS = 200;
+
+    if (res.status !== HTTP_SUCCESS_STATUS) {
       throw new Error(`indexer request failed with HTTP code ${res.status}`);
     }
     const result = await res.json();

packages/ckb-indexer/src/resultFormatter.ts

@@ -29,6 +29,8 @@ const toSearchFilter = (data: RPCType.SearchFilter): SearchFilter => {
     outputCapacityRange: data.output_capacity_range,
     scriptLenRange: data.script_len_range,
     blockRange: data.block_range,
+    outputData: data.output_data,
+    outputDataFilterMode: data.output_data_filter_mode,
   };
 };
 

packages/ckb-indexer/src/rpcType.ts

@@ -27,10 +27,13 @@ export type CellOutput = {
 
 export type HexadecimalRange = [Hexadecimal, Hexadecimal];
 export type ScriptType = "type" | "lock";
-export type ScriptSearchMode = "prefix" | "exact";
+export type ScriptSearchMode = "prefix" | "exact" | "partial";
+export type OutputDataFilterMode = "prefix" | "exact" | "partial";
 
 export interface SearchFilter {
   script?: Script;
+  output_data?: HexString;
+  output_data_filter_mode?: OutputDataFilterMode;
   output_data_len_range?: HexadecimalRange; //empty
   output_capacity_range?: HexadecimalRange; //empty
   block_range?: HexadecimalRange; //fromBlock-toBlock

packages/ckb-indexer/src/services.ts

@@ -51,6 +51,14 @@
   if (queries.scriptSearchMode) {
     script_search_mode = queries.scriptSearchMode;
   }
+  if (queries.data) {
+    if (typeof queries.data === "object") {
+      filter.output_data_filter_mode = queries.data.searchMode;
+      filter.output_data = queries.data.data;
+    } else if (typeof queries.data === "string") {
+      filter.output_data = queries.data;
+    }
+  }
   if (!script) {
     throw new Error("Either lock or type script must be provided!");
   }
@@ -88,7 +96,8 @@
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify(data.map((item) => ({ id: id++, ...item }))),
   });
-  if (res.status !== 200) {
+  const HTTP_SUCCESS_STATUS = 200;
+  if (res.status !== HTTP_SUCCESS_STATUS) {
     throw new Error(`Indexer request failed with HTTP code ${res.status}`);
   }
   const result = await res.json();

packages/ckb-indexer/src/type.ts

@@ -15,7 +15,8 @@ import { BIish } from "@ckb-lumos/bi";
 
 export type ScriptType = "type" | "lock";
 export type Order = "asc" | "desc";
-export type ScriptSearchMode = "prefix" | "exact";
+export type ScriptSearchMode = "prefix" | "exact" | "partial";
+export type OutputDataFilterMode = "prefix" | "exact" | "partial";
 
 export interface CKBIndexerQueryOptions extends QueryOptions {
   outputDataLenRange?: HexadecimalRange;
@@ -31,6 +32,8 @@ export type HexadecimalRange = [Hexadecimal, Hexadecimal];
 export interface SearchFilter {
   script?: Script;
   scriptLenRange?: HexadecimalRange;
+  outputData?: HexString;
+  outputDataFilterMode?: OutputDataFilterMode;
   outputDataLenRange?: HexadecimalRange; //empty
   outputCapacityRange?: HexadecimalRange; //empty
   blockRange?: HexadecimalRange; //fromBlock-toBlock

packages/ckb-indexer/tests/resultFormatter.unit.test.ts

@@ -31,6 +31,8 @@ test("should toSearchKey works fine", async (t) => {
       outputCapacityRange: undefined,
       scriptLenRange: undefined,
       blockRange: undefined,
+      outputData: undefined,
+      outputDataFilterMode: undefined,
       outputDataLenRange: ["0x1", "0x2"],
     },
   };

packages/ckb-indexer/tests/service.unit.test.ts

@@ -17,6 +17,8 @@ test("should generateSearchKey works fine", async (t) => {
       outputCapacityRange: undefined,
       blockRange: undefined,
       scriptLenRange: undefined,
+      outputData: undefined,
+      outputDataFilterMode: undefined,
       outputDataLenRange: ["0x1", "0x2"],
     },
   };