Scan proxy pallet business, #919 (#922)

* Handle proxy added and removed event, #919

* Save known heights, #919

* Handle removeProxies call, #919

* Improve removeProxies handling, #919

* Handle PureCreated event, #919

* Handle killPure call, #919

* Handle announce created and removed, #919

* Handle rejectAnnouncement call, #919

* Handle ProxyExecuted event, #919

* Scan known heights first if set, #919

* Add code to clean scanned business, #919

* Handle addProxy call without event, #919

* Mark known height for addProxy call, #919

* Adapt proxy storage, #919

* Handle removeProxy call without event, #919

* Handle AnonymousCreated event, #919

* Handle killAnonymous call, #919

* Revert yarn.lock change

* Update yarn.lock

* re-create lock file

* Revert "re-create lock file"

This reverts commit 5da8b8e.

* ci: bump version

* try back to yarn classic

* try yarn 3.6.4

* Init proxy type and resolvers for graphql server, #919

* Add graphql query for proxy timeline, #919

* Add graphql query for proxy calls, #919

* Add graphql query for proxy announcements, #919

* Guard proxy differences log, #919

* Add proxy prefix to announcement queries, #919

* Add proxy scan height to query, #919

* Save proxy executed event data, #919

---------

Co-authored-by: 2nthony <[email protected]>

.github/workflows/ci.yml

@@ -13,12 +13,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Set Node.js 18.x
-        uses: actions/setup-node@v3
+      - name: Set Node.js 20.x
+        uses: actions/setup-node@v4
         with:
-          node-version: 18.x
+          node-version: 20.x
           check-latest: true
 
       - name: Run backend install

backend/.yarnrc.yml

@@ -1 +1 @@
-yarnPath: .yarn/releases/yarn-3.6.0.cjs
+yarnPath: .yarn/releases/yarn-3.6.4.cjs

backend/package.json

@@ -24,5 +24,5 @@
   "devDependencies": {
     "eslint": "^8.31.0"
   },
-  "packageManager": "[email protected].0"
+  "packageManager": "[email protected].4"
 }

backend/packages/common/src/call/findTargetCallAuthor.js

@@ -0,0 +1,69 @@
+const { emptyFn } = require("@osn/scan-common/src/utils/emptyFn");
+const {
+  Modules,
+  ProxyMethods,
+  MultisigMethods,
+  UtilityMethods,
+} = require("@osn/scan-common/src/consts");
+const { calcMultisigAddress } = require("@osn/scan-common/src/utils/multisig");
+
+function findAuthorFromProxy(proxyCall, author, checkFn) {
+  const real = proxyCall.args[0].toString();
+  const innerCall = proxyCall.args[2];
+  return findCallAuthor(innerCall, real, checkFn);
+}
+
+function findAuthorFromMultisig(multisigCall, author, checkFn) {
+  const innerCall = multisigCall.args[3];
+  const threshold = multisigCall.args[0].toNumber();
+  const otherSignatories = multisigCall.args[1].toJSON();
+  const multisigAddr = calcMultisigAddress(
+    [author, ...otherSignatories],
+    threshold,
+    multisigCall.registry.chainSS58,
+  );
+
+  return findCallAuthor(innerCall, multisigAddr, checkFn);
+}
+
+function findAuthorFromBatch(batchCall, author, checkFn) {
+  for (const innerCall of batchCall.args[0]) {
+    const real = findCallAuthor(innerCall, author, checkFn);
+    if (real) {
+      return real;
+    }
+  }
+
+  return null;
+}
+
+function findCallAuthor(call, author, checkFn = emptyFn) {
+  if (checkFn(call)) {
+    return author;
+  }
+
+  const { section, method } = call;
+  if (Modules.Proxy === section && ProxyMethods.proxy === method) {
+    return findAuthorFromProxy(...arguments);
+  } else if (
+    [Modules.Multisig, Modules.Utility].includes(section) &&
+    MultisigMethods.asMulti === method
+  ) {
+    return findAuthorFromMultisig(...arguments);
+  } else if (
+    Modules.Utility === section &&
+    [
+      UtilityMethods.batch,
+      UtilityMethods.batchAll,
+      UtilityMethods.forceBatch,
+    ].includes(method)
+  ) {
+    return findAuthorFromBatch(...arguments);
+  }
+
+  return null;
+}
+
+module.exports = {
+  findTargetCallAuthor: findCallAuthor,
+};

backend/packages/common/src/call/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+  ...require("./findTargetCallAuthor"),
+};

backend/packages/common/src/index.js

@@ -4,6 +4,7 @@ const consts = require("./consts");
 const graphql = require("./graphql");
 const store = require("./store");
 const known = require("./known");
+const call = require("./call");
 
 module.exports = {
   utils,
@@ -12,4 +13,5 @@ module.exports = {
   graphql,
   store,
   known,
+  call,
 };

backend/packages/common/src/known.js

@@ -1,14 +1,19 @@
 const {
-  knownHeight: { getNextKnownHeights },
+  knownHeight: { getNextKnownHeights, saveKnownHeights, getKnownHeightDb },
 } = require("@statescan/mongo");
 const {
   chain: { fetchBlocks },
   utils: { emptyFn, sleep },
   logger,
 } = require("@osn/scan-common");
 const last = require("lodash.last");
+const {
+  hasKnownHeightMark,
+  clearKnownHeightMark,
+} = require("./store/knownHeight");
 
 let count = 0;
+
 async function scanKnownHeights(toScanHeight, handleBlockFn = emptyFn) {
   let heights = await getNextKnownHeights(toScanHeight);
   while (heights.length > 0) {
@@ -36,6 +41,19 @@ async function scanKnownHeights(toScanHeight, handleBlockFn = emptyFn) {
   }
 }
 
+async function saveKnownHeight(indexer) {
+  const height = indexer.blockHeight;
+  if (hasKnownHeightMark(height)) {
+    await saveKnownHeights([height]);
+  }
+
+  const db = await getKnownHeightDb();
+  await db.updateScanHeight(height);
+
+  clearKnownHeightMark(height);
+}
+
 module.exports = {
   scanKnownHeights,
+  saveKnownHeight,
 };

backend/packages/graphql-server/.env.example

@@ -1,16 +1,19 @@
 PORT=7100
 
 MONGO_VESTING_SCAN_URL=mongodb://127.0.0.1:27017
-MONGO_VESTING_SCAN_NAME=prod-statescan-polkadot-vesting
+MONGO_VESTING_SCAN_NAME=prod-statescan-polkadot-vesting-1
 
 MONGO_PALLET_ASSET_SCAN_URL=mongodb://127.0.0.1:27017
-MONGO_PALLET_ASSET_SCAN_NAME=prod-statescan-statemint-assets
+MONGO_PALLET_ASSET_SCAN_NAME=prod-statescan-statemint-assets-1
 
 MONGO_MULTISIG_SCAN_URL=mongodb://127.0.0.1:27017
-MONGO_MULTISIG_SCAN_NAME=prod-statescan-polkadot-multisig
+MONGO_MULTISIG_SCAN_NAME=prod-statescan-polkadot-multisig-1
 
 MONGO_IDENTITY_SCAN_URL=mongodb://127.0.0.1:27017
 MONGO_IDENTITY_SCAN_NAME=prod-statescan-polkadot-identity
 
 MONGO_PALLET_RECOVERY_SCAN_URL=mongodb://127.0.0.1:27017
 MONGO_PALLET_RECOVERY_SCAN_NAME=prod-statescan-kusama-recovery-1
+
+MONGO_PROXY_SCAN_URL=mongodb://127.0.0.1:27017
+MONGO_PROXY_SCAN_NAME=prod-statescan-polkadot-proxy-1

backend/packages/graphql-server/package.json

@@ -36,6 +36,5 @@
   "devDependencies": {
     "eslint-plugin-jest": "^27.2.1",
     "jest": "^28.1.3"
-  },
-  "packageManager": "[email protected]"
+  }
 }

backend/packages/graphql-server/src/env.js

@@ -18,10 +18,15 @@ function hasRecovery() {
   return !!process.env.MONGO_PALLET_RECOVERY_SCAN_NAME;
 }
 
+function hasProxy() {
+  return !!process.env.MONGO_PROXY_SCAN_NAME;
+}
+
 module.exports = {
   hasAssets,
   hasVesting,
   hasMultisig,
   hasIdentity,
   hasRecovery,
+  hasProxy,
 };

backend/packages/graphql-server/src/index.js

@@ -9,13 +9,15 @@ const {
   multisig: { initMultisigScanDb },
   identity: { initIdentityScanDb },
   palletRecovery: { initPalletRecoveryScanDb },
+  palletProxy: { initPalletProxyScanDb },
 } = require("@statescan/mongo");
 const {
   hasVesting,
   hasAssets,
   hasMultisig,
   hasIdentity,
   hasRecovery,
+  hasProxy,
 } = require("./env");
 
 const port = parseInt(process.env.PORT) || 7100;
@@ -36,6 +38,9 @@ async function initDbs() {
   if (hasRecovery()) {
     await initPalletRecoveryScanDb();
   }
+  if (hasProxy()) {
+    await initPalletProxyScanDb();
+  }
 }
 
 function main() {

backend/packages/graphql-server/src/schema/index.js

@@ -7,6 +7,10 @@ const {
   resolvers: recoveryResolvers,
   typeDefs: recoveryTypeDefs,
 } = require("./recovery");
+const {
+  resolvers: proxyResolvers,
+  typeDefs: proxyTypeDefs,
+} = require("./proxy");
 const {
   resolvers: assetsPalletResolvers,
   typeDefs: assetsPalletTypeDefs,
@@ -25,6 +29,7 @@ const {
   hasIdentity,
   hasVesting,
   hasRecovery,
+  hasProxy,
 } = require("../env");
 const {
   graphql: { indexer, json },
@@ -53,6 +58,10 @@ if (hasIdentity()) {
   resolvers = [...resolvers, identityResolvers];
   typeDefs = [...typeDefs, ...identityTypeDefs];
 }
+if (hasProxy()) {
+  resolvers = [...resolvers, proxyResolvers];
+  typeDefs = [...typeDefs, ...proxyTypeDefs];
+}
 
 const schema = makeExecutableSchema({
   resolvers,

backend/packages/graphql-server/src/schema/proxy/index.js

@@ -0,0 +1,13 @@
+const { typeDefs } = require("./types");
+const resolverFunctions = require("./resolvers");
+
+const resolvers = {
+  Query: {
+    ...resolverFunctions,
+  },
+};
+
+module.exports = {
+  resolvers,
+  typeDefs,
+};

backend/packages/graphql-server/src/schema/proxy/resolvers/announcement.js

@@ -0,0 +1,18 @@
+const {
+  palletProxy: { getAnnouncementCol },
+} = require("@statescan/mongo");
+const { normalizeAnnouncement } = require("./common");
+
+async function announcement(_, _args) {
+  const { announcementId } = _args;
+  const col = await getAnnouncementCol();
+  const announcement = await col.findOne(
+    { announcementId },
+    { projection: { _id: 0 } },
+  );
+  return normalizeAnnouncement(announcement);
+}
+
+module.exports = {
+  proxyAnnouncement: announcement,
+};

backend/packages/graphql-server/src/schema/proxy/resolvers/announcementTimeline.js

@@ -0,0 +1,16 @@
+const {
+  palletProxy: { getAnnouncementTimelineCol },
+} = require("@statescan/mongo");
+
+async function announcementTimeline(_, _args) {
+  const { announcementId } = _args;
+  const col = await getAnnouncementTimelineCol();
+  return await col
+    .find({ announcementId }, { projection: { _id: 0 } })
+    .sort({ "indexer.blockHeight": 1 })
+    .toArray();
+}
+
+module.exports = {
+  proxyAnnouncementTimeline: announcementTimeline,
+};

backend/packages/graphql-server/src/schema/proxy/resolvers/announcements.js

@@ -0,0 +1,49 @@
+const {
+  palletProxy: { getAnnouncementCol },
+} = require("@statescan/mongo");
+const isNil = require("lodash.isnil");
+const isEmpty = require("lodash.isempty");
+const { normalizeAnnouncement } = require("./common");
+
+async function announcements(_, _args) {
+  const { delegator, delegatee, isFinal, offset, limit } = _args;
+  if (parseInt(limit) > 100) {
+    throw new Error("Over max page size 100");
+  }
+
+  let q = {};
+  if (delegator) {
+    Object.assign(q, { real: delegator });
+  }
+  if (delegatee) {
+    Object.assign(q, { delegate: delegatee });
+  }
+  if (!isNil(isFinal)) {
+    Object.assign(q, { isFinal });
+  }
+
+  const col = await getAnnouncementCol();
+  const items = await col
+    .find(q, { projection: { _id: 0 } })
+    .sort({ "indexer.blockHeight": -1 })
+    .skip(offset)
+    .limit(limit)
+    .toArray();
+  let total;
+  if (isEmpty(q)) {
+    total = await col.estimatedDocumentCount();
+  } else {
+    total = await col.countDocuments(q);
+  }
+
+  return {
+    items: items.map(normalizeAnnouncement),
+    offset,
+    limit,
+    total,
+  };
+}
+
+module.exports = {
+  proxyAnnouncements: announcements,
+};

backend/packages/graphql-server/src/schema/proxy/resolvers/common.js

@@ -0,0 +1,28 @@
+function normalizeAnnouncement(item = {}) {
+  if (!item) {
+    return item;
+  }
+
+  return {
+    ...item,
+    delegator: item.real,
+    delegatee: item.delegate,
+  };
+}
+
+function normalizeProxyCall(item = {}) {
+  if (!item) {
+    return item;
+  }
+
+  return {
+    ...item,
+    delegator: item.real,
+    delegatee: item.delegate,
+  };
+}
+
+module.exports = {
+  normalizeAnnouncement,
+  normalizeProxyCall,
+};

backend/packages/graphql-server/src/schema/proxy/resolvers/index.js

@@ -0,0 +1,10 @@
+module.exports = {
+  ...require("./proxies"),
+  ...require("./proxy"),
+  ...require("./proxyTimeline"),
+  ...require("./proxyCalls"),
+  ...require("./announcements"),
+  ...require("./announcement"),
+  ...require("./announcementTimeline"),
+  ...require("./proxyScanHeight"),
+};