Merge pull request #948 from UniqueNetwork/ci/locks-to-freezes

CI: locks to freezes migration

.docker/Dockerfile-parachain-upgrade-data.j2

@@ -42,6 +42,7 @@ FROM ubuntu:22.04
 
 ENV RELAY_CHAIN_TYPE={{ RELAY_CHAIN_TYPE }}
 ENV REPLICA_FROM={{ REPLICA_FROM }}
+ENV DESTINATION_SPEC_VERSION={{ DESTINATION_SPEC_VERSION }}
 ENV NEW_PARA_BIN=/unique-chain/target/release/unique-collator
 ENV NEW_PARA_WASM=/unique-chain/target/release/wbuild/{{ WASM_NAME }}-runtime/{{ WASM_NAME }}_runtime.compact.compressed.wasm
 
@@ -77,7 +78,7 @@ EXPOSE 33088
 EXPOSE 33144
 EXPOSE 33155
 
-CMD export NVM_DIR="$HOME/.nvm" PATH="$PATH:/chainql/target/release" REPLICA_FROM NEW_PARA_BIN NEW_PARA_WASM RELAY_CHAIN_TYPE && \
+CMD export NVM_DIR="$HOME/.nvm" PATH="$PATH:/chainql/target/release" REPLICA_FROM NEW_PARA_BIN NEW_PARA_WASM RELAY_CHAIN_TYPE DESTINATION_SPEC_VERSION && \
     [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \
     cd /unique-chain/tests && \
     npm install --global yarn && \

.docker/docker-compose.forkless-data.j2

@@ -3,7 +3,7 @@ version: "3.5"
 services:
   forkless-data:
     image: uniquenetwork/ci-forkless-data-local:{{ NETWORK }}-{{ BUILD_TAG }}
-    container_name: forkless-data
+    container_name: forkless-data-{{ NETWORK }}
     expose:
       - 9944
       - 9933

.env

@@ -2,6 +2,7 @@ RUST_TOOLCHAIN=nightly-2022-11-15
 POLKADOT_LAUNCH_BRANCH=unique-network
 RELAY_CHAIN_TYPE=westend
 CHAINQL=v0.4.1
+DESTINATION_SPEC_VERSION=v942057
 
 POLKADOT_MAINNET_BRANCH=release-v0.9.37
 STATEMINT_BUILD_BRANCH=release-parachains-v9370
@@ -16,12 +17,12 @@ STATEMINE_BUILD_BRANCH=release-parachains-v9382
 KARURA_BUILD_BRANCH=release-karura-2.17.0
 MOONRIVER_BUILD_BRANCH=runtime-2302
 SHIDEN_BUILD_BRANCH=v5.4.0
-QUARTZ_MAINNET_BRANCH=release-v941055
+QUARTZ_MAINNET_BRANCH=release-v941056
 QUARTZ_REPLICA_FROM=wss://ws-quartz.unique.network:443
 
 UNIQUEWEST_MAINNET_BRANCH=release-v0.9.42
 WESTMINT_BUILD_BRANCH=parachains-v9420
-OPAL_MAINNET_BRANCH=release-v941055
+OPAL_MAINNET_BRANCH=release-v942057
 OPAL_REPLICA_FROM=wss://ws-opal.unique.network:443
 
 UNIQUEEAST_MAINNET_BRANCH=release-v0.9.42

.github/workflows/forkless-update-data.yml

@@ -82,6 +82,7 @@ jobs:
             MAINNET_BRANCH=${{ matrix.mainnet_branch }}
             WASM_NAME=${{ matrix.wasm_name }}
             RELAY_CHAIN_TYPE=${{ env.RELAY_CHAIN_TYPE }}
+            DESTINATION_SPEC_VERSION=${{ env.DESTINATION_SPEC_VERSION }}
             POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}
             REPLICA_FROM=${{ matrix.replica_from_address }}
             CHAINQL=${{ env.CHAINQL }}
@@ -198,14 +199,14 @@ jobs:
         run: |
           counter=160
           function check_container_status {
-                docker inspect -f {{.State.Running}} forkless-data
+                docker inspect -f {{.State.Running}} forkless-data-${{ matrix.network }}
           }
           function do_docker_logs {
-                docker logs --details forkless-data  2>&1
+                docker logs --details forkless-data-${{ matrix.network }}  2>&1
           }
           function is_started {
                 if [ "$(check_container_status)" == "true" ]; then
-                        echo "Container: forkless-data RUNNING";
+                        echo "Container: forkless-data-${{ matrix.network }} RUNNING";
                         echo "Check Docker logs"
                         DOCKER_LOGS=$(do_docker_logs)
                         if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then
@@ -219,7 +220,7 @@ jobs:
                                 return 1
                         fi
                 else
-                        echo "Container forkless-data not RUNNING"
+                        echo "Container forkless-data-${{ matrix.network }} not RUNNING"
                         echo "Halting all future checks"
                         exit 1
                 fi
@@ -249,7 +250,7 @@ jobs:
 
       - name: Show Docker logs
         if: success() || failure()
-        run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.network }}/forkless-data.log'
+        run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.network }}/forkless-data-${{ matrix.network }}.log'
 
       - name: Stop running containers
         if: always()                   # run this step always

tests/package.json

@@ -139,6 +139,7 @@
     "chai-like": "^1.1.1",
     "csv-writer": "^1.6.0",
     "find-process": "^1.4.7",
+    "lossless-json": "^2.0.9",
     "solc": "0.8.17",
     "web3": "1.10.0"
   },

tests/src/migrations/942057-appPromotion/README.md

@@ -0,0 +1,42 @@
+# Update Procedure
+
+- Enable maintenance mode
+- [Collect migration data using ChainQL](#stakers-data-loading)
+- ❗️❗️❗️ Initiate the runtime upgrade only at this point ❗️❗️❗️
+- Wait for the upgrade to complete
+- [Execute offchain migration](#execute-offchain-migration)
+- Disable maintenance mode
+
+## Stakers Data Loading
+
+Set the environment variable (WS_RPC). For example, ws://localhost:9944. Execute the following command:
+
+```sh
+chainql --tla-str=chainUrl=<WS_RPC> stakersParser.jsonnet > output.json
+```
+
+where `<WS_RPC>` - is the network address.
+
+Example for Opal:
+
+```sh
+chainql --tla-str=chainUrl=wss://eu-ws-opal.unique.network:443 stakersParser.jsonnet > output.json
+```
+
+To install chainql, execute the following command:
+
+```sh
+cargo install chainql
+```
+
+## Execute offchain migration
+
+To run, you need to set an environment variables:
+- `SUPERUSER_SEED` – the sudo key seed.
+- `WS_RPC` – the network address
+
+Run the migration by executing the following command:
+
+```sh
+npx ts-node --esm executeMigration.ts
+```

tests/src/migrations/942057-appPromotion/afterMaintenance.test.ts

@@ -0,0 +1,75 @@
+// Copyright 2019-2022 Unique Network (Gibraltar) Ltd.
+// This file is part of Unique Network.
+
+// Unique Network is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Unique Network is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Unique Network. If not, see <http://www.gnu.org/licenses/>.
+
+import {IKeyringPair} from '@polkadot/types/types';
+import {ApiPromise} from '@polkadot/api';
+import {expect, itSub, Pallets, requirePalletsOrSkip, usingPlaygrounds} from '../../util';
+import {main as testedScript} from './correctStateAfterMaintenance';
+
+async function maintenanceEnabled(api: ApiPromise): Promise<boolean> {
+  return (await api.query.maintenance.enabled()).toJSON() as boolean;
+}
+
+
+
+describe('Integration Test: Maintenance mode & App Promo', () => {
+  let superuser: IKeyringPair;
+
+  before(async function() {
+    await usingPlaygrounds(async (helper, privateKey) => {
+      requirePalletsOrSkip(this, helper, [Pallets.Maintenance]);
+      superuser = await privateKey('//Alice');
+    });
+  });
+
+  describe('Test AppPromo script for check state after Maintenance mode', () => {
+    before(async function () {
+      await usingPlaygrounds(async (helper) => {
+        if(await maintenanceEnabled(helper.getApi())) {
+          console.warn('\tMaintenance mode was left enabled BEFORE the test suite! Disabling it now.');
+          await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.disable', [])).to.be.fulfilled;
+        }
+      });
+    });
+    itSub('Can find and fix inconsistent state', async({helper}) => {
+      const api = helper.getApi();
+
+      await helper.executeExtrinsic(superuser, 'api.tx.sudo.sudo', [api.tx.system.setStorage([
+        ['0x42b67acb8bd223c60d0c8f621ffefc0ae280fa2db99bd3827aac976de75af95f5153cb1f00942ff401000000',
+          '0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d000010632d5ec76b0500000000000000'],
+        ['0x42b67acb8bd223c60d0c8f621ffefc0ae280fa2db99bd3827aac976de75af95f9eb2dcce60f37a2702000000',
+          '0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d000010632d5ec76b0500000000000000'],
+        ['0xc2261276cc9d1f8598ea4b6a74b15c2fb1c0eb12e038e5c7f91e120ed4b7ebf1de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d',
+          '0x046170707374616b656170707374616b65000020c65abc8ed70a00000000000000'],
+      ])]);
+
+      // const pendingUnstaked = await helper.staking.getPendingUnstakePerBlock()
+      expect((await api.query.appPromotion.pendingUnstake(1)).toJSON()).to.be.deep.equal([[helper.address.normalizeSubstrateToChainFormat('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'), '0x00000000000000056bc75e2d63100000']]);
+      expect((await api.query.appPromotion.pendingUnstake(2)).toJSON()).to.be.deep.equal([[helper.address.normalizeSubstrateToChainFormat('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'), '0x00000000000000056bc75e2d63100000']]);
+      await testedScript();
+
+      expect((await api.query.appPromotion.pendingUnstake(1)).toJSON()).to.be.deep.equal([]);
+      expect((await api.query.appPromotion.pendingUnstake(2)).toJSON()).to.be.deep.equal([]);
+
+    });
+
+    itSub('(!negative test!) Only works when Maintenance mode is disabled', async({helper}) => {
+      await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.enable', [])).to.be.fulfilled;
+      await expect(testedScript()).to.be.rejectedWith('The network is still in maintenance mode');
+      await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.disable', [])).to.be.fulfilled;
+    });
+  });
+});

tests/src/migrations/942057-appPromotion/collectData.ts

@@ -0,0 +1,12 @@
+import {exec} from 'child_process';
+import path from 'path';
+import {dirname} from 'path';
+import {fileURLToPath} from 'url';
+
+export const collectData = () => {
+  const dirName = dirname(fileURLToPath(import.meta.url));
+
+  const pathToScript = path.resolve(dirName, './stakersParser.jsonnet');
+  const outputPath = path.resolve(dirName, './output.json');
+  exec(`chainql --tla-str=chainUrl=ws://127.0.0.1:9944 ${pathToScript} > ${outputPath}`);
+};

tests/src/migrations/942057-appPromotion/correctStateAfterMaintenance.ts

@@ -0,0 +1,72 @@
+import {usingPlaygrounds} from '../../util';
+
+
+
+const WS_ENDPOINT = 'ws://localhost:9944';
+const DONOR_SEED = '//Alice';
+
+export const main = async(options: { wsEndpoint: string; donorSeed: string } = {
+  wsEndpoint: WS_ENDPOINT,
+  donorSeed: DONOR_SEED,
+}) => {
+  await usingPlaygrounds(async (helper, privateKey) => {
+    const api = helper.getApi();
+
+    if((await api.query.maintenance.enabled()).valueOf()) {
+      throw Error('The network is still in maintenance mode');
+    }
+
+    const pendingBlocks = (
+      await api.query.appPromotion.pendingUnstake.entries()
+    ).map(([k, _v]) =>
+      k.args[0]);
+
+    const currentBlock = await api.query.system.number();
+
+    const filteredBlocks = pendingBlocks.filter((b) => currentBlock.gt(b));
+
+    if(filteredBlocks.length != 0) {
+      console.log(
+        'During maintenance mode, %d block(s) were not processed',
+        filteredBlocks.length,
+      );
+    } else {
+      console.log('Nothing to change');
+      return;
+    }
+
+    const skippedBlocks = chunk(filteredBlocks, 10);
+
+    const signer = await privateKey(options.donorSeed);
+
+    const txs = skippedBlocks.map((b) =>
+      api.tx.sudo.sudo(api.tx.appPromotion.forceUnstake(b)));
+
+
+    const promises = txs.map((tx) => () => helper.signTransaction(signer, tx));
+
+    await Promise.allSettled(promises.map((p) => p()));
+
+    const failedBlocks: bigint[] = [];
+    let isSuccess = true;
+
+    for(const b of filteredBlocks) {
+      if(((await api.query.appPromotion.pendingUnstake(b)).toJSON() as any[]).length != 0) {
+        failedBlocks.push(b.toBigInt());
+        isSuccess = false;
+      }
+    }
+
+    if(isSuccess) {
+      console.log('Done. %d block(s) were processed.', filteredBlocks.length);
+    } else {
+      throw new Error(`Something went wrong. Block(s) have not been processed: ${failedBlocks}`);
+    }
+
+
+  }, options.wsEndpoint);
+};
+
+const chunk = <T>(arr: T[], size: number) =>
+  Array.from({length: Math.ceil(arr.length / size)}, (_: any, i: number) =>
+    arr.slice(i * size, i * size + size));

tests/src/migrations/942057-appPromotion/executeMigration.ts

@@ -0,0 +1,11 @@
+import {migrateLockedToFreeze} from './lockedToFreeze';
+
+
+const WS_RPC = process.env.WS_RPC || 'wss://ws-opal.unique.network:443';
+const SUPERUSER_SEED = process.env.SUPERUSER_SEED || '';
+
+migrateLockedToFreeze({
+  wsEndpoint: WS_RPC,
+  donorSeed: SUPERUSER_SEED,
+})
+  .catch(console.error);

tests/src/migrations/942057-appPromotion/index.ts

@@ -0,0 +1,12 @@
+import {Migration} from '../../util/frankensteinMigrate';
+import {collectData} from './collectData';
+import {migrateLockedToFreeze} from './lockedToFreeze';
+
+export const migration: Migration = {
+  async before() {
+    await collectData();
+  },
+  async after() {
+    await migrateLockedToFreeze();
+  },
+};