From 078a7488c7c9427b70fed3422e39aaa701d83669 Mon Sep 17 00:00:00 2001 From: Max Andreev Date: Fri, 9 Jun 2023 08:57:44 +0000 Subject: [PATCH] add migration scripts --- .github/workflows/forkless-update-data.yml | 12 + tests/package.json | 1 + .../migrations/942057-appPromotion/README.md | 25 ++ .../afterMaintenance.test.ts | 74 +++++ .../correctStateAfterMaintenance.ts | 72 +++++ .../942057-appPromotion/lockedToFreeze.ts | 265 ++++++++++++++++++ .../942057-appPromotion/runCheckState.ts | 13 + .../942057-appPromotion/stakersParser.jsonnet | 23 ++ tests/yarn.lock | 5 + 9 files changed, 490 insertions(+) create mode 100644 tests/src/migrations/942057-appPromotion/README.md create mode 100644 tests/src/migrations/942057-appPromotion/afterMaintenance.test.ts create mode 100644 tests/src/migrations/942057-appPromotion/correctStateAfterMaintenance.ts create mode 100644 tests/src/migrations/942057-appPromotion/lockedToFreeze.ts create mode 100644 tests/src/migrations/942057-appPromotion/runCheckState.ts create mode 100644 tests/src/migrations/942057-appPromotion/stakersParser.jsonnet diff --git a/.github/workflows/forkless-update-data.yml b/.github/workflows/forkless-update-data.yml index a5b195cf1b..de15b8d4d5 100644 --- a/.github/workflows/forkless-update-data.yml +++ b/.github/workflows/forkless-update-data.yml @@ -251,6 +251,18 @@ jobs: if: success() || failure() run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.network }}/forkless-data.log' + - name: Run AppPromo migration + if: success() || failure() + working-directory: tests + run: | + yarn install + cd src/migrations/942057-appPromotion + /home/ubuntu/.cargo/bin/chainql --tla-str=chainUrl=ws://127.0.0.1:9944 stakersParser.jsonnet > output.json + npx ts-node --esm lockedToFreeze.ts + env: + WS_RPC: ws://127.0.0.1:9944 + SUPERUSER_SEED: //Alice + - name: Stop running containers if: always() # run this step always run: docker-compose -f ".docker/docker-compose.forkless-data.${{ matrix.network }}.yml" down --volumes diff --git a/tests/package.json b/tests/package.json index 80edc8c69c..198cccc969 100644 --- a/tests/package.json +++ b/tests/package.json @@ -139,6 +139,7 @@ "chai-like": "^1.1.1", "csv-writer": "^1.6.0", "find-process": "^1.4.7", + "lossless-json": "^2.0.9", "solc": "0.8.17", "web3": "1.10.0" }, diff --git a/tests/src/migrations/942057-appPromotion/README.md b/tests/src/migrations/942057-appPromotion/README.md new file mode 100644 index 0000000000..4d0a8843b6 --- /dev/null +++ b/tests/src/migrations/942057-appPromotion/README.md @@ -0,0 +1,25 @@ +## Stakers Data Loading + +Set the environment variable (WS_RPC). For example, ws://localhost:9944. Execute the following command: + +chainql --tla-str=chainUrl= stakersParser.jsonnet > output.json + + where - is the network address. + +Example for Opal: +: + +chainql --tla-str=chainUrl=wss://eu-ws-opal.unique.network:443 stakersParser.jsonnet > output.json + +To install chainql, execute the following command: + + +cargo install chainql + +## Execute offchain migration + +To run, you need to add an environment variable (SUPERUSER_SEED) with the sudo key seed. + +Run the script by executing the following command: + +npx ts-node lockedToFreeze.ts \ No newline at end of file diff --git a/tests/src/migrations/942057-appPromotion/afterMaintenance.test.ts b/tests/src/migrations/942057-appPromotion/afterMaintenance.test.ts new file mode 100644 index 0000000000..1effa9519f --- /dev/null +++ b/tests/src/migrations/942057-appPromotion/afterMaintenance.test.ts @@ -0,0 +1,74 @@ +// Copyright 2019-2022 Unique Network (Gibraltar) Ltd. +// This file is part of Unique Network. + +// Unique Network is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Unique Network is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Unique Network. If not, see . + +import {IKeyringPair} from '@polkadot/types/types'; +import {ApiPromise} from '@polkadot/api'; +import {expect, itSub, Pallets, requirePalletsOrSkip, usingPlaygrounds} from '../../util'; +import {main as testedScript} from './correctStateAfterMaintenance'; + +async function maintenanceEnabled(api: ApiPromise): Promise { + return (await api.query.maintenance.enabled()).toJSON() as boolean; +} + + + +describe('Integration Test: Maintenance mode & App Promo', () => { + let superuser: IKeyringPair; + + before(async function() { + await usingPlaygrounds(async (helper, privateKey) => { + requirePalletsOrSkip(this, helper, [Pallets.Maintenance]); + superuser = await privateKey('//Alice'); + }); + }); + + describe('Test AppPromo script for check state after Maintenance mode', () => { + before(async function () { + await usingPlaygrounds(async (helper) => { + if(await maintenanceEnabled(helper.getApi())) { + console.warn('\tMaintenance mode was left enabled BEFORE the test suite! Disabling it now.'); + await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.disable', [])).to.be.fulfilled; + } + }); + }); + itSub('Can find and fix inconsistent state', async({helper}) => { + const api = helper.getApi(); + + await helper.executeExtrinsic(superuser, 'api.tx.sudo.sudo', [api.tx.system.setStorage([ + ['0x42b67acb8bd223c60d0c8f621ffefc0ae280fa2db99bd3827aac976de75af95f5153cb1f00942ff401000000', + '0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d000010632d5ec76b0500000000000000'], + ['0x42b67acb8bd223c60d0c8f621ffefc0ae280fa2db99bd3827aac976de75af95f9eb2dcce60f37a2702000000', + '0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d000010632d5ec76b0500000000000000'], + ['0xc2261276cc9d1f8598ea4b6a74b15c2fb1c0eb12e038e5c7f91e120ed4b7ebf1de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d', + '0x046170707374616b656170707374616b65000020c65abc8ed70a00000000000000'], + ])]); + + expect((await api.query.appPromotion.pendingUnstake(1)).toJSON()).to.be.deep.equal([['5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY', '0x00000000000000056bc75e2d63100000']]); + expect((await api.query.appPromotion.pendingUnstake(2)).toJSON()).to.be.deep.equal([['5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY', '0x00000000000000056bc75e2d63100000']]); + await testedScript(); + + expect((await api.query.appPromotion.pendingUnstake(1)).toJSON()).to.be.deep.equal([]); + expect((await api.query.appPromotion.pendingUnstake(2)).toJSON()).to.be.deep.equal([]); + + }); + + itSub('(!negative test!) Only works when Maintenance mode is disabled', async({helper}) => { + await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.enable', [])).to.be.fulfilled; + await expect(testedScript()).to.be.rejectedWith('The network is still in maintenance mode'); + await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.disable', [])).to.be.fulfilled; + }); + }); +}); diff --git a/tests/src/migrations/942057-appPromotion/correctStateAfterMaintenance.ts b/tests/src/migrations/942057-appPromotion/correctStateAfterMaintenance.ts new file mode 100644 index 0000000000..6f0849389c --- /dev/null +++ b/tests/src/migrations/942057-appPromotion/correctStateAfterMaintenance.ts @@ -0,0 +1,72 @@ +import {usingPlaygrounds} from '../../util'; + + + +const WS_ENDPOINT = 'ws://localhost:9944'; +const DONOR_SEED = '//Alice'; + +export const main = async(options: { wsEndpoint: string; donorSeed: string } = { + wsEndpoint: WS_ENDPOINT, + donorSeed: DONOR_SEED, +}) => { + await usingPlaygrounds(async (helper, privateKey) => { + const api = helper.getApi(); + + if((await api.query.maintenance.enabled()).valueOf()) { + throw Error('The network is still in maintenance mode'); + } + + const pendingBlocks = ( + await api.query.appPromotion.pendingUnstake.entries() + ).map(([k, _v]) => + k.args[0]); + + const currentBlock = await api.query.system.number(); + + const filteredBlocks = pendingBlocks.filter((b) => currentBlock.gt(b)); + + if(filteredBlocks.length != 0) { + console.log( + 'During maintenance mode, %d block(s) were not processed', + filteredBlocks.length, + ); + } else { + console.log('Nothing to change'); + return; + } + + const skippedBlocks = chunk(filteredBlocks, 10); + + const signer = await privateKey(options.donorSeed); + + const txs = skippedBlocks.map((b) => + api.tx.sudo.sudo(api.tx.appPromotion.forceUnstake(b))); + + + const promises = txs.map((tx) => () => helper.signTransaction(signer, tx)); + + await Promise.allSettled(promises.map((p) => p())); + + const failedBlocks: bigint[] = []; + let isSuccess = true; + + for(const b of filteredBlocks) { + if(((await api.query.appPromotion.pendingUnstake(b)).toJSON() as any[]).length != 0) { + failedBlocks.push(b.toBigInt()); + isSuccess = false; + } + } + + if(isSuccess) { + console.log('Done. %d block(s) were processed.', filteredBlocks.length); + } else { + throw new Error(`Something went wrong. Block(s) have not been processed: ${failedBlocks}`); + } + + + }, options.wsEndpoint); +}; + +const chunk = (arr: T[], size: number) => + Array.from({length: Math.ceil(arr.length / size)}, (_: any, i: number) => + arr.slice(i * size, i * size + size)); \ No newline at end of file diff --git a/tests/src/migrations/942057-appPromotion/lockedToFreeze.ts b/tests/src/migrations/942057-appPromotion/lockedToFreeze.ts new file mode 100644 index 0000000000..173e9414cc --- /dev/null +++ b/tests/src/migrations/942057-appPromotion/lockedToFreeze.ts @@ -0,0 +1,265 @@ +// import { usingApi, privateKey, onlySign } from "./../../load/lib"; +import * as fs from 'fs'; +import {usingPlaygrounds} from '../../util'; +import path from 'path'; +import {isInteger, parse} from 'lossless-json'; + + +const WS_ENDPOINT = 'ws://localhost:9944'; +const DONOR_SEED = '//Alice'; +const UPDATE_IF_VERSION = 942057; + +export function customNumberParser(value: any) { + return isInteger(value) ? BigInt(value) : parseFloat(value); +} + +const main = async(options: { wsEndpoint: string; donorSeed: string } = { + wsEndpoint: WS_ENDPOINT, + donorSeed: DONOR_SEED, +}) => { + await usingPlaygrounds(async (helper, privateKey) => { + const api = helper.getApi(); + // 1. Check version equal 942057 or skip + console.log((api.consts.system.version as any).specVersion.toNumber()); + if((api.consts.system.version as any).specVersion.toNumber() != UPDATE_IF_VERSION) { + console.log("Version isn't 942057."); + return; + } + + // 2. Get sudo signer + const signer = await privateKey(options.donorSeed); + console.log('2. Getting sudo:', signer.address); + + // 3. Parse data to migrate + console.log('3. Parsing chainql results...'); + const parsingResult = parse(fs.readFileSync(path.resolve('output.json'), 'utf-8'), undefined, customNumberParser); + + const chainqlImportData = parsingResult as { + address: string; + balance: string; + account: { + fee_frozen: string, + free: string, + misc_frozen: string, + reserved: string, + }, + locks: { + amount: string, + id: string, + }[], + stakes: object, + unstakes: object, + }[]; + testChainqlData(chainqlImportData); + + const stakers = chainqlImportData.map((i) => i.address); + + // 3.1 Split into chunks by 100 + console.log('3.1 Splitting into chunks...'); + const stakersChunks = chunk(stakers, 100); + console.log('3.1 Done, total chunks:', stakersChunks.length); + + // 4. Get signer/sudo nonce + console.log('4. Getting sudo nonce...'); + const signerAccount = ( + await api.query.system.account(signer.address) + ).toJSON() as any; + + let nonce: number = signerAccount.nonce; + console.log('4. Sudo nonce is:', nonce); + + // 5. Only sign upgradeAccounts-transactions for each chunk + console.log('5. Signing transactions...'); + const signedTxs = []; + for(const chunk of stakersChunks) { + const tx = api.tx.sudo.sudo(api.tx.appPromotion.upgradeAccounts(chunk)); + const signed = tx.sign(signer, { + blockHash: api.genesisHash, + genesisHash: api.genesisHash, + runtimeVersion: api.runtimeVersion, + nonce: nonce++, + }); + signedTxs.push(signed); + } + + // 6. Send all signed transactions + console.log('6. Sending transactions...'); + const promises = signedTxs.map((tx) => api.rpc.author.submitAndWatchExtrinsic(tx)); + // 6.1 Wait all transactions settled + console.log('6.1 Waiting all transactions settled...'); + const res = await Promise.allSettled(promises); + + console.log('Wait 5 blocks for transactions to be included in a block...'); + await helper.wait.newBlocks(5); + // 6.2 Filter failed transactions + console.log('6.2 Getting failed transactions...'); + const failedTx = res.filter((r) => r.status == 'rejected') as PromiseRejectedResult[]; + console.log('6.2. total failedTxs:', failedTx.length); + + // 6.3 Log the reasons of failed tx + for(const tx of failedTx) { + console.log(tx.reason); + } + + // 7. Check balances for 10 blocks: + console.log('7. Check balances...'); + let blocksLeft = 10; + let notMigrated = stakers; + const suspiciousAccounts = []; + do { + console.log('blocks left:', blocksLeft); + const _notMigrated: string[] = []; + console.log('accounts to migrate...', notMigrated.length); + for(const accountToMigrate of notMigrated) { + let accountMigrated = true; + // 7.0 get data from chainql: + const oldAccount = chainqlImportData.find(acc => acc.address === accountToMigrate); + if(!oldAccount) { + console.log('Cannot find old account data for', accountMigrated); + accountMigrated = false; + _notMigrated.push(accountToMigrate); + continue; + } + + // 7.1 system.account + const balance = await api.query.system.account(accountToMigrate) as any; + // new balances + const free = balance.data.free; + const reserved = balance.data.reserved; + const frozen = balance.data.frozen; + // old balances + const oldFree = oldAccount.account.free; + const oldReserved = oldAccount.account.reserved; + const oldFrozen = oldAccount.account.fee_frozen; + // asserts new = old + if(oldFree.toString() !== free.toString()) { + console.log('Old free !== New free, which is probably not a problem', oldFree.toString(), free.toString()); + suspiciousAccounts.push(accountToMigrate); + } + if(oldFrozen.toString() !== frozen.toString()) { + console.log('Old frozen !== New frozen, which is probably not a problem', oldFrozen.toString(), frozen.toString()); + suspiciousAccounts.push(accountToMigrate); + } + if(oldReserved.toString() !== reserved.toString()) { + console.log('Old reserved !== New reserved, which is probably not a problem', oldReserved.toString(), reserved.toString()); + suspiciousAccounts.push(accountToMigrate); + } + + // 7.2 balances.locks: no id appstake + const locks = await helper.balance.getLocked(accountToMigrate); + const appPromoLocks = locks.filter(lock => lock.id === 'appstake'); + if(appPromoLocks.length !== 0) { + console.log('Account still has app-promo lock'); + accountMigrated = false; + } + + // 7.3 balances.freezes set... + let freezes = await api.query.balances.freezes(accountToMigrate) as any; + freezes = freezes.map((freez: any) => ({id: freez.id.toString(), amount: freez.amount.toString()})); + if(!freezes) { + console.log('Account does not have freezes'); + accountMigrated = false; + } else { + const oldAppPromoLocks = oldAccount.locks.filter(l => l.id === '0x6170707374616b65'); // get app promo locks + // should be only one freez for each account + if(freezes.length !== 1) { + console.log('freezes.length !== 1 and old appPromoLocks.length', freezes.length, oldAppPromoLocks.length); + accountMigrated = false; + } else { + const appPromoFreez = freezes[0]; + // freez-amount should be equal to migrated lock amount + if(appPromoFreez.amount.toString() !== oldAppPromoLocks[0].amount.toString()) { + console.log('freezes amount !== old appPromoLocks amount', appPromoFreez.amount.toString(), oldAppPromoLocks[0].amount.toString()); + accountMigrated = false; + } + // freez id should be correct + if(appPromoFreez.id !== '0x6170707374616b656170707374616b65') { + console.log('Got freez with incorrect id:', appPromoFreez.id); + accountMigrated = false; + } + } + } + + // 7.4 Stakes number the same + const stakesNumber = await helper.staking.getStakesNumber({Substrate: accountToMigrate}); + const oldStakesNumber = oldAccount.stakes ? Object.keys(oldAccount.stakes).length : 0; + if(stakesNumber.toString() !== oldStakesNumber.toString()) { + console.log('Old stakes number !== New stakes number', oldStakesNumber, stakesNumber); + accountMigrated = false; + } + + // 7.5 Total pendingUnstake + total staked = old locked + const pendingUnstakes = await helper.staking.getPendingUnstake({Substrate: accountToMigrate}); + const totalStaked = await helper.staking.getTotalStaked({Substrate: accountToMigrate}); + const totalBalanceInAppPromo = pendingUnstakes + totalStaked; + if(totalBalanceInAppPromo.toString() !== oldAccount.balance.toString()) { + console.log('totalBalanceInAppPromo !== old locked in app promo', totalBalanceInAppPromo.toString(), oldAccount.balance.toString()); + accountMigrated = false; + } + + // 8 Add to not-migrated + if(!accountMigrated) { + console.log('Add to not migrated:', accountToMigrate); + _notMigrated.push(accountToMigrate); + } + } + + blocksLeft--; + notMigrated = _notMigrated; + await helper.wait.newBlocks(1); + } while(blocksLeft > 0 && notMigrated.length !== 0); + + console.log('Not migrated accounts...', notMigrated.length); + if(suspiciousAccounts.length > 0) { + console.log('Saving suspicious accounts to suspicious.json:'); + fs.writeFileSync('./suspicious.json', JSON.stringify(suspiciousAccounts)); + } + if(notMigrated.length > 0) { + console.log('Saving not migrated list to failed.json:'); + notMigrated.forEach(console.log); + fs.writeFileSync('./failed.json', JSON.stringify(notMigrated)); + process.exit(1); + } else { + console.log('Migration success'); + } + }, options.wsEndpoint); +}; + +const chunk = (arr: T[], size: number) => + Array.from({length: Math.ceil(arr.length / size)}, (_: any, i: number) => + arr.slice(i * size, i * size + size)); + +const testChainqlData = (data: any) => { + const wrongData = []; + for(const account of data) { + try { + if(account.address == null) throw Error('no address in data'); + if(account.balance == null) throw Error('no balance in data'); + if(account.account == null) throw Error('no account in data'); + if(account.account.fee_frozen == null) throw Error('no account.fee_frozen in data'); + if(account.account.misc_frozen == null) throw Error('no account.misc_frozen in data'); + if(account.account.free == null) throw Error('no account.free in data'); + if(account.account.reserved == null) throw Error('no account.reserved in data'); + if(account.locks == null) throw Error('no locks in data'); + if(account.locks[0].amount == null) throw Error('no locks.amount in data'); + if(account.locks[0].id == null) throw Error('no locks.id in data'); + } catch (error) { + wrongData.push(account.address); + console.log((error as Error).message, account.address); + } + if(wrongData.length > 0) { + console.log(data); + throw Error('Chainql data not correct'); + } + } + console.log('Chainql data correct'); +}; + +main({ + wsEndpoint: process.env.WS_RPC!, + donorSeed: process.env.SUPERUSER_SEED!, +}).then(() => process.exit(0)) + .catch((e) => { + console.error(e); + process.exit(1); + }); \ No newline at end of file diff --git a/tests/src/migrations/942057-appPromotion/runCheckState.ts b/tests/src/migrations/942057-appPromotion/runCheckState.ts new file mode 100644 index 0000000000..68ff3b51b0 --- /dev/null +++ b/tests/src/migrations/942057-appPromotion/runCheckState.ts @@ -0,0 +1,13 @@ +import {main} from './correctStateAfterMaintenance'; + + + + +main({ + wsEndpoint: process.env.WS_RPC!, + donorSeed: process.env.SUPERUSER_SEED!, +}).then(() => process.exit(0)) + .catch((e) => { + console.error(e); + process.exit(1); + }); \ No newline at end of file diff --git a/tests/src/migrations/942057-appPromotion/stakersParser.jsonnet b/tests/src/migrations/942057-appPromotion/stakersParser.jsonnet new file mode 100644 index 0000000000..3ae60aa101 --- /dev/null +++ b/tests/src/migrations/942057-appPromotion/stakersParser.jsonnet @@ -0,0 +1,23 @@ +function(chainUrl) + + local + state = cql.chain(chainUrl).latest, + locked_balances = state.Balances.Locks._preloadKeys, + accountsRaw = state.System.Account._preloadKeys, + stakers = state.AppPromotion.Staked._preloadKeys, + unstakes = state.AppPromotion.PendingUnstake._preloadKeys, + locks = [ + { [k]: std.filter(function(l) l.id == '0x6170707374616b65', locked_balances[k]) } + for k in std.objectFields(locked_balances) + ], + non_empty_locks = std.prune(locks) + ; + + std.map(function(a) { + address: std.objectFields(a)[0], + balance: a[std.objectFields(a)[0]][0].amount, + account: accountsRaw[std.objectFields(a)[0]].data, + locks: locked_balances[std.objectFields(a)[0]], + stakes: if std.objectHas(stakers, std.objectFields(a)[0]) then stakers[std.objectFields(a)[0]] else null, + unstakes: if std.objectHas(unstakes, std.objectFields(a)[0]) then unstakes[std.objectFields(a)[0]] else null, + }, non_empty_locks) \ No newline at end of file diff --git a/tests/yarn.lock b/tests/yarn.lock index e92f787d83..b8cb8d283b 100644 --- a/tests/yarn.lock +++ b/tests/yarn.lock @@ -3406,6 +3406,11 @@ long@^4.0.0: resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== +lossless-json@^2.0.9: + version "2.0.9" + resolved "https://registry.yarnpkg.com/lossless-json/-/lossless-json-2.0.9.tgz#2e9a71a3dcbc6c59dee565e537b9084107b7fe37" + integrity sha512-PUfJ5foxULG1x/dXpSckmt0woBDqyq/WFoI885vEqjGwuP41K2EBYh2IT3zYx9dWqcTLIfXiCE5AjhF1jk9Sbg== + loupe@^2.3.1: version "2.3.6" resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.6.tgz#76e4af498103c532d1ecc9be102036a21f787b53"