test: add stateful upgrade test

.github/workflows/execute_advanced_tests.yaml

@@ -11,6 +11,10 @@ on:
         type: boolean
         required: false
         default: false
+      e2e-stateful-upgrade-test:
+        type: boolean
+        required: false
+        default: false
       e2e-performance-test:
         type: boolean
         required: false
@@ -19,6 +23,10 @@ on:
         type: boolean
         required: false
         default: false
+      e2e-stateful-data-test:
+        type: boolean
+        required: false
+        default: false
       debug:
         type: boolean
         required: false
@@ -63,7 +71,7 @@ jobs:
 
   e2e-upgrade-test:
     if: ${{ github.event.inputs.e2e-upgrade-test == 'true' || github.event_name == 'schedule' }}
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: buildjet-16vcpu-ubuntu-2204
     timeout-minutes: 120
     steps:
       - name: "Checkout Code"
@@ -92,6 +100,37 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
 
+  e2e-stateful-upgrade-test:
+    if: ${{ github.event.inputs.e2e-stateful-upgrade-test == 'true' || github.event_name == 'schedule' }}
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    timeout-minutes: 120
+    steps:
+      - name: "Checkout Code"
+        uses: actions/checkout@v4
+
+      - name: Start Test
+        run: make start-upgrade-import-mainnet-test
+
+      - name: Watch Test
+        run: |
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" &
+          exit $(docker wait "${container_id}")
+
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          make stop-localnet
+
+      - name: Notify Slack on Failure
+        if: failure() && github.event_name == 'schedule'
+        uses: 8398a7/action-slack@v3
+        with:
+          status: ${{ job.status }}
+          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
+
   e2e-upgrade-test-light:
     if: ${{ github.event.inputs.e2e-upgrade-test-light == 'true' }}
     runs-on: buildjet-4vcpu-ubuntu-2204
@@ -114,6 +153,15 @@ jobs:
         run: |
           make stop-localnet
 
+      - name: Notify Slack on Failure
+        if: failure() && github.event_name == 'schedule'
+        uses: 8398a7/action-slack@v3
+        with:
+          status: ${{ job.status }}
+          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
+
   e2e-performance-test:
     if: ${{ github.event.inputs.e2e-performance-test == 'true' }}
     runs-on: buildjet-4vcpu-ubuntu-2204
@@ -135,3 +183,43 @@ jobs:
         if: failure()
         run: |
           make stop-localnet
+
+      - name: Notify Slack on Failure
+        if: failure() && github.event_name == 'schedule'
+        uses: 8398a7/action-slack@v3
+        with:
+          status: ${{ job.status }}
+          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}
+
+  e2e-stateful-data-test:
+    if: ${{ github.event.inputs.e2e-stateful-data-test == 'true' || github.event_name == 'schedule' }}
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    timeout-minutes: 120
+    steps:
+      - name: "Checkout Code"
+        uses: actions/checkout@v4
+
+      - name: Start Test
+        run: make start-e2e-import-mainnet-test
+
+      - name: Watch Test
+        run: |
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" &
+          exit $(docker wait "${container_id}")
+
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          make stop-localnet
+
+      - name: Notify Slack on Failure
+        if: failure() && github.event_name == 'schedule'
+        uses: 8398a7/action-slack@v3
+        with:
+          status: ${{ job.status }}
+          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}

.github/workflows/publish-release.yml

@@ -367,6 +367,8 @@ jobs:
         shell: bash
         run: |
           make start-e2e-admin-test
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" & exit $(docker wait "${container_id}")
 
       - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
@@ -377,7 +379,7 @@ jobs:
   e2e-upgrade-test:
     needs:
       - check_branch
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: buildjet-16vcpu-ubuntu-2204
     timeout-minutes: 120
     steps:
       - name: "Checkout Code"
@@ -388,7 +390,33 @@ jobs:
         if: ${{ github.event.inputs.skip_checks != 'true' }}
         shell: bash
         run: |
-          make start-upgrade-test
+          make start-upgrade-import-mainnet-test
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" & exit $(docker wait "${container_id}")
+
+      - name: Mark Job Complete Skipped
+        if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
+        run: |
+          echo "continue"
+
+  e2e-stateful-data-test:
+    needs:
+      - check_branch
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    timeout-minutes: 120
+    steps:
+      - name: "Checkout Code"
+        if: ${{ github.event.inputs.skip_checks != 'true' }}
+        uses: actions/checkout@v3
+
+      - name: Execute stateful-data-test
+        if: ${{ github.event.inputs.skip_checks != 'true' }}
+        shell: bash
+        run: |
+          make start-e2e-import-mainnet-test
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" & exit $(docker wait "${container_id}")
 
       - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
@@ -407,6 +435,7 @@ jobs:
       - smoke-test
       - build-test
       - e2e-admin-tests
+      - e2e-stateful-data-test
       - e2e-upgrade-test
       - check_branch
     runs-on: ubuntu-22.04

Makefile

@@ -204,8 +204,22 @@ mocks:
 generate: proto-gen openapi specs typescript docs-zetacored mocks fmt
 .PHONY: generate
 
+
 ###############################################################################
-###                         E2E tests and localnet                          ###
+###                         Localnet                          				###
+###############################################################################
+start-localnet: zetanode start-localnet-skip-build
+
+start-localnet-skip-build:
+	@echo "--> Starting localnet"
+	export LOCALNET_MODE=setup-only && \
+	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d
+
+stop-localnet:
+	cd contrib/localnet/ && $(DOCKER) compose down --remove-orphans
+
+###############################################################################
+###                         E2E tests               						###
 ###############################################################################
 
 zetanode:
@@ -233,11 +247,21 @@ start-e2e-performance-test: zetanode
 	export E2E_ARGS="--test-performance" && \
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d
 
+start-e2e-import-mainnet-test: zetanode
+	@echo "--> Starting e2e import-data test"
+	export ZETACORED_IMPORT_GENESIS_DATA=true && \
+	export ZETACORED_START_PERIOD=15m && \
+	cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml up -d
+
 start-stress-test: zetanode
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stress.yml up -d
 
-#TODO: replace OLD_VERSION with v16 tag once its available
+###############################################################################
+###                         Upgrade Tests              						###
+###############################################################################
+
+
 zetanode-upgrade: zetanode
 	@echo "Building zetanode-upgrade"
 	$(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v17' .
@@ -256,22 +280,13 @@ start-upgrade-test-light: zetanode-upgrade
 	export UPGRADE_HEIGHT=90 && \
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml up -d
 
-start-localnet: zetanode start-localnet-skip-build
-
-start-localnet-skip-build:
-	@echo "--> Starting localnet"
-	export LOCALNET_MODE=setup-only && \
-	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d
-
-start-e2e-import-mainnet-test: zetanode
-	@echo "--> Starting e2e import-data test"
+start-upgrade-import-mainnet-test: zetanode-upgrade
+	@echo "--> Starting import-data upgrade test"
+	export LOCALNET_MODE=upgrade && \
 	export ZETACORED_IMPORT_GENESIS_DATA=true && \
-	export ZETACORED_START_PERIOD=10m && \
-	cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml up -d
-
-stop-localnet:
-	cd contrib/localnet/ && $(DOCKER) compose down --remove-orphans
-
+	export ZETACORED_START_PERIOD=15m && \
+	export UPGRADE_HEIGHT=225 && \
+	cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml up -d
 ###############################################################################
 ###                              Monitoring                                 ###
 ###############################################################################

changelog.md

@@ -68,6 +68,7 @@
 * [2329](https://github.com/zeta-chain/node/pull/2329) - fix TODOs in rpc unit tests
 * [2342](https://github.com/zeta-chain/node/pull/2342) - extend rpc unit tests with testing extension to include synthetic ethereum txs
 * [2299](https://github.com/zeta-chain/node/pull/2299) - add `zetae2e` command to deploy test contracts
+* [2364](https://github.com/zeta-chain/node/pull/2364) - add stateful upgrade test
 * [2360](https://github.com/zeta-chain/node/pull/2360) - add stateful e2e tests.
 * [2349](https://github.com/zeta-chain/node/pull/2349) - add TestBitcoinDepositRefund and WithdrawBitcoinMultipleTimes E2E tests
 * [2368](https://github.com/zeta-chain/node/pull/2368) - eliminate panic usage across testing suite

cmd/zetacored/parse_genesis.go

@@ -119,7 +119,7 @@ func CmdParseGenesisFile() *cobra.Command {
 			return nil
 		},
 	}
-	cmd.PersistentFlags().Bool("modify", false, "Modify the genesis file before importing")
+	cmd.PersistentFlags().Bool("modify", false, "modify the genesis file before importing")
 	return cmd
 }
 

contrib/localnet/docker-compose-upgrade.yml

@@ -1,3 +1,4 @@
+version: "3"
 # This docker-compose redefine the services:
 # - ZetaChain with 2 nodes (zetacore0, zetacore1) using the upgrade option for cosmovisor
 # - ZetaChain observer set with 2 clients (zetaclient0, zetaclient1) using the background option

contrib/localnet/docker-compose.yml

@@ -18,6 +18,9 @@ services:
     image: zetanode:latest
     container_name: rosetta
     hostname: rosetta
+    depends_on:
+      zetacore0:
+        condition: service_healthy
     ports:
       - "8080:8080"
     networks:

contrib/localnet/scripts/import-data.sh

@@ -8,7 +8,7 @@ fi
 NETWORK=$1
 echo "NETWORK: ${NETWORK}"
 rm -rf ~/.zetacored/genesis_data
-mkdir ~/.zetacored/genesis_data
+mkdir -p ~/.zetacored/genesis_data
 echo "Download Latest State Export"
 LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}")
 echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}"

contrib/localnet/scripts/start-upgrade-orchestrator.sh

@@ -63,7 +63,7 @@ cat > upgrade.json <<EOF
     }
   ],
   "metadata": "",
-  "deposit": "100000000azeta",
+  "deposit": "1000000000000000000000azeta",
   "title": "${UPGRADE_NAME}",
   "summary": "${UPGRADE_NAME}"
 }

contrib/localnet/scripts/start-zetacored.sh

@@ -201,7 +201,6 @@ then
   cat $HOME/.zetacored/config/genesis.json | jq '.app_state["mint"]["params"]["mint_denom"]="azeta"' > $HOME/.zetacored/config/tmp_genesis.json && mv $HOME/.zetacored/config/tmp_genesis.json $HOME/.zetacored/config/genesis.json
   cat $HOME/.zetacored/config/genesis.json | jq '.app_state["evm"]["params"]["evm_denom"]="azeta"' > $HOME/.zetacored/config/tmp_genesis.json && mv $HOME/.zetacored/config/tmp_genesis.json $HOME/.zetacored/config/genesis.json
   cat $HOME/.zetacored/config/genesis.json | jq '.consensus_params["block"]["max_gas"]="500000000"' > $HOME/.zetacored/config/tmp_genesis.json && mv $HOME/.zetacored/config/tmp_genesis.json $HOME/.zetacored/config/genesis.json
-  cat $HOME/.zetacored/config/genesis.json | jq '.app_state["gov"]["voting_params"]["voting_period"]="100s"' > $HOME/.zetacored/config/tmp_genesis.json && mv $HOME/.zetacored/config/tmp_genesis.json $HOME/.zetacored/config/genesis.json
   cat $HOME/.zetacored/config/genesis.json | jq '.app_state["feemarket"]["params"]["min_gas_price"]="10000000000.0000"' > $HOME/.zetacored/config/tmp_genesis.json && mv $HOME/.zetacored/config/tmp_genesis.json $HOME/.zetacored/config/genesis.json
 
   # set governance parameters in new params module for sdk v0.47+
@@ -249,10 +248,13 @@ then
       scp $NODE:~/.zetacored/config/gentx/* ~/.zetacored/config/gentx/z2gentx/
   done
 
+#  TODO : USE --modify flag to modify the genesis file when v18 is released
   if [[ -n "$ZETACORED_IMPORT_GENESIS_DATA" ]]; then
     echo "Importing data"
     zetacored parse-genesis-file /root/genesis_data/exported-genesis.json
   fi
+#  Update governance voting period to 100s , to ignore the voting period imported from mainnet.
+  cat $HOME/.zetacored/config/genesis.json | jq '.app_state["gov"]["voting_params"]["voting_period"]="100s"' > $HOME/.zetacored/config/tmp_genesis.json && mv $HOME/.zetacored/config/tmp_genesis.json $HOME/.zetacored/config/genesis.json
 
 # 4. Collect all the gentx files in zetacore0 and create the final genesis.json
   zetacored collect-gentxs

docs/cli/zetacored/zetacored_parse-genesis-file.md

@@ -10,7 +10,7 @@ zetacored parse-genesis-file [import-genesis-file] [optional-genesis-file] [flag
 
 ```
   -h, --help     help for parse-genesis-file
-      --modify   Modify the genesis file before importing
+      --modify   modify the genesis file before importing
 ```
 
 ### Options inherited from parent commands

e2e/txserver/zeta_tx_server.go

@@ -118,7 +118,7 @@ func NewZetaTxServer(rpcAddr string, names []string, privateKeys []string, chain
 		txFactory:    txf,
 		name:         names,
 		address:      addresses,
-		blockTimeout: 1 * time.Minute,
+		blockTimeout: 2 * time.Minute,
 	}, nil
 }
 
@@ -199,7 +199,6 @@ func (zts ZetaTxServer) BroadcastTx(account string, msg sdktypes.Msg) (*sdktypes
 	if err != nil {
 		return nil, err
 	}
-
 	return broadcastWithBlockTimeout(zts, txBytes)
 }
 
@@ -270,7 +269,6 @@ func (zts ZetaTxServer) EnableHeaderVerification(account string, chainIDList []i
 		addr.String(),
 		chainIDList,
 	))
-
 	return err
 }