Merge pull request #3893 from eval-exec/debug/ci_cli_test

CI: Fix CKB command line usage test case by bats-core/bats

.github/workflows/ci_integration_tests_ubuntu.yaml

@@ -91,37 +91,10 @@ jobs:
     - run: rustup component add clippy
     - run: sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config libclang-dev
     - uses: actions/checkout@v3
-    - name: make and copy ckb binary and testcases to bats_testbed directory
+    - name: build ckb and run bats cli test
       run: |
         make build
-        mkdir -p /tmp/bats_testbed
-        cp target/release/ckb /tmp/bats_testbed/
-        cp util/app-config/src/tests/*.bats /tmp/bats_testbed/
-        cp util/app-config/src/tests/cli_test.sh /tmp/bats_testbed/
-        chmod a+x /tmp/bats_testbed/cli_test.sh
-    - name: Setup BATS testing framework
-      uses: mig4/[email protected]
-    - name: Setup Bats libs
-      uses: brokenpip3/[email protected]
-      with:
-        support-install: true
-        support-path: /usr/lib/bats-support
-        assert-install: true
-        assert-path: /usr/lib/bats-assert
-        detik-install: false
-        file-install: false
-    - name: checkout pre-synced file from ckb-asset repo
-      uses: actions/checkout@v3
-      with:
-        repository: nervosnetwork/ckb-assets
-        ref: v0.1.0
-    - name: init ckb bats testbed
-      run: |
-        mv cli_bats_env/ckb_mainnet_4000.json /tmp/bats_testbed/ckb_mainnet_4000.json
-        cd /tmp/bats_testbed && ./ckb init --force && ./ckb import ckb_mainnet_4000.json
-    - name: run bats cases
-      run: |
-        cd /tmp/bats_testbed && ./cli_test.sh
+        make cli-test
       shell: bash
     env:
       RUNNER_LABEL: ${{ needs.prologue.outputs.linux_runner_label }}

Makefile

@@ -23,6 +23,10 @@ GRCOV_EXCL_LINE = \s*(((log|ckg_logger)::)?(trace|debug|info|warn|error)|(debug_
 doc-test: ## Run doc tests
 	cargo test --all --doc
 
+.PHONY: cli-test
+cli-test: build # Run ckb command line usage bats test
+	./util/app-config/src/tests/cli_test.sh
+
 .PHONY: test
 test: ## Run all tests, including some tests can be time-consuming to execute (tagged with [ignore])
 	cargo nextest run ${VERBOSE} --features ${CKB_FEATURES} --workspace --hide-progress-bar --success-output immediate-final --failure-output immediate-final --run-ignored all

util/app-config/src/tests/cli.bats

@@ -30,47 +30,55 @@ _full_help() {
 #@test "ckb -V" {
 function short_version { #@test
   run _short
-  [ "$status" -eq 0 ]
+  assert_success
   assert_output --regexp "^ckb [0-9.]+[-]?[a-z]*$"
 }
 
 #@test "ckb --version" {
 function long_version { #@test
   run _long
-  [ "$status" -eq 0 ]
+  assert_success
   assert_output --regexp "^ckb [0-9.]+-.*\([0-9a-z-]+ [0-9]{4}-[0-9]{2}-[0-9]{2}\)$"
 }
 
 function help { #@test
   run _help
-  [ "$status" -eq 0 ]
-  assert_output --regexp "USAGE:.*OPTIONS:.*SUBCOMMANDS:.*"
+  assert_success
+  assert_output --regexp ".*Usage:.*Commands:.*Options:.*"
 
   run _full_help
-  [ "$status" -eq 0 ]
-  assert_output --regexp "USAGE:.*OPTIONS:.*SUBCOMMANDS:.*"
+  assert_success
+  assert_output --regexp ".*Usage:.*Commands:.*Options:.*"
 }
 
 function list_hashes { #@test
   run _list_hashes
-  [ "$status" -eq 0 ]
-  assert_output --regexp "\# Spec: ckb[_]?[a-z]*.*\[ckb[_]?[a-z]*\].*spec_hash = \"0x[0-9a-z]*\""
+  assert_success
+  assert_line --index 0 '# Generated by: ckb list-hashes'
+  assert_line --index 1 '[ckb]'
+  assert_line --index 2 --regexp 'spec_hash = "0x[0-9a-fA-F]{64}"'
+  assert_line --index 3 --regexp 'genesis = "0x[0-9a-fA-F]{64}"'
+  assert_line --index 4 --regexp 'cellbase = "0x[0-9a-fA-F]{64}"'
 }
 
 function list_bundle_hashes { #@test
   run _list_bundle_hashes
-  [ "$status" -eq 0 ]
-  assert_output --regexp "\# Spec: ckb.*\[ckb\].*spec_hash = \"0x[0-9a-z]*\""
+  assert_success
+  assert_line --index 0 '# Generated by: ckb list-hashes -b'
+  assert_line --index 1 '[ckb]'
+  assert_line --index 2 --regexp 'spec_hash = "0x[0-9a-fA-F]{64}"'
+  assert_line --index 3 --regexp 'genesis = "0x[0-9a-fA-F]{64}"'
+  assert_line --index 4 --regexp 'cellbase = "0x[0-9a-fA-F]{64}"'
 }
 
 function stats_default { #@test
   run _stats_default
-  [ "$status" -eq 0 ]
+  assert_success
   assert_output --regexp "uncle_rate:.*by_miner_script:.*by_miner_message:.*"
 }
 
 function stats_with_range { #@test
   run _stats_with_range
-  [ "$status" -eq 0 ]
+  assert_success
   assert_output --regexp "uncle_rate:.*by_miner_script:.*by_miner_message:.*"
 }

util/app-config/src/tests/cli_test.sh

@@ -1,11 +1,53 @@
 #!/usr/bin/env bash
-export PATH=$PATH:/tmp/bats_testbed
-export BATS_LIB_PATH=/usr/lib
-export CKB_DIRNAME=/tmp/bats_testbed
-export TMP_DIR=/tmp
+set -euxo pipefail
+
+CKB_BATS_TESTBED=/tmp/ckb_bats_testbed
+mkdir -p ${CKB_BATS_TESTBED}
+
+function cleanup {
+  echo "Removing ${CKB_BATS_TESTBED}"
+  rm -rf ${CKB_BATS_TESTBED}
+}
+
+trap cleanup EXIT
+
+cp target/release/ckb ${CKB_BATS_TESTBED}
+cp util/app-config/src/tests/*.bats ${CKB_BATS_TESTBED}
+cp util/app-config/src/tests/*.sh ${CKB_BATS_TESTBED}
+
+if [ ! -d "/tmp/ckb_bats_assets/" ]; then
+  git clone --depth=1 https://github.com/nervosnetwork/ckb-assets /tmp/ckb_bats_assets
+fi
+cp /tmp/ckb_bats_assets/cli_bats_env/ckb_mainnet_4000.json ${CKB_BATS_TESTBED}
+
+CKB_BATS_CORE_DIR=/tmp/ckb_bats_core
+if [ ! -d "${CKB_BATS_CORE_DIR}/bats" ]; then
+  git clone --depth 1 --branch v1.9.0 https://github.com/bats-core/bats-core.git    ${CKB_BATS_CORE_DIR}/bats
+  ${CKB_BATS_CORE_DIR}/bats/install.sh /tmp/ckb_bats_bin/tmp_install
+fi
+
+if [ ! -d "${CKB_BATS_CORE_DIR}/bats-support" ]; then
+  git clone --depth 1 --branch v0.3.0 https://github.com/bats-core/bats-support.git ${CKB_BATS_CORE_DIR}/bats-support
+fi
+bash ${CKB_BATS_CORE_DIR}/bats-support/load.bash
+
+if [ ! -d "${CKB_BATS_CORE_DIR}/bats-assert" ]; then
+  git clone --depth 1 --branch v2.1.0 https://github.com/bats-core/bats-assert.git  ${CKB_BATS_CORE_DIR}/bats-assert
+fi
+bash ${CKB_BATS_CORE_DIR}/bats-assert/load.bash
+
+cd ${CKB_BATS_TESTBED}
+
+./ckb init --force && ./ckb import ckb_mainnet_4000.json
+
+export PATH=${CKB_BATS_TESTBED}:/tmp/ckb_bats_bin/tmp_install/bin:${PATH}
+export BATS_LIB_PATH=${CKB_BATS_CORE_DIR}
+export CKB_DIRNAME=${CKB_BATS_TESTBED}
+export TMP_DIR=${CKB_BATS_TESTBED}/tmp_dir
+mkdir ${TMP_DIR}
 
 for bats_cases in *.bats; do
-  bats "$bats_cases"
+  bats --trace "$bats_cases"
   ret=$?
   if [ "$ret" -ne "0" ]; then
     exit "$ret"