From 5a29e43fc61227216c18f2177d316a66ccdd766b Mon Sep 17 00:00:00 2001 From: Mohanson Date: Wed, 20 Sep 2023 13:50:15 +0800 Subject: [PATCH] Use github action instead of Azure pipelines (#386) * Using github actions to rewrite: LinuxCIDeps * Using github actions to rewrite: LinuxCIGenerated * Using github actions to rewrite: LinuxCIASM * Using github actions to rewrite: LinuxSpawnStackCheck * Using github actions to rewrite: LinuxCI * Using github actions to rewrite: OSXCI * Using github actions to rewrite: WinCI * Using github actions to rewrite: LinuxCIASMChaos * Using github actions to rewrite: LinuxTestSuite * Using github actions to rewrite: LinuxArm64AsmCI * Using github actions to rewrite: LinuxArm64TestSuite * Clean codes * Use independent jobs * Use clearer step names --- .github/workflows/develop.yml | 179 +++++++++++++++++++ Makefile | 18 -- azure-pipelines.yml | 225 ------------------------ devtools/azure/linux-dependencies.yml | 12 -- devtools/azure/osx-dependencies.yml | 14 -- devtools/azure/windows-dependencies.yml | 28 --- 6 files changed, 179 insertions(+), 297 deletions(-) create mode 100644 .github/workflows/develop.yml delete mode 100644 azure-pipelines.yml delete mode 100644 devtools/azure/linux-dependencies.yml delete mode 100644 devtools/azure/osx-dependencies.yml delete mode 100644 devtools/azure/windows-dependencies.yml diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml new file mode 100644 index 00000000..abc2d530 --- /dev/null +++ b/.github/workflows/develop.yml @@ -0,0 +1,179 @@ +name: develop + +on: + pull_request: + push: + +jobs: + + linux-x86-deny: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + # https://github.com/EmbarkStudios/cargo-deny-action + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + arguments: --all-features + command: check advisories licenses sources bans + + linux-x86-ci-generated: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run ci-generated + run: make ci-generated + + linux-x86-ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run ci + run: rustup component add clippy rustfmt && make ci + + linux-x86-ci-asm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run ci-asm + run: make ci-asm + + linux-x86-ci-asm-chaos: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run ci-asm-chaos + run: make ci-asm-chaos + + linux-x86-test-suite: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install dependencies + run: | + sudo apt install -y git \ + build-essential \ + autoconf \ + automake \ + autotools-dev \ + libmpc-dev \ + libmpfr-dev \ + libgmp-dev \ + gawk \ + libtool \ + patchutils \ + libexpat-dev \ + zlib1g-dev + - name: Run test suite + run: | + git clone https://github.com/nervosnetwork/ckb-vm-test-suite + ln -snf .. ckb-vm-test-suite/ckb-vm + docker run --rm -v `pwd`:/code nervos/ckb-riscv-gnu-toolchain:bionic-20210804 cp -r /riscv /code/riscv + cd ckb-vm-test-suite + git checkout 86480364649c9cb6ac01674fe51156e7cf50a31a + git submodule update --init --recursive + RISCV=`pwd`/../riscv ./test.sh + + linux-x86-test-spawn: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run spawn tests in release + run: cargo test test_spawn --release --features=asm -- --nocapture + + linux-arm-ci-asm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt install -y build-essential \ + autoconf \ + automake \ + autotools-dev \ + libmpc-dev \ + libmpfr-dev \ + libgmp-dev \ + gawk \ + libtool \ + patchutils \ + libexpat-dev \ + zlib1g-dev \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + qemu-user-static + rustup target add aarch64-unknown-linux-gnu + - name: Run ci-asm + run: | + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="-C link-args=-L -C link-args=/usr/lib/gcc-cross/aarch64-linux-gnu/11" && + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64-static -L /usr/aarch64-linux-gnu" && + cargo test --features=asm --target aarch64-unknown-linux-gnu + + linux-arm-test-suite: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install dependencies + run: | + sudo apt update -y + sudo apt install -y build-essential \ + autoconf \ + automake \ + autotools-dev \ + libmpc-dev \ + libmpfr-dev \ + libgmp-dev \ + gawk \ + libtool \ + patchutils \ + libexpat-dev \ + zlib1g-dev + rustup target add aarch64-unknown-linux-gnu + - name: Build test suite + run: | + git clone https://github.com/nervosnetwork/ckb-vm-test-suite + ln -snf .. ckb-vm-test-suite/ckb-vm + docker run --rm -v `pwd`:/code nervos/ckb-riscv-gnu-toolchain:bionic-20210804 cp -r /riscv /code/riscv + cd ckb-vm-test-suite + git checkout 86480364649c9cb6ac01674fe51156e7cf50a31a + git submodule update --init --recursive + RISCV=`pwd`/../riscv ./test.sh --build-only + cd .. + - name: Run test suite + run: | + sudo apt install -y qemu binfmt-support qemu-user-static + sudo apt install -y gcc-multilib + sudo apt install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu clang + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + cd ckb-vm-test-suite + cd binary && cargo build --release --target=aarch64-unknown-linux-gnu && cd .. + cd .. + docker run --rm -v `pwd`:/code -t arm64v8/rust bash -c "RISCV=/dummy /code/ckb-vm-test-suite/test.sh --prebuilt-prefix aarch64-unknown-linux-gnu" + + macos-x86-ci-asm: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Run ci-asm + run: make ci-asm + + windows-x86-ci-asm: + # We must use windows-2019 here, otherwise we will encounter some bugs, for futher details, see + # https://github.com/yasm/yasm/issues/153 + runs-on: windows-2019 + steps: + - uses: actions/checkout@v3 + - name: Install dependencies + shell: pwsh + # https://github.com/ScoopInstaller/Install#for-admin + run: | + iex "& {$(irm get.scoop.sh)} -RunAsAdmin" + scoop install llvm + scoop install yasm + - name: Run ci-asm + shell: pwsh + run: | + $env:path="C:\Users\runneradmin\scoop\apps\llvm\current;"+$env:path + $env:path="C:\Users\runneradmin\scoop\apps\yasm\current;"+$env:path + make ci-asm diff --git a/Makefile b/Makefile index ae888cf0..36ad8fad 100644 --- a/Makefile +++ b/Makefile @@ -40,9 +40,6 @@ fuzz: ci: fmt check clippy test git diff --exit-code Cargo.lock -ci-deps: security-audit check-licenses check-crates - git diff --exit-code Cargo.lock - ci-asm: test-asm git diff --exit-code Cargo.lock @@ -52,21 +49,6 @@ ci-asm-chaos: test-asm-chaos ci-generated: update-cdefinitions git diff --exit-code src/machine/asm/cdefinitions_generated.h -# Use cargo-deny to audit Cargo.lock for crates with security vulnerabilities -security-audit: - @cargo deny --version || cargo install cargo-deny - @cargo deny check --hide-inclusion-graph --show-stats advisories sources - -# Use cargo-deny to check licenses for all dependencies. -check-licenses: - @cargo deny --version || cargo install cargo-deny - @cargo deny check --hide-inclusion-graph --show-stats licenses - -# Use cargo-deny to check specific crates, detect and handle multiple versions of the same crate and wildcards version requirement. -check-crates: - @cargo deny --version || cargo install cargo-deny - @cargo deny check --hide-inclusion-graph --show-stats bans - update-cdefinitions: cargo run --manifest-path=definitions/Cargo.toml --bin generate_asm_constants > src/machine/asm/cdefinitions_generated.h diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 65e2cca9..00000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,225 +0,0 @@ -trigger: - branches: - include: - - '*' - tags: - include: - - '*' - -variables: - TEST_SUITE_COMMIT: 86480364649c9cb6ac01674fe51156e7cf50a31a - -jobs: - - job: LinuxCIDeps - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: cargo install cargo-deny --locked --version 0.12.1 - displayName: Install cargo deny - - script: make ci-deps - displayName: Run ci-deps - - - job: LinuxCIGenerated - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: make ci-generated - displayName: Run ci-generated - - - job: LinuxCIASM - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: make ci-asm - displayName: Run ci-asm - - - job: LinuxSpawnStackCheck - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: cargo test test_spawn --release --features=asm -- --nocapture - displayName: Run spawn tests - - - job: LinuxCI - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: make ci - displayName: Run ci - - - job: LinuxCIASMChaos - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: make ci-asm-chaos - displayName: Run ci-asm-chaos - - - job: LinuxTestSuite - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: | - sudo apt install -y git \ - build-essential \ - autoconf \ - automake \ - autotools-dev \ - libmpc-dev \ - libmpfr-dev \ - libgmp-dev \ - gawk \ - libtool \ - patchutils \ - libexpat-dev \ - zlib1g-dev - - script: | - git clone https://github.com/nervosnetwork/ckb-vm-test-suite && - ln -snf .. ckb-vm-test-suite/ckb-vm && - docker run --rm -v `pwd`:/code nervos/ckb-riscv-gnu-toolchain:bionic-20210804 cp -r /riscv /code/riscv && - cd ckb-vm-test-suite && - git checkout $TEST_SUITE_COMMIT && - git submodule update --init --recursive && - RISCV=`pwd`/../riscv ./test.sh - displayName: Run test suite - - - job: LinuxArm64AsmCI - pool: - vmImage: 'ubuntu-22.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: | - sudo apt-get update -y && - sudo apt install -y git \ - build-essential \ - autoconf \ - automake \ - autotools-dev \ - libmpc-dev \ - libmpfr-dev \ - libgmp-dev \ - gawk \ - libtool \ - patchutils \ - libexpat-dev \ - zlib1g-dev \ - gcc-aarch64-linux-gnu \ - g++-aarch64-linux-gnu \ - qemu-user-static && - rustup target add aarch64-unknown-linux-gnu - - script: | - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS="-C link-args=-L -C link-args=/usr/lib/gcc-cross/aarch64-linux-gnu/11" && - export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64-static -L /usr/aarch64-linux-gnu" && - cargo test --features=asm --target aarch64-unknown-linux-gnu - displayName: Run ci-asm on arm64 machines - - - job: LinuxArm64TestSuite - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: | - sudo apt update -y && - sudo apt install -y git \ - build-essential \ - autoconf \ - automake \ - autotools-dev \ - libmpc-dev \ - libmpfr-dev \ - libgmp-dev \ - gawk \ - libtool \ - patchutils \ - libexpat-dev \ - zlib1g-dev - - script: | - git clone https://github.com/nervosnetwork/ckb-vm-test-suite && - ln -snf .. ckb-vm-test-suite/ckb-vm && - docker run --rm -v `pwd`:/code nervos/ckb-riscv-gnu-toolchain:bionic-20210804 cp -r /riscv /code/riscv && - cd ckb-vm-test-suite && - git checkout $TEST_SUITE_COMMIT && - git submodule update --init --recursive && - RISCV=`pwd`/../riscv ./test.sh --build-only && - cd .. - displayName: Build test suite - - script: | - sudo apt-get install -y qemu binfmt-support qemu-user-static - sudo apt-get install -y gcc-multilib - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu clang - rustup target add aarch64-unknown-linux-gnu - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - cd ckb-vm-test-suite && - cd binary && cargo build --release --target=aarch64-unknown-linux-gnu && cd .. && - cd .. && - docker run --rm -v `pwd`:/code -t arm64v8/rust bash -c "RISCV=/dummy /code/ckb-vm-test-suite/test.sh --prebuilt-prefix aarch64-unknown-linux-gnu" - displayName: Run test suite on arm64 machines - - - job: LinuxCodeCoverage - condition: eq(variables['Build.SourceBranch'], 'refs/heads/release') - pool: - vmImage: 'ubuntu-20.04' - steps: - - template: devtools/azure/linux-dependencies.yml - - script: | - sudo apt install -y git \ - build-essential \ - autoconf \ - automake \ - autotools-dev \ - libmpc-dev \ - libmpfr-dev \ - libgmp-dev \ - gawk \ - libtool \ - patchutils \ - libexpat-dev \ - binutils-dev \ - libcurl4-openssl-dev \ - zlib1g-dev \ - libdw-dev \ - libiberty-dev - - script: | - wget https://github.com/SimonKagstrom/kcov/archive/v36.tar.gz && - tar xzf v36.tar.gz && - cd kcov-36 && - mkdir build && - cd build && - cmake .. && - make && - sudo make install && - cd ../.. && - rm -rf kcov-36 v36.tar.gz && - git clone https://github.com/nervosnetwork/ckb-vm-test-suite && - ln -snf .. ckb-vm-test-suite/ckb-vm && - docker run --rm -v `pwd`:/code nervos/ckb-riscv-gnu-toolchain:bionic-20210804 cp -r /riscv /code/riscv && - cd ckb-vm-test-suite && git checkout $TEST_SUITE_COMMIT && git submodule update --init --recursive && - RISCV=`pwd`/../riscv ./test.sh --coverage && - cd .. && - make cov && - bash <(curl -s https://codecov.io/bash) && - echo "Uploaded code coverage" - displayName: Run code coverage - - - job: OSXCI - pool: - vmImage: 'macOS-11' - steps: - - template: devtools/azure/osx-dependencies.yml - - script: make ci-asm - displayName: Run unit tests - - - job: WinCI - pool: - vmImage: 'windows-2019' - steps: - - template: devtools/azure/windows-dependencies.yml - - script: make ci-asm - displayName: Run unit tests diff --git a/devtools/azure/linux-dependencies.yml b/devtools/azure/linux-dependencies.yml deleted file mode 100644 index 05ec5f9c..00000000 --- a/devtools/azure/linux-dependencies.yml +++ /dev/null @@ -1,12 +0,0 @@ -steps: -- script: | - sudo apt install -y build-essential -- script: | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.71.1-x86_64-unknown-linux-gnu - set PATH=$PATH:~/.cargo/bin - rustup install nightly - displayName: Install rust -- script: | - rustc --version - cargo --version - displayName: Test/query binaries diff --git a/devtools/azure/osx-dependencies.yml b/devtools/azure/osx-dependencies.yml deleted file mode 100644 index 6fa7d0e8..00000000 --- a/devtools/azure/osx-dependencies.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: -- script: | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.71.1-x86_64-apple-darwin - set PATH=$PATH:~/.cargo/bin - rustup install nightly - displayName: Install rust -- script: rustup component add rustfmt - displayName: Install rustfmt -- script: rustup component add clippy - displayName: Install clippy -- script: | - rustc --version - cargo --version - displayName: Test/query binaries diff --git a/devtools/azure/windows-dependencies.yml b/devtools/azure/windows-dependencies.yml deleted file mode 100644 index 486d41a3..00000000 --- a/devtools/azure/windows-dependencies.yml +++ /dev/null @@ -1,28 +0,0 @@ -steps: -- powershell: | - iwr -useb get.scoop.sh -outfile 'install.ps1' - .\install.ps1 -RunAsAdmin - rm install.ps1 - displayName: Install scoop -- script: | - set PATH=%PATH%;%USERPROFILE%\scoop\shims - echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\scoop\shims" - scoop help - displayName: Add scoop to path -- script: scoop install llvm - displayName: Install LLVM -- script: scoop install yasm - displayName: Install yasm -- script: | - curl -sSf -o rustup-init.exe https://win.rustup.rs - rustup-init.exe -y --default-toolchain 1.71.1-x86_64-pc-windows-msvc - set PATH=%PATH%;%USERPROFILE%\.cargo\bin - echo "##vso[task.setvariable variable=PATH;]%PATH%;%USERPROFILE%\.cargo\bin" - rustup install nightly - displayName: Install rust -- script: | - rustc --version - cargo --version - clang --version - yasm --version - displayName: Test/query binaries