From 84b5fe8b75f4e91ba442a9e4a0280f2ff41faca7 Mon Sep 17 00:00:00 2001 From: "Tobin C. Harding" Date: Thu, 2 May 2024 15:35:14 +1000 Subject: [PATCH] CI: Use script from rust-bitcoin-maintainer-tools We have a CI script in the `rust-bitcoin-maintainer-tools` repository, lets use it. --- .github/workflows/README.md | 24 ++++ .github/workflows/rust.yml | 225 ++++++++++++++++++++++++++++-------- contrib/crates.sh | 8 ++ contrib/pin.sh | 7 ++ contrib/test.sh | 100 ---------------- contrib/test_vars.sh | 19 +++ contrib/whitelist_deps.sh | 8 ++ 7 files changed, 240 insertions(+), 151 deletions(-) create mode 100644 .github/workflows/README.md create mode 100755 contrib/crates.sh create mode 100755 contrib/pin.sh delete mode 100755 contrib/test.sh create mode 100644 contrib/test_vars.sh create mode 100644 contrib/whitelist_deps.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..df332f741 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,24 @@ +# rust-miniscript workflow notes + +We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). + +ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration + +The minimal/recent lock files are handled by CI (`rust.yml`). + +## Jobs + +Run from `rust.yml` unless stated otherwise. Total 11 jobs. + +1. `Stable - minimal` +2. `Stable - recent` +3. `Nightly - minimal` +4. `Nightly - recent` +5. `MSRV - minimal` +6. `Lint` +7. `Docs` +8. `Docsrs` +9. `Bench` +10. `Format` +10. `Int-tests` +11. `Embedded` diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bb5b9b704..033cc6d93 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,64 +1,190 @@ -on: [push, pull_request] +--- # rust-miniscript CI: If you edit this file please update README.md +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - 'test-ci/**' + pull_request: name: Continuous integration jobs: - Nightly: - name: Nightly - Bench + Docs + Fmt + Stable: # 2 jobs, one per lock file. + name: Test - stable toolchain runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [minimal, recent] steps: - - name: Checkout Crate - uses: actions/checkout@v2 - - name: Checkout Toolchain - uses: actions-rs/toolchain@v1 + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 with: - profile: minimal - toolchain: nightly - override: true - - name: Install clippy + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@stable + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh stable + + Nightly: # 2 jobs, one per lock file. + name: Test - nightly toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [minimal, recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@nightly + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh nightly + + MSRV: # 1 jobs, minimal lock file only. + name: Test - 1.56.1 toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [minimal] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.56.1" + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh msrv + + Lint: + name: Lint - nightly toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@nightly + - name: "Install clippy" run: rustup component add clippy - - name: Running benchmarks - env: - DO_BENCH: true - run: ./contrib/test.sh - - name: Building docs - env: - DO_DOCS: true - run: ./contrib/test.sh - - name: Running formatter - env: - DO_FMT: true - run: ./contrib/test.sh - - name: Running linter - env: - DO_LINT: true - run: ./contrib/test.sh + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh lint - Int-tests: - name: Integration tests + Docs: + name: Docs - stable toolchain runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] steps: - - name: Checkout Crate - uses: actions/checkout@v2 - - name: Checkout Toolchain - uses: actions-rs/toolchain@v1 + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 with: - profile: minimal - toolchain: stable - override: true - - name: Running integration tests - run: ./contrib/integration_test.sh + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@stable + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh docs + + Docsrs: + name: Docs - nightly toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@nightly + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh docsrs - Tests: - name: Tests + Bench: + name: Bench - nightly toolchain runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - include: - - rust: stable - - rust: beta - - rust: nightly - - rust: "1.56.1" + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Checkout maintainer tools" + uses: actions/checkout@v4 + with: + repository: rust-bitcoin/rust-bitcoin-maintainer-tools + path: maintainer-tools + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@nightly + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./maintainer-tools/ci/run_task.sh bench + + Format: # 1 jobs, run cargo fmt directly. + name: Format - nightly toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@nightly + - name: "Install rustfmt" + run: rustup component add rustfmt + - name: "Check formatting" + run: cargo +nightly fmt --all -- --check + + Int-tests: + name: Integration tests + runs-on: ubuntu-latest steps: - name: Checkout Crate uses: actions/checkout@v2 @@ -66,13 +192,10 @@ jobs: uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: ${{ matrix.rust }} + toolchain: stable override: true - - name: Running cargo - env: - DO_FEATURE_MATRIX: true - DO_NO_STD: true - run: ./contrib/test.sh + - name: Running integration tests + run: ./contrib/integration_test.sh Embedded: runs-on: ubuntu-latest diff --git a/contrib/crates.sh b/contrib/crates.sh new file mode 100755 index 000000000..6d1159582 --- /dev/null +++ b/contrib/crates.sh @@ -0,0 +1,8 @@ +# No shebang, this file should not be executed. +# shellcheck disable=SC2148 +# +# disable verify unused vars, despite the fact that they are used when sourced +# shellcheck disable=SC2034 + +# Crates in this workspace to test (excl. fuzz an integration-tests). +CRATES=(".") # Non-workspaces don't have crates. diff --git a/contrib/pin.sh b/contrib/pin.sh new file mode 100755 index 000000000..dd7c2227e --- /dev/null +++ b/contrib/pin.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# +# Do pinning as required for current MSRV. + +set -euo pipefail + +cargo update -p cc --precise 1.0.79 diff --git a/contrib/test.sh b/contrib/test.sh deleted file mode 100755 index 15153055f..000000000 --- a/contrib/test.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/sh - -set -ex - -FEATURES="compiler serde rand base64" - -cargo --version -rustc --version - -# Cache the toolchain we are using. -NIGHTLY=false -MSRV=false -if cargo --version | grep nightly; then - NIGHTLY=true -elif cargo --version | grep "1\.56"; then - MSRV=true -fi - -if [ "$MSRV" = true ]; then - cargo update -p cc --precise 1.0.79 -fi - -# Format if told to -if [ "$DO_FMT" = true ] -then - rustup component add rustfmt - cargo fmt -- --check -fi - -# Defaults / sanity checks -cargo test - -if [ "$DO_LINT" = true ]; then - clippy="cargo +nightly clippy" - - $clippy --all-features --all-targets -- -D warnings - $clippy --all-targets -- -D warnings - $clippy --no-default-features --features=no-std --all-targets -- -D warnings -fi - -if [ "$DO_FEATURE_MATRIX" = true ] -then - # All features - cargo test --features="$FEATURES" - - # Single features - for feature in ${FEATURES} - do - cargo test --features="$feature" - done - - # Run all the examples - cargo build --examples - cargo run --example htlc --features=compiler - cargo run --example parse - cargo run --example sign_multisig - cargo run --example verify_tx > /dev/null - cargo run --example xpub_descriptors - cargo run --example taproot --features=compiler - cargo run --example psbt_sign_finalize --features=base64 -fi - -if [ "$DO_NO_STD" = true ] -then - # Build no_std, to make sure that cfg(test) doesn't hide any issues - cargo build --verbose --no-default-features --features="no-std" - - # Test no_std - cargo test --verbose --no-default-features --features="no-std" - - # Build all features - cargo build --verbose --no-default-features --features="no-std $FEATURES" - - # Build specific features - for feature in ${FEATURES} - do - cargo build --verbose --no-default-features --features="no-std $feature" - done -fi - -# Bench if told to, only works with non-stable toolchain (nightly, beta). -if [ "$DO_BENCH" = true ] -then - if [ "$NIGHTLY" = false ]; then - if [ -n "$RUSTUP_TOOLCHAIN" ]; then - echo "RUSTUP_TOOLCHAIN is set to a non-nightly toolchain but DO_BENCH requires a nightly toolchain" - else - echo "DO_BENCH requires a nightly toolchain" - fi - exit 1 - fi - RUSTFLAGS='--cfg=bench' cargo bench -fi - -# Build the docs if told to (this only works with the nightly toolchain) -if [ "$DO_DOCS" = true ]; then - RUSTDOCFLAGS="--cfg docsrs" cargo +nightly rustdoc --features="$FEATURES" -- -D rustdoc::broken-intra-doc-links -fi - -exit 0 diff --git a/contrib/test_vars.sh b/contrib/test_vars.sh new file mode 100644 index 000000000..337c4e921 --- /dev/null +++ b/contrib/test_vars.sh @@ -0,0 +1,19 @@ +# No shebang, this file should not be executed. +# shellcheck disable=SC2148 +# +# disable verify unused vars, despite the fact that they are used when sourced +# shellcheck disable=SC2034 + +# Test all these features with "std" enabled. +FEATURES_WITH_STD="compiler trace serde rand base64" + +# Test all these features with "no-std" enabled. +# rust-miniscript only: https://github.com/rust-bitcoin/rust-miniscript/issues/681 +FEATURES_WITH_NO_STD="compiler trace serde rand base64" + +# Test all these features without "std" enabled. +FEATURES_WITHOUT_STD="" + +# Run these examples. +# Note `examples/big` should not be run. +EXAMPLES="htlc:std,compiler parse:std sign_multisig:std verify_tx:std xpub_descriptors:std taproot:std,compiler psbt_sign_finalize:std,base64" diff --git a/contrib/whitelist_deps.sh b/contrib/whitelist_deps.sh new file mode 100644 index 000000000..74619acdf --- /dev/null +++ b/contrib/whitelist_deps.sh @@ -0,0 +1,8 @@ +# No shebang, this file should not be executed. +# shellcheck disable=SC2148 +# +# disable verify unused vars, despite the fact that they are used when sourced +# shellcheck disable=SC2034 + +# Remove once we upgrade to `bitcoin v0.32.0`. +DUPLICATE_DEPS=("bech32")