[wip] feat: willow implementation #7941
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ] | |
| merge_group: | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: -Dwarnings | |
| RUSTDOCFLAGS: -Dwarnings | |
| MSRV: "1.75" | |
| SCCACHE_CACHE_SIZE: "50G" | |
| jobs: | |
| tests: | |
| name: CI Test Suite | |
| if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" | |
| uses: './.github/workflows/tests.yaml' | |
| cross: | |
| if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" | |
| timeout-minutes: 30 | |
| name: Cross compile | |
| runs-on: [self-hosted, linux, X64] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - i686-unknown-linux-gnu | |
| - armv7-linux-androideabi | |
| - aarch64-linux-android | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Cleanup Docker | |
| continue-on-error: true | |
| run: | | |
| docker kill $(docker ps -q) | |
| - name: Install cross | |
| # See https://github.com/cross-rs/cross/issues/1222 | |
| run: cargo install cross --git https://github.com/cross-rs/cross | |
| - name: build | |
| # cross tests are currently broken vor armv7 and aarch64 | |
| # see https://github.com/cross-rs/cross/issues/1311. So on | |
| # those platforms we only build but do not run tests. | |
| if: matrix.target != 'i686-unknown-linux-gnu' | |
| run: cross build --all --target ${{ matrix.target }} | |
| env: | |
| RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} | |
| - name: test | |
| # cross tests are currently broken vor armv7 and aarch64 | |
| # see https://github.com/cross-rs/cross/issues/1311 | |
| if: matrix.target == 'i686-unknown-linux-gnu' | |
| run: cross test --all --target ${{ matrix.target }} -- --test-threads=12 | |
| env: | |
| RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG' }} | |
| check_semver: | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_WRAPPER: "sccache" | |
| SCCACHE_GHA_ENABLED: "on" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Setup Environment (PR) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| shell: bash | |
| run: | | |
| echo "HEAD_COMMIT_SHA=$(git rev-parse origin/${{ github.base_ref }})" >> ${GITHUB_ENV} | |
| - name: Setup Environment (Push) | |
| if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }} | |
| shell: bash | |
| run: | | |
| echo "HEAD_COMMIT_SHA=$(git rev-parse origin/main)" >> ${GITHUB_ENV} | |
| - name: Check semver | |
| # uses: obi1kenobi/cargo-semver-checks-action@v2 | |
| uses: n0-computer/cargo-semver-checks-action@feat-baseline | |
| with: | |
| package: iroh, iroh-base, iroh-bytes, iroh-cli, iroh-dns-server, iroh-gossip, iroh-metrics, iroh-net, iroh-sync | |
| baseline-rev: ${{ env.HEAD_COMMIT_SHA }} | |
| use-cache: false | |
| check_fmt_and_docs: | |
| timeout-minutes: 30 | |
| name: Checking fmt and docs | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_WRAPPER: "sccache" | |
| SCCACHE_GHA_ENABLED: "on" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt | |
| - name: Install sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: fmt | |
| run: cargo fmt --all -- --check | |
| - name: Docs | |
| run: cargo doc --workspace --all-features --no-deps --document-private-items | |
| clippy_check: | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_WRAPPER: "sccache" | |
| SCCACHE_GHA_ENABLED: "on" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - name: Install sccache | |
| uses: mozilla-actions/[email protected] | |
| # TODO: We have a bunch of platform-dependent code so should | |
| # probably run this job on the full platform matrix | |
| - name: clippy check (all features) | |
| run: cargo clippy --workspace --all-features --all-targets --bins --tests --benches | |
| - name: clippy check (no features) | |
| run: cargo clippy --workspace --no-default-features --lib --bins --tests | |
| - name: clippy check (default features) | |
| run: cargo clippy --workspace --all-targets | |
| msrv: | |
| if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" | |
| timeout-minutes: 30 | |
| name: Minimal Supported Rust Version | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_WRAPPER: "sccache" | |
| SCCACHE_GHA_ENABLED: "on" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ env.MSRV }} | |
| - name: Install sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Check MSRV all features | |
| run: | | |
| cargo +$MSRV check --workspace --all-targets | |
| cargo_deny: | |
| timeout-minutes: 30 | |
| name: cargo deny | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: EmbarkStudios/cargo-deny-action@v1 | |
| with: | |
| arguments: --workspace --all-features | |
| command: check | |
| command-arguments: "-Dwarnings" | |
| netsim-integration-tests: | |
| if: "github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'flaky-test')" | |
| timeout-minutes: 30 | |
| name: Run network simulations/benchmarks | |
| runs-on: [self-hosted, linux, X64] | |
| env: | |
| # Using self-hosted runners so use local cache for sccache and | |
| # not SCCACHE_GHA_ENABLED. | |
| RUSTC_WRAPPER: "sccache" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install sccache | |
| uses: mozilla-actions/[email protected] | |
| - name: Build iroh | |
| run: | | |
| cargo build --release --workspace --all-features | |
| - name: Fetch and build chuck | |
| run: | | |
| cd .. | |
| rm -rf chuck | |
| git clone https://github.com/n0-computer/chuck.git | |
| cd chuck | |
| cargo build --release | |
| - name: Install netsim deps | |
| run: | | |
| cd ../chuck/netsim | |
| sudo apt update | |
| ./setup.sh | |
| - name: Copy binaries to right location | |
| run: | | |
| cp target/release/iroh ../chuck/netsim/bins/iroh | |
| cp target/release/iroh-relay ../chuck/netsim/bins/iroh-relay | |
| cp ../chuck/target/release/chuck ../chuck/netsim/bins/chuck | |
| - name: Run tests | |
| run: | | |
| cd ../chuck/netsim | |
| sudo kill -9 $(pgrep ovs) || true | |
| sudo mn --clean || true | |
| sudo python3 main.py ${{ runner.debug && '--debug' || ''}} --integration sims/iroh/iroh.json | |
| sudo python3 main.py ${{ runner.debug && '--debug' || ''}} --integration sims/integration | |
| - name: Cleanup | |
| run: | | |
| sudo kill -9 $(pgrep iroh-relay) || true | |
| sudo kill -9 $(pgrep iroh) || true | |
| sudo kill -9 $(pgrep ovs) || true | |
| sudo mn --clean || true | |
| - name: Setup Environment (PR) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| shell: bash | |
| run: | | |
| echo "LAST_COMMIT_SHA=$(git rev-parse --short ${{ github.event.pull_request.head.sha }})" >> ${GITHUB_ENV} | |
| echo "HEAD_REF=${{ github.event.pull_request.head.ref }}" >> ${GITHUB_ENV} | |
| - name: Setup Environment (Push) | |
| if: ${{ github.event_name == 'push' || github.event_name == 'merge_group' }} | |
| shell: bash | |
| run: | | |
| echo "LAST_COMMIT_SHA=$(git rev-parse --short ${GITHUB_SHA})" >> ${GITHUB_ENV} | |
| echo "HEAD_REF=${{ github.head_ref }}" >> ${GITHUB_ENV} | |
| - name: Generate reports | |
| run: | | |
| cd ../chuck/netsim | |
| python3 reports_csv.py --prom --commit ${{ env.LAST_COMMIT_SHA }} > report_prom.txt | |
| python3 reports_csv.py --metro --commit ${{ env.LAST_COMMIT_SHA }} > report_metro.txt | |
| python3 reports_csv.py --metro --integration --commit ${{ env.LAST_COMMIT_SHA }} > report_metro_integration.txt | |
| - name: Dump report | |
| if: ${{ (github.event.pull_request && !github.event.pull_request.head.repo.fork) || !github.event.pull_request }} | |
| run: | | |
| export AWS_ACCESS_KEY_ID=${{secrets.S3_ACCESS_KEY_ID}} | |
| export AWS_SECRET_ACCESS_KEY=${{secrets.S3_ACCESS_KEY}} | |
| export AWS_DEFAULT_REGION=us-west-2 | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -q awscliv2.zip | |
| sudo ./aws/install --update | |
| cd ../chuck/netsim | |
| aws_fname=intg_${{ env.LAST_COMMIT_SHA }}.tar.gz | |
| tar cvzf report.tar.gz report_prom.txt report_metro.txt report_metro_integration.txt logs/ report/ viz/ | |
| aws s3 cp ./report.tar.gz s3://${{secrets.S3_REPORT_BUCKET}}/$aws_fname --no-progress | |
| - name: Echo metrics | |
| run: | | |
| cd ../chuck/netsim | |
| d=$(cat report_metro.txt) | |
| metro_data=$(printf "%s\n " "$d") | |
| echo "$metro_data" | |
| d=$(cat report_metro_integration.txt) | |
| metro_data=$(printf "%s\n " "$d") | |
| echo "$metro_data" |