From 4686a2c068a3c18988a3c3d68d76a73a00999e98 Mon Sep 17 00:00:00 2001 From: Hanno Becker Date: Sat, 21 Dec 2024 20:28:29 +0000 Subject: [PATCH] CI: Add further container-based compatibility tests Signed-off-by: Hanno Becker --- .../{setup-ubuntu => setup-apt}/action.yml | 4 +- .github/actions/setup-os/action.yml | 37 ++++ .github/actions/setup-yum/action.yml | 34 ++++ .github/workflows/bench_ec2_reusable.yml | 2 +- .github/workflows/ci.yml | 92 +++++++-- .github/workflows/ci_ec2_container.yml | 187 ++++++++++++++++++ .github/workflows/ci_ec2_reusable.yml | 35 ---- 7 files changed, 335 insertions(+), 56 deletions(-) rename .github/actions/{setup-ubuntu => setup-apt}/action.yml (93%) create mode 100644 .github/actions/setup-os/action.yml create mode 100644 .github/actions/setup-yum/action.yml create mode 100644 .github/workflows/ci_ec2_container.yml diff --git a/.github/actions/setup-ubuntu/action.yml b/.github/actions/setup-apt/action.yml similarity index 93% rename from .github/actions/setup-ubuntu/action.yml rename to .github/actions/setup-apt/action.yml index eb445ca69..a4348deba 100644 --- a/.github/actions/setup-ubuntu/action.yml +++ b/.github/actions/setup-apt/action.yml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 -name: Setup ubuntu -description: Setup ubuntu +name: Dependencies (apt) +description: Install dependencies via apt inputs: packages: diff --git a/.github/actions/setup-os/action.yml b/.github/actions/setup-os/action.yml new file mode 100644 index 000000000..b1b26feaa --- /dev/null +++ b/.github/actions/setup-os/action.yml @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 + +name: Setup OS +description: Setup OS + +inputs: + packages: + description: Space-separated list of additional packages to install + required: false + default: '' + sudo: + required: false + default: 'sudo' + +runs: + using: composite + steps: + - name: Detect OS + shell: bash + run: | + if (which yum > /dev/null); then + echo PKG="yum" >> $GITHUB_ENV + elif (which apt > /dev/null); then + echo PKG="apt" >> $GITHUB_ENV + fi + - name: Setup via yum + if: ${{ env.PKG == 'yum' }} + uses: ./.github/actions/setup-yum + with: + packages: ${{ inputs.packages }} + sudo: ${{ inputs.sudo }} + - name: Setup via apt + if: ${{ env.PKG == 'apt' }} + uses: ./.github/actions/setup-apt + with: + packages: ${{ inputs.packages }} + sudo: ${{ inputs.sudo }} diff --git a/.github/actions/setup-yum/action.yml b/.github/actions/setup-yum/action.yml new file mode 100644 index 000000000..16f9c3eda --- /dev/null +++ b/.github/actions/setup-yum/action.yml @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 + +name: Dependencies (yum) +description: Install dependencies via yum + +inputs: + packages: + description: Space-separated list of additional packages to install + required: false + default: '' + sudo: + required: false + default: 'sudo' + +runs: + using: composite + steps: + - name: Install base packages + shell: bash + run: | + ${{ inputs.sudo }} yum install make git python3-pip -y + ${{ inputs.sudo }} pip3 install virtualenv + - name: Install additional packages + if: ${{ inputs.packages != ''}} + shell: bash + run: | + ${{ inputs.sudo }} yum install ${{ inputs.packages }} -y + - name: Setup Python venv + shell: bash + run: | + virtualenv venv + source venv/bin/activate + python3 -m pip install -r requirements.txt + echo "$(pwd)/venv/bin/" >> "$GITHUB_PATH" diff --git a/.github/workflows/bench_ec2_reusable.yml b/.github/workflows/bench_ec2_reusable.yml index 7da9c19f7..9205c85c8 100644 --- a/.github/workflows/bench_ec2_reusable.yml +++ b/.github/workflows/bench_ec2_reusable.yml @@ -165,7 +165,7 @@ jobs: if: ${{ inputs.compiler != '' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: ./.github/actions/setup-ubuntu + - uses: ./.github/actions/setup-apt with: packages: ${{ inputs.additional_packages }} - name: Set compiler diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba8b172d7..753592db7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: OPT=0 make quickcheck >/dev/null make clean >/dev/null OPT=1 make quickcheck >/dev/null - - uses: ./.github/actions/setup-ubuntu + - uses: ./.github/actions/setup-apt - name: tests func run: | ./scripts/tests func @@ -93,7 +93,7 @@ jobs: OPT=0 CPPFLAGS=-std=c90 make quickcheck >/dev/null make clean >/dev/null OPT=1 CPPFLAGS=-std=c90 make quickcheck >/dev/null - - uses: ./.github/actions/setup-ubuntu + - uses: ./.github/actions/setup-apt - name: tests func run: | CPPFLAGS="-std=c90" ./scripts/tests func @@ -417,9 +417,9 @@ jobs: permissions: contents: 'read' id-token: 'write' - uses: ./.github/workflows/ci_ec2_reusable.yml needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork + uses: ./.github/workflows/ci_ec2_reusable.yml with: name: ${{ matrix.target.name }} ec2_instance_type: ${{ matrix.target.ec2_instance_type }} @@ -434,35 +434,91 @@ jobs: lint: false verbose: true secrets: inherit + compatibility_tests: + strategy: + max-parallel: 4 + fail-fast: false + matrix: + container: + - id: debian:bullseye + - id: debian:bookworm + name: Compatibility tests (${{ matrix.container.id }}) + runs-on: ubuntu-latest + needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] + container: + ${{ matrix.container.id }} + steps: + # We're not using the checkout action here because on it's not supported + # on all containers we want to test. Resort to a manual checkout. + + # We can't hoist this into an action since calling an action can only + # be done after checkout. + - name: Manual checkout + shell: bash + run: | + if (which yum > /dev/null); then + yum install git -y + elif (which apt > /dev/null); then + apt update + apt install git -y + fi + + git config --global --add safe.directory $GITHUB_WORKSPACE + git init + git remote add origin $GITHUB_SERVER_URL/$GITHUB_REPOSITORY + git fetch origin --depth 1 $GITHUB_SHA + git checkout FETCH_HEAD + - uses: ./.github/actions/setup-os + with: + sudo: "" + - name: make quickcheck + run: | + OPT=0 make quickcheck >/dev/null + make clean >/dev/null + OPT=1 make quickcheck >/dev/null + - name: Functional Tests + uses: ./.github/actions/multi-functest + with: + nix-shell: "" + gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} ec2_compatibilitytests: strategy: + max-parallel: 12 fail-fast: false matrix: container: - - id: ubuntu-22.04:gcc-12x - - id: ubuntu-22.04:gcc-11x - - id: ubuntu-20.04:gcc-8x - - id: ubuntu-20.04:gcc-7x - - id: ubuntu-20.04:clang-9x - - id: ubuntu-20.04:clang-8x - - id: ubuntu-20.04:clang-7x-bm-framework - - id: ubuntu-20.04:clang-7x - - id: ubuntu-20.04:clang-10x - - id: ubuntu-22.04:base - - id: ubuntu-20.04:base + - id: amazonlinux-2-aarch:base + - id: amazonlinux-2-aarch:gcc-7x + - id: amazonlinux-2-aarch:clang-7x + - id: amazonlinux-2023-aarch:base + - id: amazonlinux-2023-aarch:gcc-11x + - id: amazonlinux-2023-aarch:clang-15x + - id: amazonlinux-2023-aarch:clang-15x-sanitizer + # - id: amazonlinux-2023-aarch:cryptofuzz Not yet supported + - id: ubuntu-22.04-aarch:gcc-12x + - id: ubuntu-22.04-aarch:gcc-11x + - id: ubuntu-20.04-aarch:gcc-8x + - id: ubuntu-20.04-aarch:gcc-7x + - id: ubuntu-20.04-aarch:clang-9x + - id: ubuntu-20.04-aarch:clang-8x + - id: ubuntu-20.04-aarch:clang-7x-bm-framework + - id: ubuntu-20.04-aarch:clang-7x + - id: ubuntu-20.04-aarch:clang-10x + - id: ubuntu-22.04-aarch:base + - id: ubuntu-20.04-aarch:base name: Compatibility tests (${{ matrix.container.id }}) - needs: [ec2_functests] + needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] permissions: contents: 'read' id-token: 'write' - uses: ./.github/workflows/ci_ec2_reusable.yml + uses: ./.github/workflows/ci_ec2_container.yml if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork with: container: ${{ matrix.container.id }} name: ${{ matrix.container.id }} - ec2_instance_type: c7g.medium + ec2_instance_type: t4g.small ec2_ami: ubuntu-latest (custom AMI) - ec2_ami_id: ami-0f4b26c5372aa0525 # Has docker images preinstalled + ec2_ami_id: ami-0c9bc1901ef0d1066 # Has docker images preinstalled compile_mode: native opt: all functest: true diff --git a/.github/workflows/ci_ec2_container.yml b/.github/workflows/ci_ec2_container.yml new file mode 100644 index 000000000..345fa995a --- /dev/null +++ b/.github/workflows/ci_ec2_container.yml @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 + +name: ci-ec2-reusable +permissions: + contents: read +on: + workflow_call: + inputs: + name: + type: string + description: Alternative name of instance + default: Graviton2 + ec2_instance_type: + type: string + description: Type if EC2 instance to benchmark on + default: t4g.small + ec2_ami: + type: string + description: Textual description of AMI + default: ubuntu-latest (aarch64) + ec2_ami_id: + type: string + description: AMI ID + default: ami-096ea6a12ea24a797 + cflags: + type: string + description: Custom CFLAGS for compilation + default: "" + verbose: + description: Determine for the log verbosity + type: boolean + default: false + compile_mode: + type: string + description: either all, native, cross or none + default: all + opt: + type: string + description: either all, opt or no_opt + default: all + functest: + type: boolean + default: true + kattest: + type: boolean + default: true + nistkattest: + type: boolean + default: true + acvptest: + type: boolean + default: true + lint: + type: boolean + default: true + cbmc: + type: boolean + default: false + cbmc_mlkem_k: + type: string + default: 2 + container: + type: string + default: '' +env: + AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action + AWS_REGION: us-east-1 + AMI_UBUNTU_LATEST_X86_64: ami-0e86e20dae9224db8 + AMI_UBUNTU_LATEST_AARCH64: ami-096ea6a12ea24a797 +jobs: + start-ec2-runner: + name: Start instance (${{ inputs.ec2_instance_type }}) + permissions: + contents: 'read' + id-token: 'write' + runs-on: ubuntu-latest + if: ${{ always() }} # The point is to make this step non-cancellable, + # avoiding race conditions where an instance is started, + # but isn't yet done registering as a runner and reporting back. + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Determine AMI ID + id: det_ami_id + run: | + if [[ "${{ inputs.ec2_ami }}" == "ubuntu-latest (x86_64)" ]]; then + AMI_ID=${{ env.AMI_UBUNTU_LATEST_X86_64 }} + elif [[ "${{ inputs.ec2_ami }}" == "ubuntu-latest (aarch64)" ]]; then + AMI_ID=${{ env.AMI_UBUNTU_LATEST_AARCH64 }} + elif [[ "${{ inputs.ec2_ami }}" == "ubuntu-latest (custom AMI)" ]]; then + AMI_ID=${{ inputs.ec2_ami_id }} + fi + echo "Using AMI ID: $AMI_ID" + echo "AMI_ID=$AMI_ID" >> $GITHUB_OUTPUT + - name: Clear nix-installer action cache + uses: ./.github/actions/clear-cache + with: + key_prefix: determinatesystem-nix-installer- + repository: ${{ github.repository }} + gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: ${{ env.AWS_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Start EC2 runner + id: start-ec2-runner + uses: mkannwischer/ec2-github-runner@d15c8804522523d2bac7119a01ffff83b7795d87 + with: + mode: start + github-token: ${{ secrets.AWS_GITHUB_TOKEN }} + ec2-image-id: ${{ steps.det_ami_id.outputs.AMI_ID }} + ec2-instance-type: ${{ inputs.ec2_instance_type }} + subnet-id: subnet-07b2729e5e065962f + security-group-id: sg-0ab2e297196c8c381 + tests: + name: Run tests + needs: start-ec2-runner + if: ${{ inputs.container != '' }} + runs-on: ${{ needs.start-ec2-runner.outputs.label }} + container: + localhost:5000/${{ inputs.container }} + steps: + # We're not using the checkout action here because on it's not supported + # on all containers we want to test. Resort to a manual checkout. + # + # We can't hoist this into an action since calling an action can only + # be done after checkout. + - name: Manual checkout + shell: bash + run: | + if /usr/bin/which yum; then + yum install git -y + elif /usr/bin/which apt; then + apt update + apt install git -y + fi + + git init + git remote add origin $GITHUB_SERVER_URL/$GITHUB_REPOSITORY + git fetch origin --depth 1 $GITHUB_SHA + git checkout FETCH_HEAD + - uses: ./.github/actions/setup-os + with: + sudo: "" + - name: make quickcheck + run: | + OPT=0 make quickcheck >/dev/null + make clean >/dev/null + OPT=1 make quickcheck >/dev/null + - name: Functional Tests + uses: ./.github/actions/multi-functest + with: + nix-shell: "" + gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} + cflags: ${{ inputs.cflags }} + compile_mode: ${{ inputs.compile_mode }} + opt: ${{ inputs.opt }} + func: ${{ inputs.functest }} + kat: ${{ inputs.kattest }} + nistkat: ${{ inputs.nistkattest }} + acvp: ${{ inputs.acvptest }} + stop-ec2-runner: + name: Stop instance (${{ inputs.ec2_instance_type }}) + permissions: + contents: 'read' + id-token: 'write' + needs: + - start-ec2-runner + - tests + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + with: + role-to-assume: ${{ env.AWS_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Stop EC2 runner + uses: mkannwischer/ec2-github-runner@d15c8804522523d2bac7119a01ffff83b7795d87 + with: + mode: stop + github-token: ${{ secrets.AWS_GITHUB_TOKEN }} + label: ${{ needs.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-ec2-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/ci_ec2_reusable.yml b/.github/workflows/ci_ec2_reusable.yml index 807ca2466..edf32d719 100644 --- a/.github/workflows/ci_ec2_reusable.yml +++ b/.github/workflows/ci_ec2_reusable.yml @@ -59,9 +59,6 @@ on: cbmc_mlkem_k: type: string default: 2 - container: - type: string - default: '' env: AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action AWS_REGION: us-east-1 @@ -115,40 +112,9 @@ jobs: ec2-instance-type: ${{ inputs.ec2_instance_type }} subnet-id: subnet-07b2729e5e065962f security-group-id: sg-0ab2e297196c8c381 - container_tests: - name: Run container tests - needs: start-ec2-runner - if: ${{ inputs.container != '' }} - runs-on: ${{ needs.start-ec2-runner.outputs.label }} - container: - localhost:5000/${{ inputs.container }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: ./.github/actions/setup-ubuntu - with: - sudo: "" - - name: make quickcheck - run: | - OPT=0 make quickcheck >/dev/null - make clean >/dev/null - OPT=1 make quickcheck >/dev/null - - name: Functional Tests - uses: ./.github/actions/multi-functest - with: - nix-shell: "" - gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} - cflags: ${{ inputs.cflags }} - compile_mode: ${{ inputs.compile_mode }} - opt: ${{ inputs.opt }} - func: ${{ inputs.functest }} - kat: ${{ inputs.kattest }} - nistkat: ${{ inputs.nistkattest }} - acvp: ${{ inputs.acvptest }} - tests: name: Run tests needs: start-ec2-runner - if: ${{ inputs.container == '' }} runs-on: ${{ needs.start-ec2-runner.outputs.label }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -194,7 +160,6 @@ jobs: needs: - start-ec2-runner - tests - - container_tests runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: