From 06fd5bbadc1c32ccde1a7d69059fb637bb6bc7aa Mon Sep 17 00:00:00 2001 From: Hanno Becker Date: Wed, 16 Oct 2024 19:17:50 +0100 Subject: [PATCH] Modify benchmark actions and workflows to allow custom compiler - Remove redundant `nix-shell` argument from `bench` composite action - Use relative path for `tests` script in `bench` action to allow running it from non-nix shell. - Add `setup-ubuntu` composite action for installation of Ubuntu dependencies, if nix is not used. - Modify bench_ec2_reusable and bench_ec2_any to allow specification of custom compiler, instead of CC in nix shell. Signed-off-by: Hanno Becker --- .github/actions/bench/action.yml | 4 +-- .github/actions/setup-shell/action.yml | 9 ++--- .github/actions/setup-ubuntu/action.yml | 35 +++++++++++++++++++ .github/workflows/bench_ec2_any.yml | 15 ++++---- .github/workflows/bench_ec2_reusable.yml | 44 ++++++++++++++++++++---- mk/config.mk | 1 + 6 files changed, 86 insertions(+), 22 deletions(-) create mode 100644 .github/actions/setup-ubuntu/action.yml diff --git a/.github/actions/bench/action.yml b/.github/actions/bench/action.yml index 3df503473..c20638123 100644 --- a/.github/actions/bench/action.yml +++ b/.github/actions/bench/action.yml @@ -69,12 +69,12 @@ runs: - name: Run benchmark shell: ${{ env.SHELL }} run: | - tests bench -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \ + ./scripts/tests bench -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \ --cflags="${{ inputs.cflags }}" --arch-flags="${{ inputs.archflags }}" \ $([[ ${{ inputs.opt }} == "false" ]] && echo "--no-opt") \ -v --output=output.json ${{ inputs.bench_extra_args }} - tests bench --components -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \ + ./scripts/tests bench --components -c ${{ inputs.perf }} --cross-prefix="${{ inputs.cross_prefix }}" \ --cflags="${{ inputs.cflags }}" --arch-flags="${{ inputs.archflags }}" \ $([[ ${{ inputs.opt }} == "false" ]] && echo "--no-opt") \ -v --output=output.json ${{ inputs.bench_extra_args }} diff --git a/.github/actions/setup-shell/action.yml b/.github/actions/setup-shell/action.yml index b5fbd83d8..5e153d2b5 100644 --- a/.github/actions/setup-shell/action.yml +++ b/.github/actions/setup-shell/action.yml @@ -4,11 +4,8 @@ name: Set Shell description: Setup nix or custom shell for workflows inputs: - use-nix: - description: Indicate whether to use nix or not - default: 'true' nix-shell: - description: Run in the specified Nix environment if exists + description: Run in the specified Nix environment if exists. If empty, custom shell will be used instead of nix. default: 'ci' nix-cache: description: Determine whether to enable nix cache @@ -28,7 +25,7 @@ runs: steps: - name: Setup nix uses: ./.github/actions/setup-nix - if: ${{ inputs.use-nix == 'true' && (env.SHELL == '' || env.Nix_SHELL != inputs.nix-shell) }} + if: ${{ inputs.nix-shell != '' }} with: devShell: ${{ inputs.nix-shell }} verbose: ${{ inputs.nix-verbose }} @@ -36,7 +33,7 @@ runs: script: ${{ inputs.script }} - name: Set custom shell shell: bash - if: ${{ inputs.use-nix != 'true' && env.SHELL == '' }} + if: ${{ inputs.nix-shell == '' }} run: | echo SHELL="${{ inputs.custom_shell }}" >> $GITHUB_ENV diff --git a/.github/actions/setup-ubuntu/action.yml b/.github/actions/setup-ubuntu/action.yml new file mode 100644 index 000000000..a63025739 --- /dev/null +++ b/.github/actions/setup-ubuntu/action.yml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 + +name: Setup ubuntu +description: Setup ubuntu + +inputs: + packages: + description: Space-separated list of additional packages to install + required: false + default: '' + +runs: + using: composite + steps: + - name: Update package repository + shell: bash + run: | + sudo apt-get update + - name: Install base packages + shell: bash + run: | + sudo apt-get install python3-venv python3-pip make -y + - name: Install additional packages + if: ${{ inputs.packages != ''}} + shell: bash + run: | + sudo apt-get install ${{ inputs.packages }} -y + - name: Setup Python venv + shell: bash + run: | + python3 -m venv venv + source venv/bin/activate + python3 -m pip install -r requirements.txt + deactivate + echo "$(pwd)/venv/bin/" >> "$GITHUB_PATH" diff --git a/.github/workflows/bench_ec2_any.yml b/.github/workflows/bench_ec2_any.yml index 8c4c1e2fc..944414069 100644 --- a/.github/workflows/bench_ec2_any.yml +++ b/.github/workflows/bench_ec2_any.yml @@ -32,15 +32,14 @@ on: description: Run with optimized code if enabled type: boolean default: true - verbose: - description: Determine for the log verbosity - type: boolean - default: false bench_extra_args: description: Additional command line to be appended to `tests bench` script default: '' - cross_prefix: - description: "Binary prefix for cross-compilation builds" + compiler: + description: Compiler to use. When unset, default nix shell is used. + default: '' + additional_packages: + description: Additional packages to install when custom compiler is used. default: '' jobs: bench-ec2-any: @@ -55,7 +54,7 @@ jobs: opt: ${{ inputs.opt }} name: ${{ inputs.name }} store_results: false - verbose: ${{ inputs.verbose }} bench_extra_args: ${{ inputs.bench_extra_args }} - cross_prefix: ${{ inputs.cross_prefix }} + compiler: ${{ inputs.compiler }} + additional_packages: ${{ inputs.additional_packages }} secrets: inherit diff --git a/.github/workflows/bench_ec2_reusable.yml b/.github/workflows/bench_ec2_reusable.yml index 902d50644..9fbb57b97 100644 --- a/.github/workflows/bench_ec2_reusable.yml +++ b/.github/workflows/bench_ec2_reusable.yml @@ -44,9 +44,13 @@ on: type: string description: Additional command line to be appended to `bench` script default: '' - cross_prefix: + compiler: type: string - description: "Binary prefix for cross-compilation builds" + description: Compiler to use. When unset, default nix shell is used. + default: '' + additional_packages: + type: string + description: Additional packages to install when custom compiler is used. default: '' env: AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action @@ -98,10 +102,11 @@ jobs: ec2-instance-type: ${{ inputs.ec2_instance_type }} subnet-id: subnet-07b2729e5e065962f security-group-id: sg-0ab2e297196c8c381 - bench: - name: Bench ${{ inputs.name }} + bench_nix: + name: Bench (nix) runs-on: ${{ needs.start-ec2-runner.outputs.label }} needs: start-ec2-runner # required to start the main job when the runner is ready + if: ${{ inputs.compiler == '' }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/bench @@ -117,7 +122,33 @@ jobs: store_results: ${{ inputs.store_results }} bench_extra_args: ${{ inputs.bench_extra_args }} gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} - cross_prefix: ${{ inputs.cross_prefix }} + bench_custom: + name: Bench (custom compiler) + runs-on: ${{ needs.start-ec2-runner.outputs.label }} + needs: start-ec2-runner # required to start the main job when the runner is ready + if: ${{ inputs.compiler != '' }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-ubuntu + with: + packages: ${{ inputs.additional_packages }} + - name: Set compiler + run: | + echo "CC=${{ inputs.compiler }}" >> "$GITHUB_ENV" + - uses: ./.github/actions/bench + with: + nix-shell: '' + custom-shell: 'bash' + nix-cache: false + nix-verbose: ${{ inputs.verbose }} + name: ${{ inputs.name }} (${{ inputs.compiler }}) + cflags: ${{ inputs.cflags }} + archflags: ${{ inputs.archflags }} + opt: ${{ inputs.opt }} + perf: PERF + store_results: ${{ inputs.store_results }} + bench_extra_args: ${{ inputs.bench_extra_args }} + gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} stop-ec2-runner: name: Stop ${{ inputs.name }} (${{ inputs.ec2_instance_type }}) permissions: @@ -125,7 +156,8 @@ jobs: id-token: 'write' needs: - start-ec2-runner - - bench # required to wait when the main job is done + - bench_nix # required to wait when the main job is done + - bench_custom # required to wait when the main job is done runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: diff --git a/mk/config.mk b/mk/config.mk index 0644e5c0b..94b930144 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -33,6 +33,7 @@ CFLAGS += \ -Wshadow \ -Wpointer-arith \ -Wno-unknown-pragmas \ + -Wno-unused-command-line-argument \ -O3 \ -fomit-frame-pointer \ -std=c99 \