Run opt/non-opt in CI (#125)

* run opt/non-opt bench in ci Signed-off-by: Thing-han, Lim <[email protected]> * run opt/non-opt in ci Signed-off-by: Thing-han, Lim <[email protected]> * make ci_ec2_any always lint Signed-off-by: Thing-han, Lim <[email protected]> * fix ci reusable ami id Signed-off-by: Thing-han, Lim <[email protected]> * wip fix store_results Signed-off-by: Thing-han, Lim <[email protected]> * remove inputs boolean check Signed-off-by: Thing-han, Lim <[email protected]> * fix if argument is empty string Signed-off-by: Thing-han, Lim <[email protected]> * update bench matrix jobs name Signed-off-by: Thing-han, Lim <[email protected]> * update ci opt/non-opt functest names Signed-off-by: Thing-han, Lim <[email protected]> --------- Signed-off-by: Thing-han, Lim <[email protected]>
pq-code-package · Sep 12, 2024 · ad617bf · ad617bf
1 parent 13b94d9
commit ad617bf
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 20 deletions.
diff --git a/.github/actions/bench/action.yml b/.github/actions/bench/action.yml
@@ -17,6 +17,9 @@ inputs:
   archflags:
     description: ARCHFLAGS to pass to compilation
     default: ""
+  opt:
+    description: opt flag to set for tests script
+    default: "true"
   bench_extra_args:
     description: Further arguments to be appended to command line for `bench` script
     default: ""
@@ -59,9 +62,9 @@ runs:
     - name: Run benchmark
       shell: ${{ env.SHELL }}
       run: |
-        tests bench -c ${{ inputs.perf }} --cflags "${{ inputs.cflags }}" --arch-flags "${{ inputs.archflags }}" -v --output output.json ${{ inputs.bench_extra_args }}
+        tests bench -c ${{ inputs.perf }} --cflags="${{ inputs.cflags }}" --arch-flags="${{ inputs.archflags }}" --opt="${{ inputs.opt }}" -v --output=output.json ${{ inputs.bench_extra_args }}
     - name: Store benchmark result
-      if: ${{ inputs.store_results == true }}
+      if: ${{ inputs.store_results }}
       uses: benchmark-action/github-action-benchmark@v1
       with:
         name: ${{ inputs.name }}

diff --git a/.github/actions/functest/action.yml b/.github/actions/functest/action.yml
@@ -13,6 +13,9 @@ inputs:
   cross-prefix:
     description: Binary prefix for cross compilation
     default: ""
+  opt:
+    description: opt flag to pass to test script
+    default: "true"
 runs:
   using: composite
   steps:
@@ -39,7 +42,7 @@ runs:
         shell: ${{ env.SHELL }}
         run: |
           echo "::group::func_test"
-          tests func --cross-prefix=${{ inputs.cross-prefix }} --cflags ${{ inputs.cflags }} -v
+          tests func --cross-prefix=${{ inputs.cross-prefix }} --cflags=${{ inputs.cflags }} --opt=${{inputs.opt}} -v
           echo "::endgroup::"
       - name: Run KAT tests
         if: |
@@ -49,7 +52,7 @@ runs:
         shell: ${{ env.SHELL }}
         run: |
           echo "::group::kat_test"
-          tests kat --cross-prefix=${{ inputs.cross-prefix }} --cflags ${{ inputs.cflags }} -v
+          tests kat --cross-prefix=${{ inputs.cross-prefix }} --cflags=${{ inputs.cflags }} --opt=${{inputs.opt}} -v
           echo "::endgroup::"
       - name: Run Nistkat tests
         id: nistkat_test
@@ -60,5 +63,5 @@ runs:
         shell: ${{ env.SHELL }}
         run: |
           echo "::group::nistkat_test"
-          tests nistkat --cross-prefix=${{ inputs.cross-prefix }} --cflags ${{ inputs.cflags }} -v
+          tests nistkat --cross-prefix=${{ inputs.cross-prefix }} --cflags=${{ inputs.cflags }} --opt=${{inputs.opt}} -v
           echo "::endgroup::"
diff --git a/.github/workflows/bench_ec2_all.yml b/.github/workflows/bench_ec2_all.yml
@@ -10,32 +10,50 @@ on:
     types: [ "labeled" ]
 jobs:
   bench-gv2:
-    name: Graviton2
+    name: Graviton2 ${{ matrix.opt.name }}
     permissions:
       contents: 'read'
       id-token: 'write'
+    strategy:
+      fail-fast: false
+      matrix:
+        opt:
+          - name: opt
+            value: true
+          - name: non-opt
+            value: false
     uses: ./.github/workflows/bench_ec2_reusable.yml
     if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
     with:
       ec2_instance_type: t4g.small
       ec2_ami: ubuntu-latest (aarch64)
       archflags: -mcpu=cortex-a76 -march=armv8.2-a
       cflags: -DFORCE_AARCH64
+      opt: ${{ matrix.opt.value }}
       store_results: true
       name: Graviton2
     secrets: inherit
   bench-gv3:
-    name: Graviton3
+    name: Graviton3 ${{ matrix.opt.name }}
     permissions:
       contents: 'read'
       id-token: 'write'
+    strategy:
+      fail-fast: false
+      matrix:
+        opt:
+          - name: opt
+            value: true
+          - name: non-opt
+            value: false
     uses: ./.github/workflows/bench_ec2_reusable.yml
     if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
     with:
       ec2_instance_type: c7g.medium
       ec2_ami: ubuntu-latest (aarch64)
       archflags: -mcpu=neoverse-v1 -march=armv8.4-a
       cflags: -DFORCE_AARCH64
+      opt: ${{ matrix.opt.value }}
       store_results: true
       name: Graviton3
     secrets: inherit
diff --git a/.github/workflows/bench_ec2_any.yml b/.github/workflows/bench_ec2_any.yml
@@ -28,6 +28,10 @@ on:
       archflags:
         description: Custom ARCH flags for compilation
         default: -mcpu=cortex-a76 -march=armv8.2-a
+      opt:
+        description: Run with optimized code if enabled
+        type: boolean
+        default: true
       store_results:
         description: Indicates if results should be pushed to github pages
         type: boolean
@@ -49,6 +53,7 @@ jobs:
       ec2_ami_id: ${{ inputs.ec2_ami_id }}
       cflags: ${{ inputs.cflags }}
       archflags: ${{ inputs.archflags }}
+      opt: ${{ inputs.opt }}
       name: ${{ inputs.name }}
       store_results: ${{ inputs.store_results }}
       always_terminate: ${{ inputs.always_terminate }}

diff --git a/.github/workflows/bench_ec2_reusable.yml b/.github/workflows/bench_ec2_reusable.yml
@@ -28,6 +28,10 @@ on:
         type: string
         description: Custom ARCH flags for compilation
         default: -mcpu=neoverse-n1 -march=armv8.2-a
+      opt:
+        type: boolean
+        description: Runs with optimized code if enabled.
+        default: true
       store_results:
         type: boolean
         description: Indicates if results should be pushed to github pages
@@ -102,6 +106,7 @@ jobs:
           name: ${{ inputs.name }}
           cflags: ${{ inputs.cflags }}
           archflags: ${{ inputs.archflags }}
+          opt: ${{ inputs.opt }}
           perf: PERF
           store_results: ${{ inputs.store_results }}
           bench_extra_args: ${{ inputs.bench_extra_args }}
@@ -115,7 +120,7 @@ jobs:
       - start-ec2-runner
       - bench # required to wait when the main job is done
     runs-on: ubuntu-latest
-    if: ${{ (inputs.always_terminate == true && always()) || success() }} # required to stop the runner even if the error happened in the previous jobs
+    if: ${{ (inputs.always_terminate && always()) || success() }} # required to stop the runner even if the error happened in the previous jobs
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -42,11 +42,20 @@ jobs:
     runs-on: ${{ matrix.target.runner }}
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/functest
+      - name: functest opt
+        uses: ./.github/actions/functest
         with:
           use-nix: true
           cflags: ${{ matrix.target.cflags }}
           cross-prefix: ${{ matrix.target.cross-prefix }}
+          opt: true
+      - name: functest non-opt
+        uses: ./.github/actions/functest
+        with:
+          use-nix: true
+          cflags: ${{ matrix.target.cflags }}
+          cross-prefix: ${{ matrix.target.cross-prefix }}
+          opt: false
   lint:
     strategy:
       matrix:

diff --git a/.github/workflows/ci_ec2_any.yml b/.github/workflows/ci_ec2_any.yml
@@ -24,6 +24,10 @@ on:
       cflags:
         description: Custom CFLAGS for compilation
         default:
+      opt:
+        description: Run with optimized code if enabled
+        type: boolean
+        default: true
       cross-prefix:
         description: Cross-compilation binary prefix, if any
         default: ' '
@@ -35,10 +39,6 @@ on:
         description: Whether to run functional tests
         type: boolean
         default: true
-      lint:
-        description: Whether to lint
-        type: boolean
-        default: true
       cbmc:
         description: Whether to run CBMC proofs
         type: boolean
@@ -53,8 +53,9 @@ jobs:
       ec2_ami_id: ${{ inputs.ec2_ami_id }}
       cflags: ${{ inputs.cflags }}
       cross-prefix: ${{ inputs.cross-prefix }}
+      opt: ${{ inputs.opt }}
       functest: ${{ inputs.functest }}
-      lint: ${{ inputs.lint }}
+      lint: true
       cbmc: ${{ inputs.cbmc }}
       always_terminate: ${{ inputs.always_terminate }}
     secrets: inherit
diff --git a/.github/workflows/ci_ec2_reusable.yml b/.github/workflows/ci_ec2_reusable.yml
@@ -28,6 +28,10 @@ on:
         type: string
         description: Cross-compilation binary prefix, if any
         default: ' '
+      opt:
+        type: boolean
+        description: Runs with optimized code if enabled.
+        default: true
       always_terminate:
         type: boolean
         description: Indicates if instance should always be terminated, even on failure
@@ -44,6 +48,8 @@ on:
 env:
   AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action
   AWS_REGION: us-east-1
+  AMI_UBUNTU_LATEST_X86_64: ami-0e86e20dae9224db8
+  AMI_UBUNTU_LATEST_AARCH64: ami-096ea6a12ea24a797
 jobs:
   start-ec2-runner:
     name: Start ${{ inputs.name }} (${{ inputs.ec2_instance_type }})
@@ -93,19 +99,20 @@ jobs:
     name: Functional tests ${{ inputs.name }}
     needs: start-ec2-runner
     runs-on: ${{ needs.start-ec2-runner.outputs.label }}
-    if: ${{ inputs.functest == true }}
+    if: ${{ inputs.functest }}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/functest
         with:
           use-nix: true
-          cflags: ${{ matrix.target.cflags }}
-          cross-prefix: ${{ matrix.target.cross-prefix }}
+          cflags: ${{ inputs.cflags }}
+          cross-prefix: ${{ inputs.cross-prefix }}
+          opt: ${{ inputs.opt }}
   lint:
     name: Linting ${{ inputs.name }}
     needs: start-ec2-runner
     runs-on: ${{ needs.start-ec2-runner.outputs.label }}
-    if: ${{ inputs.lint == true }}
+    if: ${{ inputs.lint }}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/lint
@@ -116,7 +123,7 @@ jobs:
     name: CBMC ${{ inputs.name }}
     needs: start-ec2-runner
     runs-on: ${{ needs.start-ec2-runner.outputs.label }}
-    if: ${{ inputs.cbmc == true }}
+    if: ${{ inputs.cbmc }}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/cbmc
@@ -134,7 +141,7 @@ jobs:
       - lint
       - functest
     runs-on: ubuntu-latest
-    if: ${{ (inputs.always_terminate == 'true' && always()) || success() }} # required to stop the runner even if the error happened in the previous jobs
+    if: ${{ (inputs.always_terminate && always()) || success() }} # required to stop the runner even if the error happened in the previous jobs
     steps:
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4