From 0f00d5e999b4497d16cf24d56d2f80514127ceab Mon Sep 17 00:00:00 2001
From: Hanno Becker <beckphan@amazon.co.uk>
Date: Sat, 21 Dec 2024 07:06:33 +0000
Subject: [PATCH] CI: Add Docker-based compatibility tests

This commit extends the CI to run compatibility tests for Docker images

- ubuntu-22.04:gcc-12x
- ubuntu-22.04:gcc-11x
- ubuntu-20.04:gcc-8x
- ubuntu-20.04:gcc-7x
- ubuntu-20.04:clang-9x
- ubuntu-20.04:clang-8x
- ubuntu-20.04:clang-7x-bm-framework
- ubuntu-20.04:clang-7x
- ubuntu-20.04:clang-10x
- ubuntu-22.04:base

Those images are pre-built on a custom EC2 AMI, and match the images
used in the AWS-LC CI.

Signed-off-by: Hanno Becker <beckphan@amazon.co.uk>
---
 .github/workflows/ci.yml              | 40 ++++++++++++++++++++++++++-
 .github/workflows/ci_ec2_reusable.yml | 37 ++++++++++++++++++++++++-
 2 files changed, 75 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8504864eb..ba8b172d7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -385,7 +385,6 @@ jobs:
           kat: false
           acvp: false
   ec2_functests:
-    needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
     strategy:
       fail-fast: false
       matrix:
@@ -419,6 +418,7 @@ jobs:
       contents: 'read'
       id-token: 'write'
     uses: ./.github/workflows/ci_ec2_reusable.yml
+    needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
     if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
     with:
       name: ${{ matrix.target.name }}
@@ -434,6 +434,44 @@ jobs:
       lint: false
       verbose: true
     secrets: inherit
+  ec2_compatibilitytests:
+    strategy:
+      fail-fast: false
+      matrix:
+        container:
+          - id: ubuntu-22.04:gcc-12x
+          - id: ubuntu-22.04:gcc-11x
+          - id: ubuntu-20.04:gcc-8x
+          - id: ubuntu-20.04:gcc-7x
+          - id: ubuntu-20.04:clang-9x
+          - id: ubuntu-20.04:clang-8x
+          - id: ubuntu-20.04:clang-7x-bm-framework
+          - id: ubuntu-20.04:clang-7x
+          - id: ubuntu-20.04:clang-10x
+          - id: ubuntu-22.04:base
+          - id: ubuntu-20.04:base
+    name: Compatibility tests (${{ matrix.container.id }})
+    needs: [ec2_functests]
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    uses: ./.github/workflows/ci_ec2_reusable.yml
+    if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
+    with:
+      container: ${{ matrix.container.id }}
+      name: ${{ matrix.container.id }}
+      ec2_instance_type: c7g.medium
+      ec2_ami: ubuntu-latest (custom AMI)
+      ec2_ami_id: ami-0f4b26c5372aa0525 # Has docker images preinstalled
+      compile_mode: native
+      opt: all
+      functest: true
+      kattest: true
+      nistkattest: true
+      acvptest: true
+      lint: false
+      verbose: true
+    secrets: inherit
   cbmc_k2:
     name: CBMC (ML-KEM-512)
     needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
diff --git a/.github/workflows/ci_ec2_reusable.yml b/.github/workflows/ci_ec2_reusable.yml
index 75ab68a2b..807ca2466 100644
--- a/.github/workflows/ci_ec2_reusable.yml
+++ b/.github/workflows/ci_ec2_reusable.yml
@@ -59,6 +59,9 @@ on:
       cbmc_mlkem_k:
         type: string
         default: 2
+      container:
+        type: string
+        default: ''
 env:
   AWS_ROLE: arn:aws:iam::559050233797:role/mlkem-c-aarch64-gh-action
   AWS_REGION: us-east-1
@@ -112,9 +115,40 @@ jobs:
           ec2-instance-type: ${{ inputs.ec2_instance_type }}
           subnet-id: subnet-07b2729e5e065962f
           security-group-id: sg-0ab2e297196c8c381
+  container_tests:
+    name: Run container tests
+    needs: start-ec2-runner
+    if: ${{ inputs.container != '' }}
+    runs-on: ${{ needs.start-ec2-runner.outputs.label }}
+    container:
+      localhost:5000/${{ inputs.container }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: ./.github/actions/setup-ubuntu
+        with:
+          sudo: ""
+      - name: make quickcheck
+        run: |
+          OPT=0 make quickcheck >/dev/null
+          make clean            >/dev/null
+          OPT=1 make quickcheck >/dev/null
+      - name: Functional Tests
+        uses: ./.github/actions/multi-functest
+        with:
+          nix-shell: ""
+          gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
+          cflags: ${{ inputs.cflags }}
+          compile_mode: ${{ inputs.compile_mode }}
+          opt: ${{ inputs.opt }}
+          func: ${{ inputs.functest }}
+          kat: ${{ inputs.kattest }}
+          nistkat: ${{ inputs.nistkattest }}
+          acvp: ${{ inputs.acvptest }}
+
   tests:
-    name: Run test
+    name: Run tests
     needs: start-ec2-runner
+    if: ${{ inputs.container == '' }}
     runs-on: ${{ needs.start-ec2-runner.outputs.label }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -160,6 +194,7 @@ jobs:
     needs:
       - start-ec2-runner
       - tests
+      - container_tests
     runs-on: ubuntu-latest
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps: