CI: Add Docker-based compatibility tests, add tests to make quickcheck
#3473
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# SPDX-License-Identifier: Apache-2.0 | |
name: CI | |
permissions: | |
contents: read | |
on: | |
workflow_dispatch: | |
push: | |
branches: ["main"] | |
pull_request: | |
branches: ["main"] | |
types: [ "opened", "synchronize" ] | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
jobs: | |
lint: | |
strategy: | |
matrix: | |
system: [ubuntu-latest] | |
name: Linting | |
runs-on: ${{ matrix.system }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- uses: ./.github/actions/lint | |
with: | |
nix-shell: ci-linter | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
cross-prefix: "aarch64-unknown-linux-gnu-" | |
lint-markdown-link: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- uses: gaurav-nelson/github-action-markdown-link-check@d53a906aa6b22b8979d33bc86170567e619495ec # v1.0.15 | |
quickcheck: | |
strategy: | |
fail-fast: false | |
matrix: | |
external: | |
- ${{ github.repository_owner != 'pq-code-package' }} | |
target: | |
- runner: pqcp-arm64 | |
name: 'aarch64' | |
- runner: ubuntu-latest | |
name: 'x86_64' | |
exclude: | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'aarch64' | |
}} | |
name: Quickcheck (${{ matrix.target.name }}) | |
runs-on: ${{ matrix.target.runner }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: make quickcheck | |
run: | | |
OPT=0 make quickcheck >/dev/null | |
make clean >/dev/null | |
OPT=1 make quickcheck >/dev/null | |
- uses: ./.github/actions/setup-apt | |
- name: tests func | |
run: | | |
./scripts/tests func | |
- name: check namespacing | |
run: | | |
./scripts/ci/check-namespace | |
quickcheck-c90: | |
strategy: | |
fail-fast: false | |
matrix: | |
external: | |
- ${{ github.repository_owner != 'pq-code-package' }} | |
target: | |
- runner: pqcp-arm64 | |
name: 'aarch64' | |
- runner: ubuntu-latest | |
name: 'x86_64' | |
exclude: | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'aarch64' | |
}} | |
name: Quickcheck C90 (${{ matrix.target.name }}) | |
runs-on: ${{ matrix.target.runner }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: make quickcheck | |
run: | | |
OPT=0 CPPFLAGS=-std=c90 make quickcheck >/dev/null | |
make clean >/dev/null | |
OPT=1 CPPFLAGS=-std=c90 make quickcheck >/dev/null | |
- uses: ./.github/actions/setup-apt | |
- name: tests func | |
run: | | |
CPPFLAGS="-std=c90" ./scripts/tests func | |
- name: check namespacing | |
run: | | |
./scripts/ci/check-namespace | |
quickcheck-windows: | |
name: Quickcheck windows-latest | |
runs-on: windows-latest | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 | |
- name: Build test | |
shell: powershell | |
run: | | |
# print compiler version | |
cl | |
nmake /f ./Makefile.Microsoft_nmake quickcheck | |
quickcheck-lib: | |
name: Quickcheck lib | |
strategy: | |
matrix: | |
system: [macos-latest, ubuntu-latest] | |
runs-on: ${{ matrix.system }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: make lib | |
run: | | |
make lib | |
examples: | |
name: Examples | |
strategy: | |
matrix: | |
system: [macos-latest, ubuntu-latest] | |
runs-on: ${{ matrix.system }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: mlkem_native_as_code_package | |
run: | | |
make run -C examples/mlkem_native_as_code_package | |
- name: bring_your_own_fips202 | |
run: | | |
make run -C examples/bring_your_own_fips202 | |
- name: custom_backend | |
run: | | |
make run -C examples/custom_backend | |
build_kat: | |
needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
strategy: | |
fail-fast: false | |
matrix: | |
external: | |
- ${{ github.repository_owner != 'pq-code-package' }} | |
target: | |
- runner: macos-latest | |
name: 'MacOS' | |
arch: mac | |
mode: native | |
- runner: pqcp-arm64 | |
name: 'ubuntu-latest (aarch64)' | |
arch: aarch64 | |
mode: native | |
- runner: pqcp-arm64 | |
name: 'ubuntu-latest (aarch64)' | |
arch: x86_64 | |
mode: cross-x86_64 | |
- runner: pqcp-arm64 | |
name: 'ubuntu-latest (aarch64)' | |
arch: riscv64 | |
mode: cross-riscv64 | |
- runner: pqcp-x64 | |
name: 'ubuntu-latest (x86_64)' | |
arch: x86_64 | |
mode: native | |
- runner: pqcp-x64 | |
name: 'ubuntu-latest (x86_64)' | |
arch: aarch64 | |
mode: cross-aarch64 | |
- runner: pqcp-x64 | |
name: 'ubuntu-latest (x86_64)' | |
arch: aarch64_be | |
mode: cross-aarch64_be | |
exclude: | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'ubuntu-latest (aarch64)', | |
arch: aarch64, | |
mode: native | |
}} | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'ubuntu-latest (aarch64)', | |
arch: x86_64, | |
mode: cross-x86_64 | |
}} | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'ubuntu-latest (aarch64)', | |
arch: riscv64, | |
mode: cross-riscv64 | |
}} | |
- {external: true, | |
target: { | |
runner: pqcp-x64, | |
name: 'ubuntu-latest (x86_64)', | |
arch: x86_64, | |
mode: native | |
}} | |
- {external: true, | |
target: { | |
runner: pqcp-x64, | |
name: 'ubuntu-latest (x86_64)', | |
arch: aarch64, | |
mode: cross-aarch64 | |
}} | |
- {external: true, | |
target: { | |
runner: pqcp-x64, | |
name: 'ubuntu-latest (x86_64)', | |
arch: aarch64_be, | |
mode: cross-aarch64_be | |
}} | |
name: Functional tests (${{ matrix.target.arch }}${{ matrix.target.mode != 'native' && ', cross' || ''}}) | |
runs-on: ${{ matrix.target.runner }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: build + test | |
uses: ./.github/actions/multi-functest | |
with: | |
nix-shell: ${{ matrix.target.mode == 'native' && 'ci' || 'ci-cross' }} | |
nix-cache: ${{ matrix.target.mode == 'native' && 'false' || 'true' }} | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: ${{ matrix.target.mode }} | |
# There is no native code on R-V or AArch64_be yet, so no point running opt tests | |
opt: ${{ (matrix.target.arch != 'riscv64' && matrix.target.arch != 'aarch64_be') && 'all' || 'no_opt' }} | |
- name: build + test (+debug+memsan+ubsan) | |
uses: ./.github/actions/multi-functest | |
if: ${{ matrix.target.mode == 'native' }} | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
cflags: "-DMLKEM_DEBUG -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all" | |
compiler_tests: | |
name: Compiler tests (${{ matrix.target.name }}) | |
needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
strategy: | |
fail-fast: false | |
matrix: | |
external: | |
- ${{ github.repository_owner != 'pq-code-package' }} | |
target: | |
- runner: pqcp-arm64 | |
name: 'aarch64' | |
- runner: ubuntu-latest | |
name: 'x86_64' | |
- runner: macos-latest | |
name: 'macos' | |
exclude: | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'aarch64' | |
}} | |
runs-on: ${{ matrix.target.runner }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: native build+functest (gcc-4.8) | |
if: ${{ matrix.target.runner != 'macos-latest' }} | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
func: true | |
nistkat: false | |
kat: false | |
acvp: false | |
nix-shell: "ci_gcc48" | |
- name: native build+functest (gcc-4.9) | |
if: ${{ matrix.target.runner != 'macos-latest' }} | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
func: true | |
nistkat: false | |
kat: false | |
acvp: false | |
nix-shell: "ci_gcc49" | |
- name: native build+functest (gcc-7) | |
if: ${{ matrix.target.runner != 'macos-latest' }} | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
func: true | |
nistkat: false | |
kat: false | |
acvp: false | |
nix-shell: "ci_gcc7" | |
- name: native build+functest (gcc-11) | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
func: true | |
nistkat: false | |
kat: false | |
acvp: false | |
nix-shell: "ci_gcc11" | |
- name: native build+functest (gcc-14) | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
func: true | |
nistkat: false | |
kat: false | |
acvp: false | |
nix-shell: "ci_gcc14" | |
- name: native build+functest (clang-18) | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
func: true | |
nistkat: false | |
kat: false | |
acvp: false | |
nix-shell: "ci_clang18" | |
# The purpose of this job is to test non-default yet valid configurations | |
config_variations: | |
name: Non-standard configurations | |
needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
strategy: | |
fail-fast: false | |
matrix: | |
external: | |
- ${{ github.repository_owner != 'pq-code-package' }} | |
target: | |
- runner: pqcp-arm64 | |
name: 'ubuntu-latest (aarch64)' | |
- runner: pqcp-x64 | |
name: 'ubuntu-latest (x86_64)' | |
exclude: | |
- {external: true, | |
target: { | |
runner: pqcp-arm64, | |
name: 'ubuntu-latest (aarch64)', | |
}} | |
- {external: true, | |
target: { | |
runner: pqcp-x64, | |
name: 'ubuntu-latest (x86_64)', | |
}} | |
runs-on: ${{ matrix.target.runner }} | |
steps: | |
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
- name: "MLKEM_GEN_MATRIX_NBLOCKS=1" | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
cflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -DMLKEM_GEN_MATRIX_NBLOCKS=1" | |
func: true | |
nistkat: true | |
kat: false | |
acvp: false | |
- name: "MLKEM_GEN_MATRIX_NBLOCKS=2" | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
cflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -DMLKEM_GEN_MATRIX_NBLOCKS=2" | |
func: true | |
nistkat: true | |
kat: false | |
acvp: false | |
- name: "MLKEM_GEN_MATRIX_NBLOCKS=4" | |
uses: ./.github/actions/multi-functest | |
with: | |
gh_token: ${{ secrets.GITHUB_TOKEN }} | |
compile_mode: native | |
cflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -DMLKEM_GEN_MATRIX_NBLOCKS=4" | |
func: true | |
nistkat: true | |
kat: false | |
acvp: false | |
# ec2_functests: | |
# strategy: | |
# fail-fast: false | |
# matrix: | |
# target: | |
# - name: AMD EPYC 4th gen (t3a) | |
# ec2_instance_type: t3a.small | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-0d47e137a1108e078 # x86_64 ubuntu-latest, 32g | |
# compile_mode: native | |
# opt: all | |
# - name: Intel Xeon 4th gen (t3) | |
# ec2_instance_type: t3.small | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-0d47e137a1108e078 # x86_64 ubuntu-latest, 32g | |
# compile_mode: native | |
# opt: all | |
# - name: Graviton2 (c6g.medium) | |
# ec2_instance_type: c6g.medium | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g | |
# compile_mode: native | |
# opt: all | |
# - name: Graviton3 (c7g.medium) | |
# ec2_instance_type: c7g.medium | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g | |
# compile_mode: native | |
# opt: all | |
# name: Platform tests (${{ matrix.target.name }}) | |
# permissions: | |
# contents: 'read' | |
# id-token: 'write' | |
# uses: ./.github/workflows/ci_ec2_reusable.yml | |
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork | |
# with: | |
# name: ${{ matrix.target.name }} | |
# ec2_instance_type: ${{ matrix.target.ec2_instance_type }} | |
# ec2_ami: ${{ matrix.target.ec2_ami }} | |
# ec2_ami_id: ${{ matrix.target.ec2_ami_id }} | |
# compile_mode: ${{ matrix.target.compile_mode }} | |
# opt: ${{ matrix.target.opt }} | |
# functest: true | |
# kattest: true | |
# nistkattest: true | |
# acvptest: true | |
# lint: false | |
# verbose: true | |
# secrets: inherit | |
compatibility_tests: | |
strategy: | |
max-parallel: 4 | |
fail-fast: false | |
matrix: | |
container: | |
- id: fedora:38 | |
- id: fedora:39 | |
- id: fedora:40 | |
- id: debian:bullseye | |
- id: debian:bookworm | |
name: Compatibility tests (${{ matrix.container.id }}) | |
runs-on: ubuntu-latest | |
container: | |
${{ matrix.container.id }} | |
steps: | |
- name: Manual checkout | |
shell: bash | |
run: | | |
if which yum; then | |
yum install git -y | |
elif which apt; then | |
apt update | |
apt install git -y | |
fi | |
git init | |
git remote add origin $GITHUB_SERVER_URL/$GITHUB_REPOSITORY | |
git fetch origin --depth 1 $GITHUB_SHA | |
git checkout FETCH_HEAD | |
- uses: ./.github/actions/setup-os | |
with: | |
sudo: "" | |
- name: make quickcheck | |
run: | | |
OPT=0 make quickcheck >/dev/null | |
make clean >/dev/null | |
OPT=1 make quickcheck >/dev/null | |
- name: Functional Tests | |
uses: ./.github/actions/multi-functest | |
with: | |
nix-shell: "" | |
gh_token: ${{ secrets.AWS_GITHUB_TOKEN }} | |
cflags: ${{ inputs.cflags }} | |
compile_mode: ${{ inputs.compile_mode }} | |
opt: ${{ inputs.opt }} | |
func: ${{ inputs.functest }} | |
kat: ${{ inputs.kattest }} | |
nistkat: ${{ inputs.nistkattest }} | |
acvp: ${{ inputs.acvptest }} | |
ec2_compatibilitytests: | |
strategy: | |
max-parallel: 8 | |
fail-fast: false | |
matrix: | |
container: | |
- id: amazonlinux-2-aarch:base | |
- id: amazonlinux-2-aarch:gcc-7x | |
- id: amazonlinux-2-aarch:clang-7x | |
- id: amazonlinux-2023-aarch:base | |
- id: amazonlinux-2023-aarch:gcc-11x | |
- id: amazonlinux-2023-aarch:clang-15x | |
- id: amazonlinux-2023-aarch:clang-15x-sanitizer | |
# - id: amazonlinux-2023-aarch:cryptofuzz Not yet supported | |
- id: ubuntu-22.04-aarch:gcc-12x | |
- id: ubuntu-22.04-aarch:gcc-11x | |
- id: ubuntu-20.04-aarch:gcc-8x | |
- id: ubuntu-20.04-aarch:gcc-7x | |
- id: ubuntu-20.04-aarch:clang-9x | |
- id: ubuntu-20.04-aarch:clang-8x | |
- id: ubuntu-20.04-aarch:clang-7x-bm-framework | |
- id: ubuntu-20.04-aarch:clang-7x | |
- id: ubuntu-20.04-aarch:clang-10x | |
- id: ubuntu-22.04-aarch:base | |
- id: ubuntu-20.04-aarch:base | |
name: Compatibility tests (${{ matrix.container.id }}) | |
# needs: [ec2_functests] | |
permissions: | |
contents: 'read' | |
id-token: 'write' | |
uses: ./.github/workflows/ci_ec2_container.yml | |
if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork | |
with: | |
container: ${{ matrix.container.id }} | |
name: ${{ matrix.container.id }} | |
ec2_instance_type: c7g.medium | |
ec2_ami: ubuntu-latest (custom AMI) | |
ec2_ami_id: ami-0c9bc1901ef0d1066 # Has docker images preinstalled | |
compile_mode: native | |
opt: all | |
functest: true | |
kattest: true | |
nistkattest: true | |
acvptest: true | |
lint: false | |
verbose: true | |
secrets: inherit | |
# cbmc_k2: | |
# name: CBMC (ML-KEM-512) | |
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
# permissions: | |
# contents: 'read' | |
# id-token: 'write' | |
# uses: ./.github/workflows/ci_ec2_reusable.yml | |
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork | |
# with: | |
# name: CBMC (MLKEM-512) | |
# ec2_instance_type: c7g.2xlarge | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g | |
# compile_mode: native | |
# opt: no_opt | |
# lint: false | |
# verbose: true | |
# functest: true | |
# kattest: false | |
# nistkattest: false | |
# acvptest: false | |
# cbmc: true | |
# cbmc_mlkem_k: 2 | |
# secrets: inherit | |
# cbmc_k3: | |
# name: CBMC (ML-KEM-768) | |
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
# permissions: | |
# contents: 'read' | |
# id-token: 'write' | |
# uses: ./.github/workflows/ci_ec2_reusable.yml | |
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork | |
# with: | |
# name: CBMC (MLKEM-768) | |
# ec2_instance_type: c7g.2xlarge | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g | |
# compile_mode: native | |
# opt: no_opt | |
# lint: false | |
# verbose: true | |
# functest: true | |
# kattest: false | |
# nistkattest: false | |
# acvptest: false | |
# cbmc: true | |
# cbmc_mlkem_k: 3 | |
# secrets: inherit | |
# cbmc_k4: | |
# name: CBMC (ML-KEM-1024) | |
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link] | |
# permissions: | |
# contents: 'read' | |
# id-token: 'write' | |
# uses: ./.github/workflows/ci_ec2_reusable.yml | |
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork | |
# with: | |
# name: CBMC (MLKEM-1024) | |
# ec2_instance_type: c7g.2xlarge | |
# ec2_ami: ubuntu-latest (custom AMI) | |
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g | |
# compile_mode: native | |
# opt: no_opt | |
# lint: false | |
# verbose: true | |
# functest: true | |
# kattest: false | |
# nistkattest: false | |
# acvptest: false | |
# cbmc: true | |
# cbmc_mlkem_k: 4 | |
# secrets: inherit |