Skip to content

CI: Add Docker-based compatibility tests, add tests to make quickcheck #3459

CI: Add Docker-based compatibility tests, add tests to make quickcheck

CI: Add Docker-based compatibility tests, add tests to make quickcheck #3459

Workflow file for this run

# SPDX-License-Identifier: Apache-2.0
name: CI
permissions:
contents: read
on:
workflow_dispatch:
push:
branches: ["main"]
pull_request:
branches: ["main"]
types: [ "opened", "synchronize" ]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
strategy:
matrix:
system: [ubuntu-latest]
name: Linting
runs-on: ${{ matrix.system }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ./.github/actions/lint
with:
nix-shell: ci-linter
gh_token: ${{ secrets.GITHUB_TOKEN }}
cross-prefix: "aarch64-unknown-linux-gnu-"
lint-markdown-link:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: gaurav-nelson/github-action-markdown-link-check@d53a906aa6b22b8979d33bc86170567e619495ec # v1.0.15
quickcheck:
strategy:
fail-fast: false
matrix:
external:
- ${{ github.repository_owner != 'pq-code-package' }}
target:
- runner: pqcp-arm64
name: 'aarch64'
- runner: ubuntu-latest
name: 'x86_64'
exclude:
- {external: true,
target: {
runner: pqcp-arm64,
name: 'aarch64'
}}
name: Quickcheck (${{ matrix.target.name }})
runs-on: ${{ matrix.target.runner }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: make quickcheck
run: |
OPT=0 make quickcheck >/dev/null
make clean >/dev/null
OPT=1 make quickcheck >/dev/null
- uses: ./.github/actions/setup-ubuntu
- name: tests func
run: |
./scripts/tests func
- name: check namespacing
run: |
./scripts/ci/check-namespace
quickcheck-c90:
strategy:
fail-fast: false
matrix:
external:
- ${{ github.repository_owner != 'pq-code-package' }}
target:
- runner: pqcp-arm64
name: 'aarch64'
- runner: ubuntu-latest
name: 'x86_64'
exclude:
- {external: true,
target: {
runner: pqcp-arm64,
name: 'aarch64'
}}
name: Quickcheck C90 (${{ matrix.target.name }})
runs-on: ${{ matrix.target.runner }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: make quickcheck
run: |
OPT=0 CPPFLAGS=-std=c90 make quickcheck >/dev/null
make clean >/dev/null
OPT=1 CPPFLAGS=-std=c90 make quickcheck >/dev/null
- uses: ./.github/actions/setup-ubuntu
- name: tests func
run: |
CPPFLAGS="-std=c90" ./scripts/tests func
- name: check namespacing
run: |
./scripts/ci/check-namespace
quickcheck-windows:
name: Quickcheck windows-latest
runs-on: windows-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
- name: Build test
shell: powershell
run: |
# print compiler version
cl
nmake /f ./Makefile.Microsoft_nmake quickcheck
quickcheck-lib:
name: Quickcheck lib
strategy:
matrix:
system: [macos-latest, ubuntu-latest]
runs-on: ${{ matrix.system }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: make lib
run: |
make lib
examples:
name: Examples
strategy:
matrix:
system: [macos-latest, ubuntu-latest]
runs-on: ${{ matrix.system }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: mlkem_native_as_code_package
run: |
make run -C examples/mlkem_native_as_code_package
- name: bring_your_own_fips202
run: |
make run -C examples/bring_your_own_fips202
- name: custom_backend
run: |
make run -C examples/custom_backend
build_kat:
needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
strategy:
fail-fast: false
matrix:
external:
- ${{ github.repository_owner != 'pq-code-package' }}
target:
- runner: macos-latest
name: 'MacOS'
arch: mac
mode: native
- runner: pqcp-arm64
name: 'ubuntu-latest (aarch64)'
arch: aarch64
mode: native
- runner: pqcp-arm64
name: 'ubuntu-latest (aarch64)'
arch: x86_64
mode: cross-x86_64
- runner: pqcp-arm64
name: 'ubuntu-latest (aarch64)'
arch: riscv64
mode: cross-riscv64
- runner: pqcp-x64
name: 'ubuntu-latest (x86_64)'
arch: x86_64
mode: native
- runner: pqcp-x64
name: 'ubuntu-latest (x86_64)'
arch: aarch64
mode: cross-aarch64
- runner: pqcp-x64
name: 'ubuntu-latest (x86_64)'
arch: aarch64_be
mode: cross-aarch64_be
exclude:
- {external: true,
target: {
runner: pqcp-arm64,
name: 'ubuntu-latest (aarch64)',
arch: aarch64,
mode: native
}}
- {external: true,
target: {
runner: pqcp-arm64,
name: 'ubuntu-latest (aarch64)',
arch: x86_64,
mode: cross-x86_64
}}
- {external: true,
target: {
runner: pqcp-arm64,
name: 'ubuntu-latest (aarch64)',
arch: riscv64,
mode: cross-riscv64
}}
- {external: true,
target: {
runner: pqcp-x64,
name: 'ubuntu-latest (x86_64)',
arch: x86_64,
mode: native
}}
- {external: true,
target: {
runner: pqcp-x64,
name: 'ubuntu-latest (x86_64)',
arch: aarch64,
mode: cross-aarch64
}}
- {external: true,
target: {
runner: pqcp-x64,
name: 'ubuntu-latest (x86_64)',
arch: aarch64_be,
mode: cross-aarch64_be
}}
name: Functional tests (${{ matrix.target.arch }}${{ matrix.target.mode != 'native' && ', cross' || ''}})
runs-on: ${{ matrix.target.runner }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: build + test
uses: ./.github/actions/multi-functest
with:
nix-shell: ${{ matrix.target.mode == 'native' && 'ci' || 'ci-cross' }}
nix-cache: ${{ matrix.target.mode == 'native' && 'false' || 'true' }}
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: ${{ matrix.target.mode }}
# There is no native code on R-V or AArch64_be yet, so no point running opt tests
opt: ${{ (matrix.target.arch != 'riscv64' && matrix.target.arch != 'aarch64_be') && 'all' || 'no_opt' }}
- name: build + test (+debug+memsan+ubsan)
uses: ./.github/actions/multi-functest
if: ${{ matrix.target.mode == 'native' }}
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
cflags: "-DMLKEM_DEBUG -fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all"
compiler_tests:
name: Compiler tests (${{ matrix.target.name }})
needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
strategy:
fail-fast: false
matrix:
external:
- ${{ github.repository_owner != 'pq-code-package' }}
target:
- runner: pqcp-arm64
name: 'aarch64'
- runner: ubuntu-latest
name: 'x86_64'
- runner: macos-latest
name: 'macos'
exclude:
- {external: true,
target: {
runner: pqcp-arm64,
name: 'aarch64'
}}
runs-on: ${{ matrix.target.runner }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: native build+functest (gcc-4.8)
if: ${{ matrix.target.runner != 'macos-latest' }}
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
func: true
nistkat: false
kat: false
acvp: false
nix-shell: "ci_gcc48"
- name: native build+functest (gcc-4.9)
if: ${{ matrix.target.runner != 'macos-latest' }}
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
func: true
nistkat: false
kat: false
acvp: false
nix-shell: "ci_gcc49"
- name: native build+functest (gcc-7)
if: ${{ matrix.target.runner != 'macos-latest' }}
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
func: true
nistkat: false
kat: false
acvp: false
nix-shell: "ci_gcc7"
- name: native build+functest (gcc-11)
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
func: true
nistkat: false
kat: false
acvp: false
nix-shell: "ci_gcc11"
- name: native build+functest (gcc-14)
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
func: true
nistkat: false
kat: false
acvp: false
nix-shell: "ci_gcc14"
- name: native build+functest (clang-18)
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
func: true
nistkat: false
kat: false
acvp: false
nix-shell: "ci_clang18"
# The purpose of this job is to test non-default yet valid configurations
config_variations:
name: Non-standard configurations
needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
strategy:
fail-fast: false
matrix:
external:
- ${{ github.repository_owner != 'pq-code-package' }}
target:
- runner: pqcp-arm64
name: 'ubuntu-latest (aarch64)'
- runner: pqcp-x64
name: 'ubuntu-latest (x86_64)'
exclude:
- {external: true,
target: {
runner: pqcp-arm64,
name: 'ubuntu-latest (aarch64)',
}}
- {external: true,
target: {
runner: pqcp-x64,
name: 'ubuntu-latest (x86_64)',
}}
runs-on: ${{ matrix.target.runner }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: "MLKEM_GEN_MATRIX_NBLOCKS=1"
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
cflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -DMLKEM_GEN_MATRIX_NBLOCKS=1"
func: true
nistkat: true
kat: false
acvp: false
- name: "MLKEM_GEN_MATRIX_NBLOCKS=2"
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
cflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -DMLKEM_GEN_MATRIX_NBLOCKS=2"
func: true
nistkat: true
kat: false
acvp: false
- name: "MLKEM_GEN_MATRIX_NBLOCKS=4"
uses: ./.github/actions/multi-functest
with:
gh_token: ${{ secrets.GITHUB_TOKEN }}
compile_mode: native
cflags: "-fsanitize=address -fsanitize=undefined -fno-sanitize-recover=all -DMLKEM_GEN_MATRIX_NBLOCKS=4"
func: true
nistkat: true
kat: false
acvp: false
# ec2_functests:
# strategy:
# fail-fast: false
# matrix:
# target:
# - name: AMD EPYC 4th gen (t3a)
# ec2_instance_type: t3a.small
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-0d47e137a1108e078 # x86_64 ubuntu-latest, 32g
# compile_mode: native
# opt: all
# - name: Intel Xeon 4th gen (t3)
# ec2_instance_type: t3.small
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-0d47e137a1108e078 # x86_64 ubuntu-latest, 32g
# compile_mode: native
# opt: all
# - name: Graviton2 (c6g.medium)
# ec2_instance_type: c6g.medium
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g
# compile_mode: native
# opt: all
# - name: Graviton3 (c7g.medium)
# ec2_instance_type: c7g.medium
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g
# compile_mode: native
# opt: all
# name: Platform tests (${{ matrix.target.name }})
# permissions:
# contents: 'read'
# id-token: 'write'
# uses: ./.github/workflows/ci_ec2_reusable.yml
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
# with:
# name: ${{ matrix.target.name }}
# ec2_instance_type: ${{ matrix.target.ec2_instance_type }}
# ec2_ami: ${{ matrix.target.ec2_ami }}
# ec2_ami_id: ${{ matrix.target.ec2_ami_id }}
# compile_mode: ${{ matrix.target.compile_mode }}
# opt: ${{ matrix.target.opt }}
# functest: true
# kattest: true
# nistkattest: true
# acvptest: true
# lint: false
# verbose: true
# secrets: inherit
ec2_compatibilitytests:
strategy:
fail-fast: false
matrix:
container:
# - id: ubuntu-22.04-aarch:gcc-12x
# - id: ubuntu-22.04-aarch:gcc-11x
# - id: ubuntu-20.04-aarch:gcc-8x
# - id: ubuntu-20.04-aarch:gcc-7x
# - id: ubuntu-20.04-aarch:clang-9x
# - id: ubuntu-20.04-aarch:clang-8x
# - id: ubuntu-20.04-aarch:clang-7x-bm-framework
# - id: ubuntu-20.04-aarch:clang-7x
# - id: ubuntu-20.04-aarch:clang-10x
# - id: ubuntu-22.04-aarch:base
- id: ubuntu-20.04-aarch:base
- id: amazonlinux-2-aarch:base
- id: amazonlinux-2-aarch:gcc-7x
- id: amazonlinux-2-aarch:clang-7x
- id: amazonlinux-2023-aarch:base
- id: amazonlinux-2023-aarch:gcc-11x
- id: amazonlinux-2023-aarch:clang-15x
- id: amazonlinux-2023-aarch:clang-15x-sanitizer
- id: amazonlinux-2023-aarch:cryptofuzz
name: Compatibility tests (${{ matrix.container.id }})
# needs: [ec2_functests]
permissions:
contents: 'read'
id-token: 'write'
uses: ./.github/workflows/ci_ec2_reusable.yml
if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
with:
container: ${{ matrix.container.id }}
name: ${{ matrix.container.id }}
ec2_instance_type: c7g.medium
ec2_ami: ubuntu-latest (custom AMI)
ec2_ami_id: ami-0c9bc1901ef0d1066 # Has docker images preinstalled
compile_mode: native
opt: all
functest: true
kattest: true
nistkattest: true
acvptest: true
lint: false
verbose: true
secrets: inherit
# cbmc_k2:
# name: CBMC (ML-KEM-512)
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
# permissions:
# contents: 'read'
# id-token: 'write'
# uses: ./.github/workflows/ci_ec2_reusable.yml
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
# with:
# name: CBMC (MLKEM-512)
# ec2_instance_type: c7g.2xlarge
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g
# compile_mode: native
# opt: no_opt
# lint: false
# verbose: true
# functest: true
# kattest: false
# nistkattest: false
# acvptest: false
# cbmc: true
# cbmc_mlkem_k: 2
# secrets: inherit
# cbmc_k3:
# name: CBMC (ML-KEM-768)
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
# permissions:
# contents: 'read'
# id-token: 'write'
# uses: ./.github/workflows/ci_ec2_reusable.yml
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
# with:
# name: CBMC (MLKEM-768)
# ec2_instance_type: c7g.2xlarge
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g
# compile_mode: native
# opt: no_opt
# lint: false
# verbose: true
# functest: true
# kattest: false
# nistkattest: false
# acvptest: false
# cbmc: true
# cbmc_mlkem_k: 3
# secrets: inherit
# cbmc_k4:
# name: CBMC (ML-KEM-1024)
# needs: [quickcheck, quickcheck-windows, quickcheck-c90, quickcheck-lib, examples, lint, lint-markdown-link]
# permissions:
# contents: 'read'
# id-token: 'write'
# uses: ./.github/workflows/ci_ec2_reusable.yml
# if: github.repository_owner == 'pq-code-package' && !github.event.pull_request.head.repo.fork
# with:
# name: CBMC (MLKEM-1024)
# ec2_instance_type: c7g.2xlarge
# ec2_ami: ubuntu-latest (custom AMI)
# ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g
# compile_mode: native
# opt: no_opt
# lint: false
# verbose: true
# functest: true
# kattest: false
# nistkattest: false
# acvptest: false
# cbmc: true
# cbmc_mlkem_k: 4
# secrets: inherit