From 33ccd2a089d1a3d96e3a1168a0a9a242eb1316d9 Mon Sep 17 00:00:00 2001 From: "Matthias J. Kannwischer" Date: Wed, 26 Jun 2024 12:00:30 +0800 Subject: [PATCH 1/4] add github benchmark action Signed-off-by: Matthias J. Kannwischer --- .github/workflows/bench.yml | 13 ++++++++++++- scripts/tests | 30 +++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 02ed5017e..04adcbe35 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -10,6 +10,8 @@ on: jobs: bench: runs-on: self-hosted-rpi4 + permissions: + contents: write if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main') steps: - uses: actions/checkout@v4 @@ -34,4 +36,13 @@ jobs: - name: Run benchmark shell: nix develop .#ci -c bash -e {0} run: | - tests bench -c PMU --cflags -mcpu=cortex-a72 -v + tests bench -c PMU --cflags -mcpu=cortex-a72 -v --output output.json + - name: Store benchmark result + if: github.repository_owner == 'pq-code-package' && github.ref == 'refs/heads/main' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Arm Cortex-A72 (Raspberry Pi 4) benchmarks + tool: 'customSmallerIsBetter' + output-file-path: output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true diff --git a/scripts/tests b/scripts/tests index 23cee2837..46a8ae667 100755 --- a/scripts/tests +++ b/scripts/tests @@ -345,7 +345,13 @@ def kat(force_qemu, verbose, cflags, arch_flags): default="NO", help="Method for counting clock cycles. PMU requires (user-space) access to the Arm Performance Monitor Unit (PMU). PERF requires a kernel with perf support.", ) -def bench(force_qemu, verbose, cycles, cflags, arch_flags): +@click.option( + "-output", + "--output", + nargs=1, + help="Path to output file in json format", +) +def bench(force_qemu, verbose, cycles, cflags, arch_flags, output): config_logger(verbose) results = test_schemes( @@ -360,6 +366,28 @@ def bench(force_qemu, verbose, cycles, cflags, arch_flags): [f"CYCLES={cycles}"], ) + if output: + import json + + with open(output, "w") as f: + v = [] + for scheme in results: + schemeStr = str(scheme) + r = results[scheme] + d = { + k: int(v) + for k, v in (l.decode().split("=") for l in r.splitlines()[:3]) + } + for primitive in ["keypair", "encaps", "decaps"]: + v.append( + { + "name": f"{schemeStr} {primitive}", + "unit": "cycles", + "value": d[f"{primitive} cycles"], + } + ) + f.write(json.dumps(v)) + @click.group(invoke_without_command=True) def cli(): From f1eef28cf4f4f7942415eee407d1abba992fa51f Mon Sep 17 00:00:00 2001 From: "Matthias J. Kannwischer" Date: Wed, 26 Jun 2024 18:07:13 +0800 Subject: [PATCH 2/4] -output should be -o Signed-off-by: Matthias J. Kannwischer --- scripts/tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests b/scripts/tests index 46a8ae667..f1d86de38 100755 --- a/scripts/tests +++ b/scripts/tests @@ -346,7 +346,7 @@ def kat(force_qemu, verbose, cflags, arch_flags): help="Method for counting clock cycles. PMU requires (user-space) access to the Arm Performance Monitor Unit (PMU). PERF requires a kernel with perf support.", ) @click.option( - "-output", + "-o", "--output", nargs=1, help="Path to output file in json format", From 38f7423d15a6484ebeff22ccee643967d43e312f Mon Sep 17 00:00:00 2001 From: "Matthias J. Kannwischer" Date: Wed, 26 Jun 2024 19:35:06 +0800 Subject: [PATCH 3/4] add comment on output format Signed-off-by: Matthias J. Kannwischer --- scripts/tests | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/tests b/scripts/tests index f1d86de38..bbf8d0f0b 100755 --- a/scripts/tests +++ b/scripts/tests @@ -374,6 +374,12 @@ def bench(force_qemu, verbose, cycles, cflags, arch_flags, output): for scheme in results: schemeStr = str(scheme) r = results[scheme] + + # The first 3 lines of the output are expected to be + # keypair cycles=X + # encaps cycles=X + # decaps cycles=X + d = { k: int(v) for k, v in (l.decode().split("=") for l in r.splitlines()[:3]) From e9bbebd3c095cadbbb126825e77f05eea7ae9dbd Mon Sep 17 00:00:00 2001 From: "Matthias J. Kannwischer" Date: Wed, 26 Jun 2024 19:37:11 +0800 Subject: [PATCH 4/4] format Signed-off-by: Matthias J. Kannwischer --- scripts/tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests b/scripts/tests index bbf8d0f0b..b1365d9e1 100755 --- a/scripts/tests +++ b/scripts/tests @@ -375,7 +375,7 @@ def bench(force_qemu, verbose, cycles, cflags, arch_flags, output): schemeStr = str(scheme) r = results[scheme] - # The first 3 lines of the output are expected to be + # The first 3 lines of the output are expected to be # keypair cycles=X # encaps cycles=X # decaps cycles=X