diff --git a/.github/workflows/compare-benchmarks.yaml b/.github/workflows/compare-benchmarks.yaml new file mode 100644 index 000000000..976820efa --- /dev/null +++ b/.github/workflows/compare-benchmarks.yaml @@ -0,0 +1,27 @@ +--- +name: Benchmarks on AMD64 +permissions: read-all +on: [push, pull_request] +jobs: + ci-benchmark-tests: + name: ci-benchmark-tests + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + + - name: Set up Go 1.x + uses: actions/setup-go@v5 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + id: go + + - name: Install Benchstat + run: | + make install-benchstat + + - name: Benchmark tests + run: | + make test-benchmark-compare diff --git a/Makefile b/Makefile index b0d019802..aefb537c0 100644 --- a/Makefile +++ b/Makefile @@ -94,3 +94,14 @@ test-failpoint: test-robustness: gofail-enable build sudo env PATH=$$PATH go test -v ${TESTFLAGS} ./tests/dmflakey -test.root sudo env PATH=$(PWD)/bin:$$PATH go test -v ${TESTFLAGS} ${ROBUSTNESS_TESTFLAGS} ./tests/robustness -test.root + +.PHONY: test-benchmark-compare +# Runs benchmark tests on the current git ref and the last release and compares +# the two. +test-benchmark-compare: + @git fetch + ./tests/compare_benchmarks.sh main + +.PHONY: install-benchstat +install-benchstat: + go install golang.org/x/perf/cmd/benchstat@latest diff --git a/tests/compare_benchmarks.sh b/tests/compare_benchmarks.sh new file mode 100755 index 000000000..e2b343024 --- /dev/null +++ b/tests/compare_benchmarks.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# https://github.com/kubernetes/kube-state-metrics/blob/main/tests/compare_benchmarks.sh (originally written by mxinden) + +# exit immediately when a command fails +set -e +# only exit with zero if all commands of the pipeline exit successfully +set -o pipefail +# error on unset variables +set -u + +[[ "$#" -eq 1 ]] || echo "One argument required, $# provided." + +REF_CURRENT="$(git rev-parse --abbrev-ref HEAD)" +REF_TO_COMPARE=$1 + +RESULT_CURRENT="$(mktemp)-${REF_CURRENT}" +RESULT_TO_COMPARE="$(mktemp)-${REF_TO_COMPARE}" + +echo "" +echo "### Testing ${REF_CURRENT}" + +go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_CURRENT}" + +echo "" +echo "### Done testing ${REF_CURRENT}" + +echo "" +echo "### Testing ${REF_TO_COMPARE}" + +git checkout "${REF_TO_COMPARE}" + +go test -benchmem -run=NONE -bench=. ./... | tee "${RESULT_TO_COMPARE}" + +echo "" +echo "### Done testing ${REF_TO_COMPARE}" + +git checkout - + +echo "" +echo "### Result" +echo "old=${REF_TO_COMPARE} new=${REF_CURRENT}" + +benchstat "${RESULT_TO_COMPARE}" "${RESULT_CURRENT}"