Merge pull request #563 from patchlevel/bench-assertion

Bench assertion
patchlevel · Apr 5, 2024 · e24daa1 · e24daa1
2 parents 44d2a2d + 368980e
commit e24daa1
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 9 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -67,16 +67,19 @@ jobs:
           dependency-versions: ${{ matrix.dependencies }}
 
       - name: "phpbench diff"
-        run: "vendor/bin/phpbench run tests/Benchmark --progress=none --report=diff --ref=base > bench.txt"
+        id: bench
+        run: 'vendor/bin/phpbench run tests/Benchmark --progress=none --report=diff --ref=base --assert="mode(variant.time.avg) <= mode(baseline.time.avg) +/- 5%"  --assert="mode(variant.mem.peak) <= mode(baseline.mem.peak) +/- 5%" > bench.txt'
 
       - name: "Get Bench Result"
+        if: ${{ success() || (failure() && steps.bench.conclusion == 'failure') }}
         id: phpbench
         run: |
           echo 'BENCH_RESULT<<EOF' >> $GITHUB_ENV
           cat bench.txt >> $GITHUB_ENV
           echo 'EOF' >> $GITHUB_ENV
 
       - uses: actions/github-script@v7
+        if: ${{ success() || (failure() && steps.bench.conclusion == 'failure') }}
         with:
           script: |
             // Get the existing comments.

diff --git a/Makefile b/Makefile
@@ -70,8 +70,8 @@ benchmark: vendor
 
 .PHONY: benchmark-diff-test
 benchmark-diff-test: vendor                                                   	## run benchmarks
-	vendor/bin/phpbench run tests/Benchmark --revs=1 --report=default --progress=none --tag=base
-	vendor/bin/phpbench run tests/Benchmark --revs=1 --report=diff --progress=none --ref=base
+	DB_URL=sqlite3:///:memory: vendor/bin/phpbench run tests/Benchmark --revs=1 --report=default --progress=none --tag=base
+	DB_URL=sqlite3:///:memory: vendor/bin/phpbench run tests/Benchmark --revs=1 --report=diff --progress=none --ref=base
 
 .PHONY: dev
 dev: static test                                                                ## run dev tools

diff --git a/phpbench.json b/phpbench.json
@@ -9,7 +9,7 @@
       "components": [
         {
           "component": "section",
-          "title": "{{ first(frame[\"benchmark_name\"]) }}",
+          "title": "{{ first(frame['benchmark_name']) }}",
           "components": [
             {
               "component": "table_aggregate",
@@ -18,23 +18,32 @@
               {
                 "time (kde mode)":
                 {
-                  "cols": ["time"]
+                  "cols": ["time", "time-diff"]
                 },
                 "memory":
                 {
-                  "cols": ["memory"]
+                  "cols": ["memory", "memory-diff"]
                 }
               },
               "row":
               {
-                "subject": "first(partition[\"subject_name\"]) ~ \" (\" ~ first(partition[\"variant_name\"]) ~ \")\"",
+                "subject": "first(partition['subject_name']) ~ ' (' ~ first(partition['variant_name']) ~ ')'",
                 "time":
                 {
                   "type": "expand",
                   "partition": "suite_tag",
                   "cols":
                   {
-                    "Tag: {{ key }}": "mode(partition[\"result_time_avg\"]) as time ~ ' (' ~ rstdev(partition['result_time_avg']) ~ ')'"
+                    "Tag: {{ key }}": "mode(partition['result_time_avg']) as time ~ ' (' ~ rstdev(partition['result_time_avg']) ~ ')'"
+                  }
+                },
+                "time-diff":
+                {
+                  "type": "expand",
+                  "partition": "subject_name",
+                  "cols":
+                  {
+                    "time-diff": "percent_diff(partition['result_time_avg'][1], partition['result_time_avg'][0])"
                   }
                 },
                 "memory":
@@ -43,7 +52,16 @@
                   "partition": "suite_tag",
                   "cols":
                   {
-                    "Tag: {{ key }} ": "mode(partition[\"result_mem_peak\"]) as memory"
+                    "Tag: {{ key }} ": "mode(partition['result_mem_peak']) as memory"
+                  }
+                },
+                "memory-diff":
+                {
+                  "type": "expand",
+                  "partition": "subject_name",
+                  "cols":
+                  {
+                    "memory-diff": "percent_diff(partition['result_mem_peak'][1], partition['result_mem_peak'][0])"
                   }
                 }
               }