Add initial Haskell support

.github/workflows/cabal.yml

@@ -0,0 +1,48 @@
+name: Cabal Example
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Run Cabal benchmark example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: haskell/actions/[email protected]
+      - name: Run benchmark
+        run: cd examples/cabal && cabal run -- --csv output.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Cabal Benchmark
+          tool: 'cabal'
+          output-file-path: examples/cabal/output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@charrsky'
+
+      - name: Store benchmark result - separate results repo
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Cabal Benchmark
+          tool: 'cabal'
+          output-file-path: examples/cabal/output.txt
+          github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@charrsky'
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

.github/workflows/ci.yml

@@ -293,6 +293,39 @@ jobs:
           fail-on-alert: true
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark'
 
+  cabal:
+    name: Run Cabal benchmark example
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - uses: haskell/actions/setup@v2
+      - uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+      - run: npm ci
+      - run: npm run build
+      - name: Save previous data.js
+        run: |
+          git fetch origin gh-pages
+          git checkout gh-pages
+          cp ./dev/bench/data.js before_data.js
+          git checkout -
+      - name: Run benchmark
+        run: cd examples/cabal && cabal run -- --csv output.txt
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Cabal Benchmark
+          tool: 'cabal'
+          output-file-path: examples/cabal/output.txt
+          skip-fetch-gh-pages: true
+          fail-on-alert: true
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark'
+
   only-alert-with-cache:
     name: Run alert check with actions/cache
     runs-on: ubuntu-latest

README.md

@@ -25,6 +25,7 @@ This action currently supports the following tools:
 - [BenchmarkTools.jl][] for Julia packages
 - [Benchmark.Net][benchmarkdotnet] for .Net projects
 - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects
+- [criterion](https://hackage.haskell.org/package/criterion) for Haskell projects
 - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter'
 
 Multiple languages in the same repository are supported for polyglot projects.
@@ -46,9 +47,11 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor
 | Python       | [![pytest-benchmark Example Workflow][pytest-benchmark-badge]][pytest-workflow-example] | [examples/pytest](./examples/pytest)           |
 | C++          | [![C++ Example Workflow][cpp-badge]][cpp-workflow-example]                              | [examples/cpp](./examples/cpp)                 |
 | C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example]                 | [examples/catch2](./examples/catch2)           |
-| Julia | [![Julia Example][julia-badge]][julia-workflow-example]                 | [examples/julia](./examples/julia)           |
+| Julia        | [![Julia Example][julia-badge]][julia-workflow-example]                 | [examples/julia](./examples/julia)           |
 | .Net         | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) |
 | Luau         | Coming soon | Coming soon |
+| Haskell      |             |             |
+
 
 All benchmark charts from above workflows are gathered in GitHub pages:
 

examples/cabal/Fibber.hs

@@ -0,0 +1,20 @@
+-- The simplest/silliest of all benchmarks!
+
+import Criterion.Main
+
+fib :: Integer -> Integer
+fib m | m < 0     = error "negative!"
+      | otherwise = go m
+  where
+    go 0 = 0
+    go 1 = 1
+    go n = go (n-1) + go (n-2)
+
+main :: IO ()
+main = defaultMain [
+  bgroup "fib" [ bench "1"  $ whnf fib 1
+               , bench "5"  $ whnf fib 5
+               , bench "9"  $ whnf fib 9
+               , bench "11" $ whnf fib 11
+               ]
+  ]

examples/cabal/README.md

@@ -0,0 +1,43 @@
+Cabal example for benchmarking with `criterion`
+=================================================
+
+- [Workflow for this example](../../.github/workflows/cabal.yml)
+- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/)
+- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/)
+
+This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark)
+with `criterion` package.
+
+You can see a quick `criterion` tutorial [here](http://www.serpentine.com/criterion/tutorial.html).
+
+NB: As for now, this GitHub action only supports benchmarks made with `criterion` package outputted as a .csv file.
+
+## Run benchmarks
+
+If the file with the benchmarks is also a "benchmark" according to your .cabal file, then
+
+```yaml
+- name: Run benchmark
+  run: cabal bench --benchmark-options="--csv <FILENAME>"
+```
+
+should do the trick. If it is an "executable" (as it is in the example), then run it with
+
+```yaml
+- name: Run benchmark
+  run: cabal run -- --csv <FILENAME>"
+```
+
+## Process benchmark results
+
+Store the benchmark results with step using the action.
+
+```yaml
+- name: Store benchmark result
+  uses: benchmark-action/github-action-benchmark@v1
+  with:
+    tool: 'cabal'
+    output-file-path: <FILENAME>
+```
+
+Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage.

examples/cabal/fibber.cabal

@@ -0,0 +1,40 @@
+name:          fibber
+version:       0
+synopsis:      Examples for the criterion benchmarking system
+description:   Examples for the criterion benchmarking system.
+homepage:      https://github.com/haskell/criterion
+license:       BSD3
+license-file:  LICENSE
+author:        Bryan O'Sullivan <[email protected]>
+maintainer:    Bryan O'Sullivan <[email protected]>
+category:      Benchmarks
+build-type:    Simple
+cabal-version: >=1.10
+tested-with:
+  GHC==7.4.2,
+  GHC==7.6.3,
+  GHC==7.8.4,
+  GHC==7.10.3,
+  GHC==8.0.2,
+  GHC==8.2.2,
+  GHC==8.4.4,
+  GHC==8.6.5,
+  GHC==8.8.4,
+  GHC==8.10.7,
+  GHC==9.0.2,
+  GHC==9.2.2
+
+flag conduit-vs-pipes
+  default: True
+
+flag maps
+  default: True
+
+executable fibber
+  main-is: Fibber.hs
+
+  default-language: Haskell2010
+  ghc-options: -Wall -rtsopts
+  build-depends:
+    base == 4.*,
+    criterion

src/config.ts

@@ -35,6 +35,7 @@ export const VALID_TOOLS = [
     'catch2',
     'julia',
     'benchmarkdotnet',
+    'cabal',
     'customBiggerIsBetter',
     'customSmallerIsBetter',
 ] as const;

src/default_index_html.ts

@@ -122,6 +122,7 @@ export const DEFAULT_INDEX_HTML = String.raw`<!DOCTYPE html>
           benchmarkdotnet: '#178600',
           customBiggerIsBetter: '#38ff38',
           customSmallerIsBetter: '#ff3838',
+          cabal: '#5e5086'
           _: '#333333'
         };
 

src/extract.ts

@@ -614,6 +614,32 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] {
     return results;
 }
 
+function extractCabalBenchmarkResult(output: string): BenchmarkResult[] {
+    const lines0 = output.split(/\n/);
+    const lines = lines0.splice(0, 1);
+    const res = [];
+
+    for (const line of lines) {
+        const [name, meanSec, meanLBSec, meanUBSec, stddevSec, stddevLBSec, stddevUBSec] = line.split(',');
+        const [mean, meanUnit] = getHumanReadableUnitValue(parseFloat(meanSec));
+        const [meanLB, meanLBUnit] = getHumanReadableUnitValue(parseFloat(meanLBSec));
+        const [meanUB, meanUBUnit] = getHumanReadableUnitValue(parseFloat(meanUBSec));
+        const [stddev, stddevUnit] = getHumanReadableUnitValue(parseFloat(stddevSec));
+        const [stddevLB, stddevLBUnit] = getHumanReadableUnitValue(parseFloat(stddevLBSec));
+        const [stddevUB, stddevUBUnit] = getHumanReadableUnitValue(parseFloat(stddevUBSec));
+
+        res.push({
+            name: name,
+            value: mean,
+            unit: meanUnit,
+            range: `±${stddev} ${stddevUnit}`,
+            extra: `Mean lower bound: ${meanLB} ${meanLBUnit}\nMean upper bound: ${meanUB} ${meanUBUnit}\nStandard deviation LB: ${stddevLB} ${stddevLBUnit}\nStandard deviation UB: ${stddevUB} ${stddevUBUnit}`,
+        });
+    }
+
+    return res;
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool, githubToken } = config;
@@ -644,6 +670,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'benchmarkdotnet':
             benches = extractBenchmarkDotnetResult(output);
             break;
+        case 'cabal':
+            benches = extractCabalBenchmarkResult(output);
+            break;
         case 'customBiggerIsBetter':
             benches = extractCustomBenchmarkResult(output);
             break;

src/write.ts

@@ -77,6 +77,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return false;
         case 'benchmarkdotnet':
             return false;
+        case 'cabal':
+            return false;
         case 'customBiggerIsBetter':
             return true;
         case 'customSmallerIsBetter':

test/data/extract/cabal_output.txt

@@ -0,0 +1,3 @@
+Name,Mean,MeanLB,MeanUB,Stddev,StddevLB,StddevUB
+fib/1,1.2128410840686445e-8,1.2074536540005714e-8,1.2202216975790019e-8,2.145268159803889e-10,1.6198043933498568e-10,2.7812295435222917e-10
+fib/5,2.3776579995877364e-7,2.343297169016553e-7,2.4262841729253593e-7,1.3331560091086646e-8,9.007202760821372e-9,1.9969513691856127e-8

test/extract.spec.ts

@@ -417,6 +417,25 @@ describe('extractResult()', function () {
                 },
             ],
         },
+        {
+            tool: 'cabal',
+            expected: [
+                {
+                    name: 'fib/1',
+                    unit: 'nsec',
+                    value: 12.128410840686445,
+                    range: '±0.2145268159803889 nsec',
+                    extra: 'Mean lower bound: 12.074536540005713 nsec\nMean upper bound: 12.20221697579002 nsec\nStandard deviation LB: 0.16198043933498568 nsec\nStandard deviation UB: 0.27812295435222917 nsec',
+                },
+                {
+                    name: 'fib/5',
+                    unit: 'nsec',
+                    value: 237.76579995877364,
+                    range: '±13.331560091086645 nsec',
+                    extra: 'Mean lower bound: 234.3297169016553 nsec\nMean upper bound: 242.62841729253594 nsec\nStandard deviation LB: 9.007202760821372 nsec\nStandard deviation UB: 19.96951369185613 nsec',
+                },
+            ],
+        },
     ];
 
     for (const test of normalCases) {