Moved benchmarking project go Github Actions

.github/workflows/benchmark-test.yml

@@ -0,0 +1,84 @@
+name: benchmark-windows
+on:
+  # To enable running it manually, possibility of adding parameters. It can be run manually only if this file is in master.
+  workflow_dispatch:
+  push:
+    branches: 
+      - master
+env:
+  # Branch for Github Pages
+  gh-pages-branch: gh-pages
+jobs:
+  build-wrappers:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Check if cache contains wrappers already
+        id: wrapper-cache
+        uses: actions/cache@v2
+        with:
+          path: .\wrappers\build\Windows\Release-x64\
+          key: wrapper-windows-release-64-${{hashfiles('.\wrappers\**')}}
+      - name: Get CMake
+        # If there is a cache hit, no need to do any of the following steps. Unfortunately there is no way to cancel the rest of the job.
+        if: steps.wrapper-cache.outputs.cache-hit != 'true'
+        uses: lukka/get-cmake@latest
+      - name: Restore from cache and run vcpkg
+        if: steps.wrapper-cache.outputs.cache-hit != 'true'
+        uses: lukka/run-vcpkg@v5
+        with:
+          # Contains the list of packages and the triplet. If we need more than one triplet, we can't use the file it seems
+          vcpkgArguments: '@${{ env.vcpkgResponseFile }}'
+          vcpkgGitCommitId: ec6fe06e8da05a8157dc8581fa96b36b571c1bd5
+          # If the cache key is the same, it does not execute
+          appendedCacheKey: ${{ hashFiles(env.vcpkgResponseFile) }}
+        env:
+          vcpkgResponseFile: '${{ github.workspace }}\.github\workflows\response_file.txt'
+      - name: Build wrappers Release
+        if: steps.wrapper-cache.outputs.cache-hit != 'true'
+        run: .\wrappers\build.ps1 Windows -Configuration Release -Platforms x64 -Toolchain "${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake"
+  execute-benchmarks:
+    needs: [build-wrappers]
+    runs-on: windows-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+      - name: Download previously built wrappers
+        id: wrapper-cache
+        uses: actions/cache@v2
+        with:
+          path: .\wrappers\build\Windows\Release-x64\
+          # This needs to be changed to something meaningful
+          key: wrapper-windows-release-64-${{hashfiles('.\wrappers\**')}}
+      - name: Check cache hit
+        # If there is no cache hit, just fail
+        if: steps.wrapper-cache.outputs.cache-hit != 'true'
+        run: exit 1
+      - name: Run benchmarks
+        run:  |
+          dotnet run -p .\Tests\PerformanceTests\ run -c Release --framework net5.0 -- -f * --join
+          rename-item -path ".\BenchmarkDotNet.Artifacts\results\*.json" -newname "benchmark-output.json"
+      - name: Save artifacts
+        # Not mandatory, but maybe it's good to do it?
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmark-results
+          # Could be put in an environmental variable
+          path: .\BenchmarkDotNet.Artifacts\results
+      # Download previous benchmark result from cache (if exists)    
+      - name: Store benchmark results
+        uses: Happypig375/[email protected]
+        with:
+          tool: 'benchmarkdotnet'
+          # Where the output from the benchmark tool is stored
+          output-file-path: .\BenchmarkDotNet.Artifacts\results\benchmark-output.json
+          # Workflow will not fail when an alert happens
+          fail-on-alert: false
+          gh-pages-branch: ${{env.gh-pages-branch}}
+          auto-push: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}      

.github/workflows/response_file.txt

@@ -0,0 +1,4 @@
+zlib
+openssl
+--triplet
+x64-windows-static

Jenkinsfile

@@ -334,21 +334,6 @@ stage('Test') {
           reportTests 'TestResults.Weaver.xml'
         }
       }
-    },
-    'Benchmarks': {
-      rlmNode('dotnet && windows') {
-        unstash 'dotnet-source'
-        dir('Realm/packages') { unstash 'packages' }
-
-        dir('Tests/PerformanceTests') {
-          bat """
-            dotnet build -c ${configuration} -f net5.0 -p:RestoreConfigFile=${env.WORKSPACE}/Tests/Test.NuGet.Config -p:UseRealmNupkgsWithVersion=${packageVersion}
-            dotnet run -c ${configuration} -f net5.0 --no-build -- -f *
-          """.trim()
-
-          archiveArtifacts "BenchmarkDotNet.Artifacts/results/*.md"
-        }
-      }
     }
   ]
 
@@ -487,4 +472,4 @@ List<List<?>> mapToList(Map map) {
 @NonCPS
 String getVersion(String name) {
   return (name =~ /Realm.Fody.(.+).nupkg/)[0][1]
-}
+}

Tests/PerformanceTests/Program.cs

@@ -16,21 +16,15 @@
 //
 ////////////////////////////////////////////////////////////////////////////
 
-using System;
 using System.Linq;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
-using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Exporters;
 using BenchmarkDotNet.Exporters.Json;
 using BenchmarkDotNet.Jobs;
-using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Order;
-using BenchmarkDotNet.Portability.Cpu;
-using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.InProcess.Emit;
-using Perfolizer.Horology;
 
 namespace PerformanceTests
 {
@@ -50,32 +44,9 @@ public static void Main(string[] args)
                 .AddDiagnoser(MemoryDiagnoser.Default)
                 .AddJob(Job.Default.WithToolchain(InProcessEmitToolchain.Instance))
                 .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method, MethodOrderPolicy.Alphabetical))
-                .AddExporter(new JenkinsHtmlExporter(), MarkdownExporter.GitHub, JsonExporter.Full);
+                .AddExporter(MarkdownExporter.GitHub, JsonExporter.FullCompressed);
 
             BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config);
         }
-
-        // This is needed because the Jenkins user does not have permissions to use System.Management.ManagementObjectSearcher.
-        // There's no public API to prevent BenchmarkDotNet from looking it up, so when building the summary, we get an access
-        // denied exception. This uses reflection to replace the HostEnvironmentInfo.CpuInfo lazy with one that returns a mock
-        // CpuInfo. Since this is the first exporter being run, it will modify the summary for the rest, so we don't need to
-        // fiddle with them. Once we move to Github actions, this can be removed.
-        private class JenkinsHtmlExporter : HtmlExporter
-        {
-            private CpuInfo _mockCpuInfo = new CpuInfo("MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz",
-                                              physicalProcessorCount: 1,
-                                              physicalCoreCount: 4,
-                                              logicalCoreCount: 8,
-                                              nominalFrequency: Frequency.FromMHz(3100),
-                                              maxFrequency: Frequency.FromMHz(3100),
-                                              minFrequency: Frequency.FromMHz(3100));
-
-            public override void ExportToLog(Summary summary, ILogger logger)
-            {
-                var cpuInfoProp = typeof(HostEnvironmentInfo).GetProperty(nameof(HostEnvironmentInfo.CpuInfo));
-                cpuInfoProp.SetValue(summary.HostEnvironmentInfo, new Lazy<CpuInfo>(() => _mockCpuInfo));
-                base.ExportToLog(summary, logger);
-            }
-        }
     }
 }

wrappers/build.ps1

@@ -22,6 +22,8 @@ param(
     [Parameter(Position=0)]
     [string]$Target = 'Windows',
 
+    [string]$Toolchain = 'c:\\src\\vcpkg\\scripts\\buildsystems\\vcpkg.cmake',
+
     [Switch]$Incremental
 )
 
@@ -40,7 +42,7 @@ $vs = Get-VSSetupInstance | Select-VSSetupInstance -Product * -Latest -Require M
 $Env:path += ";$($vs.InstallationPath)\MSBuild\Current\Bin"
 
 $cmake = Join-Path $vs.InstallationPath -ChildPath "Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe"
-$cmakeArgs = "-DCMAKE_GENERATOR_INSTANCE=$($vs.InstallationPath)", "-DCMAKE_BUILD_TYPE=$Configuration", "-DCMAKE_SYSTEM_NAME=$Target", "-DCMAKE_INSTALL_PREFIX=$PSScriptRoot\build", "-DCMAKE_TOOLCHAIN_FILE=c:\\src\\vcpkg\\scripts\\buildsystems\\vcpkg.cmake"
+$cmakeArgs = "-DCMAKE_GENERATOR_INSTANCE=$($vs.InstallationPath)", "-DCMAKE_BUILD_TYPE=$Configuration", "-DCMAKE_SYSTEM_NAME=$Target", "-DCMAKE_INSTALL_PREFIX=$PSScriptRoot\build", "-DCMAKE_TOOLCHAIN_FILE=$Toolchain"
 
 if ($Target -eq 'WindowsStore') {
     $cmakeArgs += "-DCMAKE_SYSTEM_VERSION='10.0'"