diff --git a/.github/workflows/benchmark-test.yml b/.github/workflows/benchmark-test.yml new file mode 100644 index 0000000000..20cfd03125 --- /dev/null +++ b/.github/workflows/benchmark-test.yml @@ -0,0 +1,84 @@ +name: benchmark-windows +on: + # To enable running it manually, possibility of adding parameters. It can be run manually only if this file is in master. + workflow_dispatch: + push: + branches: + - master +env: + # Branch for Github Pages + gh-pages-branch: gh-pages +jobs: + build-wrappers: + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Check if cache contains wrappers already + id: wrapper-cache + uses: actions/cache@v2 + with: + path: .\wrappers\build\Windows\Release-x64\ + key: wrapper-windows-release-64-${{hashfiles('.\wrappers\**')}} + - name: Get CMake + # If there is a cache hit, no need to do any of the following steps. Unfortunately there is no way to cancel the rest of the job. + if: steps.wrapper-cache.outputs.cache-hit != 'true' + uses: lukka/get-cmake@latest + - name: Restore from cache and run vcpkg + if: steps.wrapper-cache.outputs.cache-hit != 'true' + uses: lukka/run-vcpkg@v5 + with: + # Contains the list of packages and the triplet. If we need more than one triplet, we can't use the file it seems + vcpkgArguments: '@${{ env.vcpkgResponseFile }}' + vcpkgGitCommitId: ec6fe06e8da05a8157dc8581fa96b36b571c1bd5 + # If the cache key is the same, it does not execute + appendedCacheKey: ${{ hashFiles(env.vcpkgResponseFile) }} + env: + vcpkgResponseFile: '${{ github.workspace }}\.github\workflows\response_file.txt' + - name: Build wrappers Release + if: steps.wrapper-cache.outputs.cache-hit != 'true' + run: .\wrappers\build.ps1 Windows -Configuration Release -Platforms x64 -Toolchain "${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake" + execute-benchmarks: + needs: [build-wrappers] + runs-on: windows-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Download previously built wrappers + id: wrapper-cache + uses: actions/cache@v2 + with: + path: .\wrappers\build\Windows\Release-x64\ + # This needs to be changed to something meaningful + key: wrapper-windows-release-64-${{hashfiles('.\wrappers\**')}} + - name: Check cache hit + # If there is no cache hit, just fail + if: steps.wrapper-cache.outputs.cache-hit != 'true' + run: exit 1 + - name: Run benchmarks + run: | + dotnet run -p .\Tests\PerformanceTests\ run -c Release --framework net5.0 -- -f * --join + rename-item -path ".\BenchmarkDotNet.Artifacts\results\*.json" -newname "benchmark-output.json" + - name: Save artifacts + # Not mandatory, but maybe it's good to do it? + uses: actions/upload-artifact@v2 + with: + name: benchmark-results + # Could be put in an environmental variable + path: .\BenchmarkDotNet.Artifacts\results + # Download previous benchmark result from cache (if exists) + - name: Store benchmark results + uses: Happypig375/github-action-benchmark@v1.8.2 + with: + tool: 'benchmarkdotnet' + # Where the output from the benchmark tool is stored + output-file-path: .\BenchmarkDotNet.Artifacts\results\benchmark-output.json + # Workflow will not fail when an alert happens + fail-on-alert: false + gh-pages-branch: ${{env.gh-pages-branch}} + auto-push: true + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/response_file.txt b/.github/workflows/response_file.txt new file mode 100644 index 0000000000..b73a8fd81b --- /dev/null +++ b/.github/workflows/response_file.txt @@ -0,0 +1,4 @@ +zlib +openssl +--triplet +x64-windows-static \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index dc4cafcfad..10fc21be2c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -334,21 +334,6 @@ stage('Test') { reportTests 'TestResults.Weaver.xml' } } - }, - 'Benchmarks': { - rlmNode('dotnet && windows') { - unstash 'dotnet-source' - dir('Realm/packages') { unstash 'packages' } - - dir('Tests/PerformanceTests') { - bat """ - dotnet build -c ${configuration} -f net5.0 -p:RestoreConfigFile=${env.WORKSPACE}/Tests/Test.NuGet.Config -p:UseRealmNupkgsWithVersion=${packageVersion} - dotnet run -c ${configuration} -f net5.0 --no-build -- -f * - """.trim() - - archiveArtifacts "BenchmarkDotNet.Artifacts/results/*.md" - } - } } ] @@ -487,4 +472,4 @@ List> mapToList(Map map) { @NonCPS String getVersion(String name) { return (name =~ /Realm.Fody.(.+).nupkg/)[0][1] -} \ No newline at end of file +} diff --git a/Tests/PerformanceTests/Program.cs b/Tests/PerformanceTests/Program.cs index 9c0a65a71e..cf99f165e4 100644 --- a/Tests/PerformanceTests/Program.cs +++ b/Tests/PerformanceTests/Program.cs @@ -16,21 +16,15 @@ // //////////////////////////////////////////////////////////////////////////// -using System; using System.Linq; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Exporters.Json; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; -using BenchmarkDotNet.Portability.Cpu; -using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.InProcess.Emit; -using Perfolizer.Horology; namespace PerformanceTests { @@ -50,32 +44,9 @@ public static void Main(string[] args) .AddDiagnoser(MemoryDiagnoser.Default) .AddJob(Job.Default.WithToolchain(InProcessEmitToolchain.Instance)) .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Method, MethodOrderPolicy.Alphabetical)) - .AddExporter(new JenkinsHtmlExporter(), MarkdownExporter.GitHub, JsonExporter.Full); + .AddExporter(MarkdownExporter.GitHub, JsonExporter.FullCompressed); BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); } - - // This is needed because the Jenkins user does not have permissions to use System.Management.ManagementObjectSearcher. - // There's no public API to prevent BenchmarkDotNet from looking it up, so when building the summary, we get an access - // denied exception. This uses reflection to replace the HostEnvironmentInfo.CpuInfo lazy with one that returns a mock - // CpuInfo. Since this is the first exporter being run, it will modify the summary for the rest, so we don't need to - // fiddle with them. Once we move to Github actions, this can be removed. - private class JenkinsHtmlExporter : HtmlExporter - { - private CpuInfo _mockCpuInfo = new CpuInfo("MockIntel(R) Core(TM) i7-6700HQ CPU 2.60GHz", - physicalProcessorCount: 1, - physicalCoreCount: 4, - logicalCoreCount: 8, - nominalFrequency: Frequency.FromMHz(3100), - maxFrequency: Frequency.FromMHz(3100), - minFrequency: Frequency.FromMHz(3100)); - - public override void ExportToLog(Summary summary, ILogger logger) - { - var cpuInfoProp = typeof(HostEnvironmentInfo).GetProperty(nameof(HostEnvironmentInfo.CpuInfo)); - cpuInfoProp.SetValue(summary.HostEnvironmentInfo, new Lazy(() => _mockCpuInfo)); - base.ExportToLog(summary, logger); - } - } } } diff --git a/wrappers/build.ps1 b/wrappers/build.ps1 index 6c1b5d80ef..3fae605090 100644 --- a/wrappers/build.ps1 +++ b/wrappers/build.ps1 @@ -22,6 +22,8 @@ param( [Parameter(Position=0)] [string]$Target = 'Windows', + [string]$Toolchain = 'c:\\src\\vcpkg\\scripts\\buildsystems\\vcpkg.cmake', + [Switch]$Incremental ) @@ -40,7 +42,7 @@ $vs = Get-VSSetupInstance | Select-VSSetupInstance -Product * -Latest -Require M $Env:path += ";$($vs.InstallationPath)\MSBuild\Current\Bin" $cmake = Join-Path $vs.InstallationPath -ChildPath "Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe" -$cmakeArgs = "-DCMAKE_GENERATOR_INSTANCE=$($vs.InstallationPath)", "-DCMAKE_BUILD_TYPE=$Configuration", "-DCMAKE_SYSTEM_NAME=$Target", "-DCMAKE_INSTALL_PREFIX=$PSScriptRoot\build", "-DCMAKE_TOOLCHAIN_FILE=c:\\src\\vcpkg\\scripts\\buildsystems\\vcpkg.cmake" +$cmakeArgs = "-DCMAKE_GENERATOR_INSTANCE=$($vs.InstallationPath)", "-DCMAKE_BUILD_TYPE=$Configuration", "-DCMAKE_SYSTEM_NAME=$Target", "-DCMAKE_INSTALL_PREFIX=$PSScriptRoot\build", "-DCMAKE_TOOLCHAIN_FILE=$Toolchain" if ($Target -eq 'WindowsStore') { $cmakeArgs += "-DCMAKE_SYSTEM_VERSION='10.0'"