diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97c075e95..38052e1d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,11 @@ jobs: ${{ runner.os }}-node- - run: npm install - run: npm run build + - name: Save previous data.js + run: | + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark run: cd examples/rust && cargo +nightly bench | tee output.txt @@ -25,7 +30,7 @@ jobs: name: Rust Benchmark tool: 'cargo' output-file-path: examples/rust/output.txt - - run: echo 'TODO Verify result here!' + - run: node ./scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' go: name: Run Go benchmark example runs-on: ubuntu-latest @@ -41,6 +46,11 @@ jobs: ${{ runner.os }}-node- - run: npm install - run: npm run build + - name: Save previous data.js + run: | + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - name: Run benchmark run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - name: Store benchmark result @@ -49,7 +59,7 @@ jobs: name: Go Benchmark tool: 'go' output-file-path: examples/go/output.txt - - run: echo 'TODO Verify result here!' + - run: node ./scripts/ci_validate_modification.js before_data.js 'Go Benchmark' benchmarkjs: name: Run JavaScript benchmark example runs-on: ubuntu-latest @@ -64,6 +74,11 @@ jobs: ${{ runner.os }}-node- - run: npm install - run: npm run build + - name: Save previous data.js + run: | + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - name: Run benchmark run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - name: Store benchmark result @@ -72,7 +87,7 @@ jobs: name: Benchmark.js Benchmark tool: 'benchmarkjs' output-file-path: examples/benchmarkjs/output.txt - - run: echo 'TODO Verify result here!' + - run: node ./scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' pytest-benchmark: name: Run Pytest benchmark example runs-on: ubuntu-latest @@ -88,6 +103,11 @@ jobs: ${{ runner.os }}-node- - run: npm install - run: npm run build + - name: Save previous data.js + run: | + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - name: Run benchmark run: | cd examples/pytest @@ -99,7 +119,7 @@ jobs: name: Python Benchmark with pytest-benchmark tool: 'pytest' output-file-path: examples/pytest/output.json - - run: echo 'TODO Verify result here!' + - run: node ./scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' unit-tests: name: Run unit tests runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index c942baf40..8c95ac9bd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ /examples/rust/target /test/*.js /test/*.js.map +/scripts/*.js +/scripts/*.js.map diff --git a/config.ts b/config.ts index ef55efcd7..70ab786ba 100644 --- a/config.ts +++ b/config.ts @@ -14,7 +14,7 @@ export interface Config { autoPush: boolean; } -const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest']; +export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest']; function validateToolType(tool: string): asserts tool is ToolType { if ((VALID_TOOLS as string[]).includes(tool)) { diff --git a/package-lock.json b/package-lock.json index f7ccd4127..d5ff3c287 100644 --- a/package-lock.json +++ b/package-lock.json @@ -167,6 +167,18 @@ "@types/node": "*" } }, + "@types/deep-diff": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/deep-diff/-/deep-diff-1.0.0.tgz", + "integrity": "sha512-ENsJcujGbCU/oXhDfQ12mSo/mCBWodT2tpARZKmatoSrf8+cGRCPi0KVj3I0FORhYZfLXkewXu7AoIWqiBLkNw==", + "dev": true + }, + "@types/deep-equal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@types/deep-equal/-/deep-equal-1.0.1.tgz", + "integrity": "sha512-mMUu4nWHLBlHtxXY17Fg6+ucS/MnndyOWyOe7MmwkoMYxvfQU2ajtRaEvqSUv+aVkMqH/C0NCI8UoVfRNQ10yg==", + "dev": true + }, "@types/eslint-visitor-keys": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", @@ -633,6 +645,26 @@ "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", "dev": true }, + "deep-diff": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/deep-diff/-/deep-diff-1.0.2.tgz", + "integrity": "sha512-aWS3UIVH+NPGCD1kki+DCU9Dua032iSsO43LqQpcs4R3+dVv7tX0qBGjiVHJHjplsoUM2XRO/KB92glqc68awg==", + "dev": true + }, + "deep-equal": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz", + "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==", + "dev": true, + "requires": { + "is-arguments": "^1.0.4", + "is-date-object": "^1.0.1", + "is-regex": "^1.0.4", + "object-is": "^1.0.1", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.2.0" + } + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -1347,6 +1379,12 @@ } } }, + "is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", + "dev": true + }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -1755,6 +1793,12 @@ "integrity": "sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==", "dev": true }, + "object-is": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.0.1.tgz", + "integrity": "sha1-CqYOyZiaCz7Xlc9NBvYs8a1lObY=", + "dev": true + }, "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", @@ -2079,6 +2123,15 @@ "util-deprecate": "^1.0.1" } }, + "regexp.prototype.flags": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.2.0.tgz", + "integrity": "sha512-ztaw4M1VqgMwl9HlPpOuiYgItcHlunW0He2fE6eNfT6E/CF2FtYi9ofOYe4mKntstYk0Fyh/rDRBdS3AnxjlrA==", + "dev": true, + "requires": { + "define-properties": "^1.1.2" + } + }, "regexpp": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", diff --git a/package.json b/package.json index 311807a0a..5ab7e0822 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,8 @@ "devDependencies": { "@types/acorn": "^4.0.5", "@types/cheerio": "^0.22.13", + "@types/deep-diff": "^1.0.0", + "@types/deep-equal": "^1.0.1", "@types/mocha": "^5.2.7", "@types/mock-require": "^2.0.0", "@types/node": "^12.12.7", @@ -42,6 +44,8 @@ "@typescript-eslint/parser": "^2.7.0", "acorn": "^7.1.0", "cheerio": "^1.0.0-rc.3", + "deep-diff": "^1.0.2", + "deep-equal": "^1.1.1", "eslint": "^6.6.0", "eslint-config-prettier": "^6.5.0", "eslint-plugin-mocha": "^6.2.1", diff --git a/scripts/ci_validate_modification.ts b/scripts/ci_validate_modification.ts new file mode 100644 index 000000000..233c693a4 --- /dev/null +++ b/scripts/ci_validate_modification.ts @@ -0,0 +1,237 @@ +import * as path from 'path'; +import { promises as fs } from 'fs'; +import * as cp from 'child_process'; +import { DataJson, BenchmarkEntries, SCRIPT_PREFIX } from '../write'; +import { VALID_TOOLS } from '../config'; +import { Benchmark } from '../extract'; +import { diff, Diff, DiffNew, DiffEdit, DiffArray } from 'deep-diff'; +import deepEq = require('deep-equal'); + +function help(): never { + throw new Error('Usage: node ci_validate_modification.js before_data.js'); +} + +async function exec(cmd: string): Promise { + console.log(`+ ${cmd}`); + return new Promise((resolve, reject) => { + cp.exec(cmd, (err, stdout, stderr) => { + if (err) { + reject(new Error(`Exec '${cmd}' failed with error ${err.message}. Stderr: '${stderr}'`)); + return; + } + resolve(stdout); + }); + }); +} + +async function readDataJson(file: string): Promise { + const content = await fs.readFile(file, 'utf8'); + return JSON.parse(content.slice(SCRIPT_PREFIX.length)); +} + +function validateDataJson(data: DataJson) { + const { lastUpdate, repoUrl, entries } = data; + const now = Date.now(); + if (lastUpdate > now) { + throw new Error(`Last update is not correct: ${lastUpdate} v.s. ${now}`); + } + + if (repoUrl !== 'https://github.com/rhysd/github-action-benchmark') { + throw new Error(`repoUrl is not correct: ${repoUrl}`); + } + + for (const benchName of Object.keys(entries)) { + for (const entry of entries[benchName]) { + const { commit, tool, date, benches } = entry; + if (!(VALID_TOOLS as string[]).includes(tool)) { + throw new Error(`Invalid tool ${tool}`); + } + if (!commit.url.startsWith('https://github.com/rhysd/github-action-benchmark/commit/')) { + throw new Error(`Invalid commit url: ${commit.url}`); + } + if (!commit.url.endsWith(commit.id)) { + throw new Error(`Commit ID ${commit.id} does not match to URL ${commit.url}`); + } + if (date > now) { + throw new Error(`Benchmark date is not correct: ${date} v.s. ${now}`); + } + for (const bench of benches) { + const { name, value, unit, range, extra } = bench; + const json = JSON.stringify(bench); + if (!name) { + throw new Error(`Benchmark result name is invalid: ${name} (${json})`); + } + if (typeof value !== 'number' || isNaN(value)) { + throw new Error(`Benchmark result value is invalid: ${value} (${json})`); + } + if (typeof unit !== 'string') { + throw new Error(`Benchmark result unit is invalid: ${unit} (${json})`); + } + if (range && typeof range !== 'string') { + throw new Error(`Benchmark result range is invalid: ${range} (${json})`); + } + if (extra && typeof extra !== 'string') { + throw new Error(`Benchmark result extra is invalid: ${extra} (${json})`); + } + } + } + } +} + +function assertNumberDiffEdit(diff: Diff): asserts diff is DiffEdit { + if (diff.kind !== 'E') { + throw new Error(`Given diff is not DiffEdit: ${JSON.stringify(diff)}`); + } + if (typeof diff.lhs !== 'number') { + throw new Error(`Given DiffEdit's lhs is not for number: ${diff.lhs}`); + } + if (typeof diff.rhs !== 'number') { + throw new Error(`Given DiffEdit's rhs is not for number: ${diff.rhs}`); + } +} + +function validateLastUpdateMod(diff: Diff) { + assertNumberDiffEdit(diff); + if (!deepEq(diff.path, ['lastUpdate'])) { + throw new Error(`Not diff for lastUpdate: ${JSON.stringify(diff.path)}`); + } + const { lhs, rhs } = diff; + if (lhs >= rhs) { + throw new Error(`Update of datetime is not correct. New is older: ${lhs} v.s. ${rhs}`); + } +} + +function assertDiffArray(diff: Diff): asserts diff is DiffArray { + if (diff.kind !== 'A') { + throw new Error(`Given diff is not DiffArray: ${JSON.stringify(diff)}`); + } +} + +function assertDiffNewBench(diff: Diff): asserts diff is DiffNew { + if (diff.kind !== 'N') { + throw new Error(`Given diff is not DiffNew: ${JSON.stringify(diff)}`); + } + const { rhs } = diff; + if (typeof rhs !== 'object' || rhs === null) { + throw new Error(`DiffNew for Benchmark object is actually not a object: ${rhs}`); + } + for (const prop of ['commit', 'date', 'tool', 'benches']) { + if (!(prop in rhs)) { + throw new Error(`Not a valid benchmark object in DiffNew: ${JSON.stringify(rhs)}`); + } + } +} + +function validateBenchmarkResultMod(diff: Diff, expectedBenchName: string, afterEntries: BenchmarkEntries) { + if (!(expectedBenchName in afterEntries)) { + throw new Error(`data.js after action does not contain '${expectedBenchName}' benchmark`); + } + + const benchEntries = afterEntries[expectedBenchName]; + if (benchEntries.length === 0) { + throw new Error('Benchmark entry is empty after action'); + } + + assertDiffArray(diff); + + if (!deepEq(diff.path, ['entries', expectedBenchName])) { + throw new Error(`Diff path is not expected for adding new benchmark: ${JSON.stringify(diff.path)}`); + } + + diff = diff.item; + assertDiffNewBench(diff); + + const added: Benchmark = diff.rhs; + const last = benchEntries[benchEntries.length - 1]; + if (last.commit.id !== added.commit.id) { + throw new Error( + `Newly added benchmark ${JSON.stringify(added)} is not the last one in data.js ${JSON.stringify(last)}`, + ); + } + + for (const entry of benchEntries) { + if (entry.date > added.date) { + throw new Error(`Older entry's date ${JSON.stringify(entry)} is newer than added ${JSON.stringify(added)}`); + } + + if (entry.tool !== added.tool) { + throw new Error(`Tool is different between ${JSON.stringify(entry)} and ${JSON.stringify(added)}`); + } + + for (const addedBench of added.benches) { + for (const prevBench of entry.benches) { + if (prevBench.name === addedBench.name) { + if (prevBench.unit !== addedBench.unit) { + throw new Error( + `Unit is different between previous benchmark and newly added benchmark: ${JSON.stringify( + prevBench, + )} v.v. ${JSON.stringify(addedBench)}`, + ); + } + } + } + } + } +} + +async function main() { + console.log('Start validating modifications by action with args', process.argv); + + if (process.argv.length != 4) { + help(); + } + + if (['-h', '--help'].includes(process.argv[2])) { + help(); + } + + console.log('Checking pre-condition'); + const stats = await fs.stat(path.resolve('.git')); + if (!stats.isDirectory()) { + throw new Error('This script must be run at root directory of repository'); + } + + const beforeDataJs = path.resolve(process.argv[2]); + const expectedBenchName = process.argv[3]; + + console.log('Validating modifications by action'); + console.log(` data.js before action: ${beforeDataJs}`); + + console.log('Reading data.js before action as JSON'); + const beforeJson = await readDataJson(beforeDataJs); + + console.log('Validating current branch'); + const stdout = await exec('git show -s --pretty=%d HEAD'); + if (stdout.includes('HEAD -> ')) { + throw new Error(`Current branch is not detached head: '${stdout}'`); + } + + console.log('Retrieving data.js after action'); + await exec('git checkout gh-pages'); + const afterJson = await readDataJson('dev/bench/data.js'); + await exec('git checkout -'); + + console.log('Validating data.js both before/after action'); + validateDataJson(beforeJson); + validateDataJson(afterJson); + + const diffs = diff(beforeJson, afterJson); + console.log('Validating diffs:', diffs); + + if (!diffs || diffs.length !== 2) { + throw new Error('Number of diffs are incorrect. Exact 2 diffs are expected'); + } + + console.log('Validating lastUpdate modification'); + validateLastUpdateMod(diffs[0]); + + console.log('Validating benchmark result modification'); + validateBenchmarkResultMod(diffs[1], expectedBenchName, afterJson.entries); + + console.log('👌'); +} + +main().catch(err => { + console.error(err); + process.exit(110); +}); diff --git a/test/config.ts b/test/config.ts index 4b6fd7941..cce4d022c 100644 --- a/test/config.ts +++ b/test/config.ts @@ -21,7 +21,7 @@ mock('@actions/core', { }); // This line must be called after mocking -const { configFromJobInput } = require('../config'); +const { configFromJobInput, VALID_TOOLS } = require('../config'); describe('configFromJobInput()', function() { const cwd = process.cwd(); @@ -36,7 +36,7 @@ describe('configFromJobInput()', function() { }); const tests = [ - ...(['cargo', 'go', 'benchmarkjs', 'pytest'] as const).map(tool => ({ + ...VALID_TOOLS.map((tool: string) => ({ what: 'valid inputs for ' + tool, inputs: { name: 'Benchmark', diff --git a/tsconfig.json b/tsconfig.json index 67ac1f32f..332002718 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,6 +26,7 @@ "write.ts", "default_index_html.ts", "test/default_index_html.ts", - "octokit_graphql.d.ts" + "octokit_graphql.d.ts", + "scripts/ci_validate_modification.ts" ] } diff --git a/write.ts b/write.ts index 1eb133b92..fed75e44e 100644 --- a/write.ts +++ b/write.ts @@ -8,14 +8,14 @@ import { Benchmark } from './extract'; import { Config } from './config'; import { DEFAULT_INDEX_HTML } from './default_index_html'; -type BenchmarkEntries = { [name: string]: Benchmark[] }; -interface DataJson { +export type BenchmarkEntries = { [name: string]: Benchmark[] }; +export interface DataJson { lastUpdate: number; repoUrl: string; entries: BenchmarkEntries; } -const SCRIPT_PREFIX = 'window.BENCHMARK_DATA = '; +export const SCRIPT_PREFIX = 'window.BENCHMARK_DATA = '; async function loadDataJson(dataPath: string): Promise { try {