From 2d85c877aae7de85d4a0e881e12da8c2ad971ccf Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Fri, 5 Jul 2024 10:08:01 +0200 Subject: [PATCH] Feat: support skipping tests (#419) * feat: support skipping tests Skipping tests can be useful in a CI environment where specific tests can be disabled viam moonwall config * update biome config * test: add tests to skip smoke tests * fix lint errors * add test execution to ci * fix: regexp and execution * test: add test cases * fix lint errors * remove unused import --- .github/workflows/main.yml | 3 ++- biome.json | 4 ++-- packages/cli/src/cmds/runTests.ts | 6 ++++++ packages/types/config_schema.json | 21 ++++++++++++++++++ packages/types/src/config.ts | 22 +++++++++++++++++++ test/moonwall.config.json | 34 ++++++++++++++++++++++++++++++ test/suites/smoke/test_skipMany.ts | 32 ++++++++++++++++++++++++++++ test/suites/smoke/test_skipNone.ts | 24 +++++++++++++++++++++ test/suites/smoke/test_skipOne.ts | 24 +++++++++++++++++++++ 9 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 test/suites/smoke/test_skipMany.ts create mode 100644 test/suites/smoke/test_skipNone.ts create mode 100644 test/suites/smoke/test_skipOne.ts diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42480eb3..8e596580 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,7 +110,7 @@ jobs: strategy: fail-fast: false matrix: - suite: ["dev_test", "dev_multi", "dev_seq"] + suite: ["dev_test", "dev_multi", "dev_seq", "dev_smoke"] steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v1 @@ -244,3 +244,4 @@ jobs: run: | cd test bun moonwall test ${{matrix.suite}} + diff --git a/biome.json b/biome.json index 3c9c687d..518d4214 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/1.6.1/schema.json", + "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", "files": { "include": ["*.js", "*.ts", "*.json", "*.yml", "*.md"], "ignore": ["./dist/*", "./node_modules/*", "package.json"] @@ -13,7 +13,7 @@ }, "javascript": { "formatter": { - "trailingComma": "es5", + "trailingCommas": "es5", "semicolons": "always", "indentStyle": "space", "lineWidth": 100 diff --git a/packages/cli/src/cmds/runTests.ts b/packages/cli/src/cmds/runTests.ts index b85842d1..26f9c13f 100644 --- a/packages/cli/src/cmds/runTests.ts +++ b/packages/cli/src/cmds/runTests.ts @@ -106,6 +106,12 @@ export async function executeTests(env: Environment, additionalArgs?: any) { }, } satisfies UserConfig; + // transform in regexp pattern + if (env.skipTests && env.skipTests.length > 0) { + // the final pattern will look like this: "^((?!SO00T02|SM00T01|SM00T03).)*$" + additionalArgs.testNamePattern = `^((?!${env.skipTests?.map((test) => `${test.name}`).join("|")}).)*$`; + } + // TODO: Create options builder class const options = addThreadConfig(baseOptions, env.multiThreads); diff --git a/packages/types/config_schema.json b/packages/types/config_schema.json index c89a148a..3d8d2355 100644 --- a/packages/types/config_schema.json +++ b/packages/types/config_schema.json @@ -631,6 +631,27 @@ }, "type": "array" }, + "skipTests": { + "description": "A list of test to skip.", + "items": { + "properties": { + "name": { + "description": "The name of the test to skip. Eg. S22C500", + "type": "string" + }, + "reason": { + "description": "The reason for skipping the test. Must be provided. Eg. https://github.com/org/repo/issues/123.", + "type": "string" + }, + "since": { + "description": "The date when the test was skipped. Must be provided and be RFC3339 compliant. Eg. 2021-09-01T00:00:00Z", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "testFileDir": { "description": "An array of directories with test files.", "items": { diff --git a/packages/types/src/config.ts b/packages/types/src/config.ts index acba348e..80362014 100644 --- a/packages/types/src/config.ts +++ b/packages/types/src/config.ts @@ -127,6 +127,28 @@ export type Environment = { * Toggle whether createBlock() will finalize blocks by default or not. */ defaultFinalization?: boolean; + + /** + * A list of test to skip. + */ + skipTests?: SkipTestSpec[]; +}; + +export type SkipTestSpec = { + /** + * The name of the test to skip. Eg. S22C500 + */ + name: string; + + /** + * The reason for skipping the test. Must be provided. Eg. https://github.com/org/repo/issues/123. + */ + reason: string; + + /** + * The date when the test was skipped. Must be provided and be RFC3339 compliant. Eg. 2021-09-01T00:00:00Z + */ + since: string; }; /** diff --git a/test/moonwall.config.json b/test/moonwall.config.json index 1c88f7e6..6d328553 100644 --- a/test/moonwall.config.json +++ b/test/moonwall.config.json @@ -794,6 +794,40 @@ }, "connections": [] }, + { + "name": "dev_smoke", + "foundation": { + "launchSpec": { + "disableRuntimeVersionCheck": true + }, + "type": "read_only" + }, + "connections": [ + { + "name": "para", + "type": "polkadotJs", + "endpoints": ["wss://moonbeam-rpc.dwellir.com"] + } + ], + "testFileDir": ["/suites/smoke"], + "skipTests": [ + { + "name": "SO00T02", + "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/1", + "since": "2024-01-28T00:00:00Z" + }, + { + "name": "SM00T01", + "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/2", + "since": "2024-01-28T00:00:00Z" + }, + { + "name": "SM00T03", + "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/3", + "since": "2024-01-28T00:00:00Z" + } + ] + }, { "name": "multi_chopsticks", "testFileDir": ["suites/multi_chopsticks"], diff --git a/test/suites/smoke/test_skipMany.ts b/test/suites/smoke/test_skipMany.ts new file mode 100644 index 00000000..c2f9d82c --- /dev/null +++ b/test/suites/smoke/test_skipMany.ts @@ -0,0 +1,32 @@ +import { describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "SM00", + title: "New Test Suite", + foundationMethods: "read_only", + testCases: ({ it }) => { + it({ + id: "T01", + title: "Skipped Test", + test: () => { + expect(false).to.be.true; + }, + }); + + it({ + id: "T02", + title: "Passing test", + test: () => { + expect(true).to.be.true; + }, + }); + + it({ + id: "T03", + title: "Skipped test", + test: () => { + expect(false).to.be.true; + }, + }); + }, +}); diff --git a/test/suites/smoke/test_skipNone.ts b/test/suites/smoke/test_skipNone.ts new file mode 100644 index 00000000..41062603 --- /dev/null +++ b/test/suites/smoke/test_skipNone.ts @@ -0,0 +1,24 @@ +import { describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "SN00", + title: "New Test Suite", + foundationMethods: "read_only", + testCases: ({ it }) => { + it({ + id: "T01", + title: "Passing Test", + test: () => { + expect(true).to.be.true; + }, + }); + + it({ + id: "T02", + title: "Passing test", + test: () => { + expect(true).to.be.true; + }, + }); + }, +}); diff --git a/test/suites/smoke/test_skipOne.ts b/test/suites/smoke/test_skipOne.ts new file mode 100644 index 00000000..e9addcac --- /dev/null +++ b/test/suites/smoke/test_skipOne.ts @@ -0,0 +1,24 @@ +import { describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "SO00", + title: "New Test Suite", + foundationMethods: "read_only", + testCases: ({ it }) => { + it({ + id: "T01", + title: "Passing Test", + test: () => { + expect(true).to.be.true; + }, + }); + + it({ + id: "T02", + title: "Skipped test", + test: () => { + expect(false).to.be.true; + }, + }); + }, +});