From 2d85c877aae7de85d4a0e881e12da8c2ad971ccf Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Fri, 5 Jul 2024 10:08:01 +0200 Subject: [PATCH 1/3] Feat: support skipping tests (#419) * feat: support skipping tests Skipping tests can be useful in a CI environment where specific tests can be disabled viam moonwall config * update biome config * test: add tests to skip smoke tests * fix lint errors * add test execution to ci * fix: regexp and execution * test: add test cases * fix lint errors * remove unused import --- .github/workflows/main.yml | 3 ++- biome.json | 4 ++-- packages/cli/src/cmds/runTests.ts | 6 ++++++ packages/types/config_schema.json | 21 ++++++++++++++++++ packages/types/src/config.ts | 22 +++++++++++++++++++ test/moonwall.config.json | 34 ++++++++++++++++++++++++++++++ test/suites/smoke/test_skipMany.ts | 32 ++++++++++++++++++++++++++++ test/suites/smoke/test_skipNone.ts | 24 +++++++++++++++++++++ test/suites/smoke/test_skipOne.ts | 24 +++++++++++++++++++++ 9 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 test/suites/smoke/test_skipMany.ts create mode 100644 test/suites/smoke/test_skipNone.ts create mode 100644 test/suites/smoke/test_skipOne.ts diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42480eb3..8e596580 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,7 +110,7 @@ jobs: strategy: fail-fast: false matrix: - suite: ["dev_test", "dev_multi", "dev_seq"] + suite: ["dev_test", "dev_multi", "dev_seq", "dev_smoke"] steps: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v1 @@ -244,3 +244,4 @@ jobs: run: | cd test bun moonwall test ${{matrix.suite}} + diff --git a/biome.json b/biome.json index 3c9c687d..518d4214 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/1.6.1/schema.json", + "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json", "files": { "include": ["*.js", "*.ts", "*.json", "*.yml", "*.md"], "ignore": ["./dist/*", "./node_modules/*", "package.json"] @@ -13,7 +13,7 @@ }, "javascript": { "formatter": { - "trailingComma": "es5", + "trailingCommas": "es5", "semicolons": "always", "indentStyle": "space", "lineWidth": 100 diff --git a/packages/cli/src/cmds/runTests.ts b/packages/cli/src/cmds/runTests.ts index b85842d1..26f9c13f 100644 --- a/packages/cli/src/cmds/runTests.ts +++ b/packages/cli/src/cmds/runTests.ts @@ -106,6 +106,12 @@ export async function executeTests(env: Environment, additionalArgs?: any) { }, } satisfies UserConfig; + // transform in regexp pattern + if (env.skipTests && env.skipTests.length > 0) { + // the final pattern will look like this: "^((?!SO00T02|SM00T01|SM00T03).)*$" + additionalArgs.testNamePattern = `^((?!${env.skipTests?.map((test) => `${test.name}`).join("|")}).)*$`; + } + // TODO: Create options builder class const options = addThreadConfig(baseOptions, env.multiThreads); diff --git a/packages/types/config_schema.json b/packages/types/config_schema.json index c89a148a..3d8d2355 100644 --- a/packages/types/config_schema.json +++ b/packages/types/config_schema.json @@ -631,6 +631,27 @@ }, "type": "array" }, + "skipTests": { + "description": "A list of test to skip.", + "items": { + "properties": { + "name": { + "description": "The name of the test to skip. Eg. S22C500", + "type": "string" + }, + "reason": { + "description": "The reason for skipping the test. Must be provided. Eg. https://github.com/org/repo/issues/123.", + "type": "string" + }, + "since": { + "description": "The date when the test was skipped. Must be provided and be RFC3339 compliant. Eg. 2021-09-01T00:00:00Z", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, "testFileDir": { "description": "An array of directories with test files.", "items": { diff --git a/packages/types/src/config.ts b/packages/types/src/config.ts index acba348e..80362014 100644 --- a/packages/types/src/config.ts +++ b/packages/types/src/config.ts @@ -127,6 +127,28 @@ export type Environment = { * Toggle whether createBlock() will finalize blocks by default or not. */ defaultFinalization?: boolean; + + /** + * A list of test to skip. + */ + skipTests?: SkipTestSpec[]; +}; + +export type SkipTestSpec = { + /** + * The name of the test to skip. Eg. S22C500 + */ + name: string; + + /** + * The reason for skipping the test. Must be provided. Eg. https://github.com/org/repo/issues/123. + */ + reason: string; + + /** + * The date when the test was skipped. Must be provided and be RFC3339 compliant. Eg. 2021-09-01T00:00:00Z + */ + since: string; }; /** diff --git a/test/moonwall.config.json b/test/moonwall.config.json index 1c88f7e6..6d328553 100644 --- a/test/moonwall.config.json +++ b/test/moonwall.config.json @@ -794,6 +794,40 @@ }, "connections": [] }, + { + "name": "dev_smoke", + "foundation": { + "launchSpec": { + "disableRuntimeVersionCheck": true + }, + "type": "read_only" + }, + "connections": [ + { + "name": "para", + "type": "polkadotJs", + "endpoints": ["wss://moonbeam-rpc.dwellir.com"] + } + ], + "testFileDir": ["/suites/smoke"], + "skipTests": [ + { + "name": "SO00T02", + "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/1", + "since": "2024-01-28T00:00:00Z" + }, + { + "name": "SM00T01", + "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/2", + "since": "2024-01-28T00:00:00Z" + }, + { + "name": "SM00T03", + "reason": "https://github.com/moonbeam-foundation/moonbeam/issue/3", + "since": "2024-01-28T00:00:00Z" + } + ] + }, { "name": "multi_chopsticks", "testFileDir": ["suites/multi_chopsticks"], diff --git a/test/suites/smoke/test_skipMany.ts b/test/suites/smoke/test_skipMany.ts new file mode 100644 index 00000000..c2f9d82c --- /dev/null +++ b/test/suites/smoke/test_skipMany.ts @@ -0,0 +1,32 @@ +import { describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "SM00", + title: "New Test Suite", + foundationMethods: "read_only", + testCases: ({ it }) => { + it({ + id: "T01", + title: "Skipped Test", + test: () => { + expect(false).to.be.true; + }, + }); + + it({ + id: "T02", + title: "Passing test", + test: () => { + expect(true).to.be.true; + }, + }); + + it({ + id: "T03", + title: "Skipped test", + test: () => { + expect(false).to.be.true; + }, + }); + }, +}); diff --git a/test/suites/smoke/test_skipNone.ts b/test/suites/smoke/test_skipNone.ts new file mode 100644 index 00000000..41062603 --- /dev/null +++ b/test/suites/smoke/test_skipNone.ts @@ -0,0 +1,24 @@ +import { describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "SN00", + title: "New Test Suite", + foundationMethods: "read_only", + testCases: ({ it }) => { + it({ + id: "T01", + title: "Passing Test", + test: () => { + expect(true).to.be.true; + }, + }); + + it({ + id: "T02", + title: "Passing test", + test: () => { + expect(true).to.be.true; + }, + }); + }, +}); diff --git a/test/suites/smoke/test_skipOne.ts b/test/suites/smoke/test_skipOne.ts new file mode 100644 index 00000000..e9addcac --- /dev/null +++ b/test/suites/smoke/test_skipOne.ts @@ -0,0 +1,24 @@ +import { describeSuite, expect } from "@moonwall/cli"; + +describeSuite({ + id: "SO00", + title: "New Test Suite", + foundationMethods: "read_only", + testCases: ({ it }) => { + it({ + id: "T01", + title: "Passing Test", + test: () => { + expect(true).to.be.true; + }, + }); + + it({ + id: "T02", + title: "Skipped test", + test: () => { + expect(false).to.be.true; + }, + }); + }, +}); From b5f857fe7180995e73447895cc475d95f174ebbe Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Fri, 5 Jul 2024 10:28:59 +0200 Subject: [PATCH 2/3] add changeset for skip tests (#421) --- .changeset/clever-melons-battle.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/clever-melons-battle.md diff --git a/.changeset/clever-melons-battle.md b/.changeset/clever-melons-battle.md new file mode 100644 index 00000000..63ebe0ba --- /dev/null +++ b/.changeset/clever-melons-battle.md @@ -0,0 +1,10 @@ +--- +"@moonwall/cli": minor +"@moonwall/types": minor +"@moonwall/tests": minor +--- + +Support skipping tests by ID in config + + +This feature is amazing but also dangerous! Remember with great power comes great responsibilities \ No newline at end of file From a5d76fcef8b4beadf20692d728a4566b5da6facb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 10:33:04 +0200 Subject: [PATCH 3/3] Version Packages (#417) Co-authored-by: github-actions[bot] --- .changeset/clever-melons-battle.md | 10 ---------- .changeset/dry-hounds-nail.md | 6 ------ docs/CHANGELOG.md | 2 ++ docs/package.json | 2 +- packages/cli/CHANGELOG.md | 15 +++++++++++++++ packages/cli/package.json | 2 +- packages/types/CHANGELOG.md | 8 ++++++++ packages/types/package.json | 2 +- packages/util/CHANGELOG.md | 7 +++++++ packages/util/package.json | 2 +- test/CHANGELOG.md | 12 ++++++++++++ test/package.json | 2 +- 12 files changed, 49 insertions(+), 21 deletions(-) delete mode 100644 .changeset/clever-melons-battle.md delete mode 100644 .changeset/dry-hounds-nail.md diff --git a/.changeset/clever-melons-battle.md b/.changeset/clever-melons-battle.md deleted file mode 100644 index 63ebe0ba..00000000 --- a/.changeset/clever-melons-battle.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -"@moonwall/cli": minor -"@moonwall/types": minor -"@moonwall/tests": minor ---- - -Support skipping tests by ID in config - - -This feature is amazing but also dangerous! Remember with great power comes great responsibilities \ No newline at end of file diff --git a/.changeset/dry-hounds-nail.md b/.changeset/dry-hounds-nail.md deleted file mode 100644 index 2fbcbd5a..00000000 --- a/.changeset/dry-hounds-nail.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@moonwall/cli": patch -"@moonwall/tests": patch ---- - -downloader fix diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5ae6822f..4a553681 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,7 @@ # @moonwall/docs +## 5.3.0 + ## 5.2.0 ## 5.1.5 diff --git a/docs/package.json b/docs/package.json index 0ab8fc4f..7d1963ac 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,7 +1,7 @@ { "name": "@moonwall/docs", "type": "module", - "version": "5.2.0", + "version": "5.3.0", "private": true, "description": "Testing framework for the Moon family of projects", "scripts": { diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 963b8a47..b9e95b12 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,20 @@ # @moonwall/cli +## 5.3.0 + +### Minor Changes + +- b5f857f: Support skipping tests by ID in config + + This feature is amazing but also dangerous! Remember with great power comes great responsibilities + +### Patch Changes + +- a73230d: downloader fix +- Updated dependencies [b5f857f] + - @moonwall/types@5.3.0 + - @moonwall/util@5.3.0 + ## 5.2.0 ### Minor Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index ea7371cc..da083158 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@moonwall/cli", "type": "module", - "version": "5.2.0", + "version": "5.3.0", "description": "Testing framework for the Moon family of projects", "author": "timbrinded", "license": "ISC", diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index 78cea4b5..9e373e15 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -1,5 +1,13 @@ # @moonwall/types +## 5.3.0 + +### Minor Changes + +- b5f857f: Support skipping tests by ID in config + + This feature is amazing but also dangerous! Remember with great power comes great responsibilities + ## 5.2.0 ### Minor Changes diff --git a/packages/types/package.json b/packages/types/package.json index d50db257..6fec6070 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,7 +1,7 @@ { "name": "@moonwall/types", "type": "module", - "version": "5.2.0", + "version": "5.3.0", "description": "Testing framework for the Moon family of projects", "author": "timbrinded", "license": "ISC", diff --git a/packages/util/CHANGELOG.md b/packages/util/CHANGELOG.md index 5b348210..544788fb 100644 --- a/packages/util/CHANGELOG.md +++ b/packages/util/CHANGELOG.md @@ -1,5 +1,12 @@ # @moonwall/util +## 5.3.0 + +### Patch Changes + +- Updated dependencies [b5f857f] + - @moonwall/types@5.3.0 + ## 5.2.0 ### Minor Changes diff --git a/packages/util/package.json b/packages/util/package.json index c5e3e433..003e0560 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -1,7 +1,7 @@ { "name": "@moonwall/util", "type": "module", - "version": "5.2.0", + "version": "5.3.0", "description": "Testing framework for the Moon family of projects", "author": "timbrinded", "license": "ISC", diff --git a/test/CHANGELOG.md b/test/CHANGELOG.md index 80dc37ae..b469795b 100644 --- a/test/CHANGELOG.md +++ b/test/CHANGELOG.md @@ -1,5 +1,17 @@ # @moonwall/tests +## 5.3.0 + +### Minor Changes + +- b5f857f: Support skipping tests by ID in config + + This feature is amazing but also dangerous! Remember with great power comes great responsibilities + +### Patch Changes + +- a73230d: downloader fix + ## 5.2.0 ### Minor Changes diff --git a/test/package.json b/test/package.json index 96634674..7a4738dd 100644 --- a/test/package.json +++ b/test/package.json @@ -1,7 +1,7 @@ { "name": "@moonwall/tests", "type": "module", - "version": "5.2.0", + "version": "5.3.0", "private": true, "packageManager": "pnpm@9.1.4", "description": "Testing framework for the Moon family of projects",