Skip to content

Commit

Permalink
ci: further parallelize execution of Jest tests (#4334)
Browse files Browse the repository at this point in the history
  • Loading branch information
adrians5j authored Oct 15, 2024
1 parent a11bb5b commit 5e0b6cb
Show file tree
Hide file tree
Showing 7 changed files with 808 additions and 669 deletions.
290 changes: 127 additions & 163 deletions .github/workflows/pullRequests.yml

Large diffs are not rendered by default.

187 changes: 127 additions & 60 deletions .github/workflows/pushDev.yml

Large diffs are not rendered by default.

187 changes: 127 additions & 60 deletions .github/workflows/pushNext.yml

Large diffs are not rendered by default.

218 changes: 110 additions & 108 deletions .github/workflows/wac/pullRequests.wac.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,114 +192,116 @@ export const pullRequests = createWorkflow({
jestTestsNoStorage: createJestTestsJob(null),
jestTestsDdb: createJestTestsJob("ddb"),
jestTestsDdbEs: createJestTestsJob("ddb-es"),
jestTestsDdbOs: createJestTestsJob("ddb-os"),
jestTestsDdbOs: createJestTestsJob("ddb-os")

verdaccioPublish: createJob({
name: "Publish to Verdaccio",
needs: ["constants", "build"],
if: "needs.constants.outputs.is-fork-pr != 'true'",
checkout: {
"fetch-depth": 0,
ref: "${{ github.event.pull_request.head.ref }}",
path: DIR_WEBINY_JS
},
steps: [
...yarnCacheSteps,
...runBuildCacheSteps,
...installBuildSteps,
...withCommonParams(
[
{
name: "Start Verdaccio local server",
run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
},
{
name: "Configure NPM to use local registry",
run: "npm config set registry http://localhost:4873"
},
{
name: "Set git email",
run: 'git config --global user.email "[email protected]"'
},
{
name: "Set git username",
run: 'git config --global user.name "webiny-bot"'
},
{
name: 'Create ".npmrc" file in the project root, with a dummy auth token',
run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
},
{
name: "Version and publish to Verdaccio",
run: "yarn release --type=verdaccio"
}
],
{ "working-directory": DIR_WEBINY_JS }
),
{
name: "Upload verdaccio files",
uses: "actions/upload-artifact@v4",
with: {
name: "verdaccio-files",
"retention-days": 1,
"include-hidden-files": true,
path: [
DIR_WEBINY_JS + "/.verdaccio/",
DIR_WEBINY_JS + "/.verdaccio.yaml"
].join("\n")
}
}
]
}),
testCreateWebinyProject: createJob({
name: 'Test "create-webiny-project"',
needs: "verdaccioPublish",
strategy: {
"fail-fast": false,
matrix: {
os: ["ubuntu-latest"],
node: [NODE_VERSION]
}
},
"runs-on": "${{ matrix.os }}",
checkout: false,
setupNode: {
"node-version": "${{ matrix.node }}"
},
steps: [
{
uses: "actions/download-artifact@v4",
with: {
name: "verdaccio-files",
path: "verdaccio-files"
}
},
{
name: "Start Verdaccio local server",
"working-directory": "verdaccio-files",
run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
},
{
name: "Configure NPM to use local registry",
run: "npm config set registry http://localhost:4873"
},
{
name: "Set git email",
run: 'git config --global user.email "[email protected]"'
},
{
name: "Set git username",
run: 'git config --global user.name "webiny-bot"'
},
{
name: "Disable Webiny telemetry",
run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
},
{
name: "Create a new Webiny project",
run: `npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"${AWS_REGION}"}\'\n`
}
]
})
// We commented out these tests because, A) they don't bring much value, and B) these are
// run within "push" workflows anyway (we deploy a Webiny instance and run E2E tests there).
// verdaccioPublish: createJob({
// name: "Publish to Verdaccio",
// needs: ["constants", "build"],
// if: "needs.constants.outputs.is-fork-pr != 'true'",
// checkout: {
// "fetch-depth": 0,
// ref: "${{ github.event.pull_request.head.ref }}",
// path: DIR_WEBINY_JS
// },
// steps: [
// ...yarnCacheSteps,
// ...runBuildCacheSteps,
// ...installBuildSteps,
// ...withCommonParams(
// [
// {
// name: "Start Verdaccio local server",
// run: "npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: 'Create ".npmrc" file in the project root, with a dummy auth token',
// run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc"
// },
// {
// name: "Version and publish to Verdaccio",
// run: "yarn release --type=verdaccio"
// }
// ],
// { "working-directory": DIR_WEBINY_JS }
// ),
// {
// name: "Upload verdaccio files",
// uses: "actions/upload-artifact@v4",
// with: {
// name: "verdaccio-files",
// "retention-days": 1,
// "include-hidden-files": true,
// path: [
// DIR_WEBINY_JS + "/.verdaccio/",
// DIR_WEBINY_JS + "/.verdaccio.yaml"
// ].join("\n")
// }
// }
// ]
// }),
// testCreateWebinyProject: createJob({
// name: 'Test "create-webiny-project"',
// needs: "verdaccioPublish",
// strategy: {
// "fail-fast": false,
// matrix: {
// os: ["ubuntu-latest"],
// node: [NODE_VERSION]
// }
// },
// "runs-on": "${{ matrix.os }}",
// checkout: false,
// setupNode: {
// "node-version": "${{ matrix.node }}"
// },
// steps: [
// {
// uses: "actions/download-artifact@v4",
// with: {
// name: "verdaccio-files",
// path: "verdaccio-files"
// }
// },
// {
// name: "Start Verdaccio local server",
// "working-directory": "verdaccio-files",
// run: "yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml"
// },
// {
// name: "Configure NPM to use local registry",
// run: "npm config set registry http://localhost:4873"
// },
// {
// name: "Set git email",
// run: 'git config --global user.email "[email protected]"'
// },
// {
// name: "Set git username",
// run: 'git config --global user.name "webiny-bot"'
// },
// {
// name: "Disable Webiny telemetry",
// run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n'
// },
// {
// name: "Create a new Webiny project",
// run: 'npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc \'{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}\' --template-options \'{"region":"eu-central-1"}\'\n'
// }
// ]
// })
}
});
52 changes: 40 additions & 12 deletions .github/workflows/wac/utils/listPackagesWithJestTests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@
* Dictates how package tests will be executed. With this script, we achieve
* parallelization of execution of Jest tests. Note: do not use any 3rd party
* libraries because we need this script to be executed in our CI/CD, as fast as possible.
* Using 3rd party libraries would require `yarn install` to be run before this script is executed.
*/

import fs from "fs";
import path from "path";
import crypto from "crypto";

/**
* Some packages require custom handling.
Expand All @@ -20,6 +22,18 @@ interface PackageWithTestsWithId extends PackageWithTests {
id: string;
}

// Takes a PackageWithTests object and returns an array of commands, where each
// command is just running a subset of tests. This is achieved by using the
// Jest's `--shard` option.
const shardPackageTestExecution = (pkg: PackageWithTests, shardsCount: number = 6) => {
const commands: PackageWithTests[] = [];
for (let currentShard = 1; currentShard <= shardsCount; currentShard++) {
commands.push({ ...pkg, cmd: pkg.cmd + ` --shard=${currentShard}/${shardsCount}` });
}

return commands;
};

const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
// Ignore "i18n" package.
i18n: () => [],
Expand Down Expand Up @@ -84,9 +98,18 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {

"api-page-builder": () => {
return [
{ cmd: "packages/api-page-builder --storage=ddb-es,ddb", storage: "ddb-es" },
{ cmd: "packages/api-page-builder --storage=ddb-os,ddb", storage: "ddb-os" },
{ cmd: "packages/api-page-builder --storage=ddb", storage: "ddb" }
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb-es,ddb",
storage: "ddb-es"
}),
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb-os,ddb",
storage: "ddb-os"
}),
...shardPackageTestExecution({
cmd: "packages/api-page-builder --storage=ddb",
storage: "ddb"
})
];
},
"api-page-builder-so-ddb-es": () => {
Expand Down Expand Up @@ -125,9 +148,18 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {

"api-headless-cms": () => {
return [
{ cmd: "packages/api-headless-cms --storage=ddb", storage: "ddb" },
{ cmd: "packages/api-headless-cms --storage=ddb-es,ddb", storage: "ddb-es" },
{ cmd: "packages/api-headless-cms --storage=ddb-os,ddb", storage: "ddb-os" }
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb",
storage: "ddb"
}),
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb-es,ddb",
storage: "ddb-es"
}),
...shardPackageTestExecution({
cmd: "packages/api-headless-cms --storage=ddb-os,ddb",
storage: "ddb-os"
})
];
},
"api-headless-cms-import-export": () => {
Expand Down Expand Up @@ -296,12 +328,8 @@ const CUSTOM_HANDLERS: Record<string, () => Array<PackageWithTests>> = {
const testFilePattern = /test\.j?t?sx?$/;

const cmdToId = (cmd: string) => {
return cmd
.replace("packages/", "")
.replace("--storage=", "")
.replace(/[,\s]/g, "_")
.replace(/[\(\)\[\]]/g, "")
.toLowerCase();
// Just convert the command to kebab-case.
return crypto.createHash("md5").update(cmd).digest("hex");
};

/**
Expand Down
2 changes: 1 addition & 1 deletion docs/DEPLOY_WEBINY_PROJECT_CF_TEMPLATE.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -538,4 +538,4 @@ Resources:
GroupName:
Ref: DeployWebinyProjectGroup3
Users:
- Ref: Username
- Ref: Username
Loading

0 comments on commit 5e0b6cb

Please sign in to comment.