diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3adaacdde..4066e5507 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,32 +12,50 @@ jobs: runs-on: ${{ matrix.os }} steps: # TODO use YAML anchors feature when it gets supported - - { uses: actions/checkout@v2 } - - { uses: actions/cache@v2, id: cache-node-modules, with: { path: ./node_modules, key: "node-modules-cache-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.NODE_MODULES_CACHE_VERSION }}" } } - - { name: system setup, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } - { uses: actions/setup-node@v1, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}" } } - - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os != 'Windows', run: "yarn --pure-lockfile --network-timeout 60000" } - - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/install-node-modules.ps1 } + - { uses: actions/checkout@v2 } + - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } + - name: get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache yarn cache directory + uses: actions/cache@v2 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: "yarn-cache-dir-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.YARN_CACHE_DIR_CACHE_VERSION }}" + - { name: install node-gyp, if: runner.os == 'Windows', run: ./scripts/ci/github/install-node-gyp.ps1 } + - { name: install node modules, run: "yarn --pure-lockfile --network-timeout 60000" } # - name: cache artifact uses: actions/cache@v2 - with: { path: ./output/git/backup, key: "proton-clients-cache-${{ hashFiles('./src/shared/proton-apps-constants.ts') }}-${{ secrets.PROTON_CLIENTS_ARTIFACT_CACHE_VERSION }}" } + with: + path: | + ./output/git/backup + ./app/assets/db-search-monaco-editor + key: "proton-clients-cache-${{ hashFiles('./src/shared/proton-apps-constants.ts') }}-${{ secrets.PROTON_CLIENTS_ARTIFACT_CACHE_VERSION }}" - { name: print ./output/git/backup files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./output/git/backup } - { name: build, run: ./scripts/ci/prepare-webclients.sh } - { name: print ./output/git/backup files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./output/git/backup } - - { name: tar artifact, run: tar -cvf proton-clients-artifact.tar ./output/git/backup } + - { name: tar artifact, run: tar -cvf proton-clients-artifact.tar ./output/git/backup ./app/assets/db-search-monaco-editor } - { name: upload artifact, uses: actions/upload-artifact@v2, with: { name: proton-clients-artifact, path: ./proton-clients-artifact.tar, if-no-files-found: error } } build-app: strategy: { matrix: { os: [ ubuntu-20.04, windows-2019, macos-10.15 ] }, fail-fast: false } runs-on: ${{ matrix.os }} steps: # TODO use YAML anchors feature when it gets supported - - { uses: actions/checkout@v2 } - - { uses: actions/cache@v2, id: cache-node-modules, with: { path: ./node_modules, key: "node-modules-cache-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.NODE_MODULES_CACHE_VERSION }}" } } - - { name: system setup, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } - { uses: actions/setup-node@v1, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}" } } - - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os != 'Windows', run: "yarn --pure-lockfile --network-timeout 60000" } - - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/install-node-modules.ps1 } + - { uses: actions/checkout@v2 } + - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } + - name: get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache yarn cache directory + uses: actions/cache@v2 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: "yarn-cache-dir-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.YARN_CACHE_DIR_CACHE_VERSION }}" + - { name: install node-gyp, if: runner.os == 'Windows', run: ./scripts/ci/github/install-node-gyp.ps1 } + - { name: install node modules, run: "yarn --pure-lockfile --network-timeout 60000" } # - { name: build, run: 'npm exec --package=npm-run-all -- npm-run-all lint test:electron-main build' } - { name: print ./app files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./app } @@ -49,19 +67,26 @@ jobs: runs-on: ${{ matrix.os }} steps: # TODO use YAML anchors feature when it gets supported - - { uses: actions/checkout@v2 } - - { uses: actions/cache@v2, id: cache-node-modules, with: { path: ./node_modules, key: "node-modules-cache-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.NODE_MODULES_CACHE_VERSION }}" } } - - { name: system setup, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } - { uses: actions/setup-node@v1, with: { node-version: "${{ env.ELECTRON_MAIL_NODE_VERSION }}" } } - - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os != 'Windows', run: "yarn --pure-lockfile --network-timeout 60000" } - - { name: install node modules, if: steps.cache-node-modules.outputs.cache-hit != 'true' && runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/install-node-modules.ps1 } + - { uses: actions/checkout@v2 } + - { name: system setup, if: runner.os == 'Linux', run: ./scripts/ci/github/system-setup-linux.sh } + - name: get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + - name: cache yarn cache directory + uses: actions/cache@v2 + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: "yarn-cache-dir-${{ matrix.os }}-${{ hashFiles('./yarn.lock', './patch-package/*.patch') }}-${{ secrets.YARN_CACHE_DIR_CACHE_VERSION }}" + - { name: install node-gyp, if: runner.os == 'Windows', run: ./scripts/ci/github/install-node-gyp.ps1 } + - { name: install node modules, run: "yarn --pure-lockfile --network-timeout 60000" } # - { name: download proton clients artifact, uses: actions/download-artifact@v2, with: { name: proton-clients-artifact } } - { name: unpack proton clients artifact, shell: bash, run: tar -xvf ./proton-clients-artifact.tar } - { name: download app artifact, uses: actions/download-artifact@v2, with: { name: 'app-artifact-${{ runner.os }}', path: ./app } } - { name: assets, run: "yarn assets", env: { ELECTRON_MAIL_SHOULD_FAIL_ON_BUILD: "true" } } - { name: print ./app files, shell: bash, run: ./scripts/ci/list-fs-content.sh ./app } - - { name: build, if: runner.os == 'Windows', shell: pwsh, run: ./scripts/ci/github/package-app.ps1 } + - { name: build, if: runner.os == 'Windows', run: ./scripts/ci/github/package-app.ps1 } - { name: build, if: runner.os == 'Linux', run: ./scripts/ci/github/package-app-linux.sh } - { name: build, if: matrix.os == 'macos-10.15', run: ./scripts/ci/github/package-app-osx.sh, env: { ARTIFACT_NAME_POSTFIX: "-catalina" } } - name: upload e2e/spectron output diff --git a/.travis.yml b/.travis.yml index 43434bbe3..7c46e0b2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,9 @@ jobs: include: - stage: "build proton clients" <<: *macos-common - cache: { yarn: false, npm: false, directories: [ "./output/git/backup" ] } + cache: { yarn: false, npm: false, directories: [ "./output/git/backup", "./app/assets/db-search-monaco-editor" ] } script: ./scripts/ci/prepare-webclients.sh - workspaces: { create: { name: "macos-webclients-artifact", paths: [ "./output/git/backup" ] } } + workspaces: { create: { name: "macos-webclients-artifact", paths: [ "./output/git/backup", "./app/assets/db-search-monaco-editor" ] } } - stage: "build app package" <<: *macos-common <<: *macos-build-app diff --git a/package.json b/package.json index d4a8d74fb..1cbfd44c1 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,6 @@ "test:electron-main:compiled:compile": "cross-env TS_NODE_FILES=true npx --no --node-options=\"--require tsconfig-paths/register\" ttsc --project ./src/electron-main/__test__/tsconfig.json", "test:web": "cross-env NODE_ENV=test TS_NODE_FILES=true npx --no --node-options=\"--require tsconfig-paths/register\" karma start ./src/web/browser-window/test/karma.conf.ts --single-run", "scripts/ci/appveyor/download-webclients-artifact": "yarn ts-node:shortcut ./scripts/ci/appveyor/download-webclients-artifact.ts", - "scripts/ci/print-webclients-dist-paths-pattern": "yarn --silent ts-node:shortcut ./scripts/ci/print-webclients-dist-paths-pattern.ts", "scripts/code-generation/electron-main": "yarn ts-node:shortcut ./scripts/code-generation/electron-main.ts", "scripts/dist-packages/print-hashes": "yarn ts-node:shortcut ./scripts/dist-packages/print-hashes.ts ./dist", "scripts/dist-packages/upload": "yarn ts-node:shortcut ./scripts/dist-packages/upload.ts ./dist ./output", diff --git a/scripts/ci/archive-webclients-dist-only.sh b/scripts/ci/archive-webclients-dist-only.sh index dc7fb8e46..1e4934f88 100755 --- a/scripts/ci/archive-webclients-dist-only.sh +++ b/scripts/ci/archive-webclients-dist-only.sh @@ -5,7 +5,7 @@ set -ev ARCHIVE_FILE=$1 # paths string separated with ";" -WEBCLIENT_DIST_PATHS_STRING=$(yarn --silent scripts/ci/print-webclients-dist-paths-pattern) +WEBCLIENT_DIST_PATHS_STRING=$(yarn --silent ts-node:shortcut ./scripts/ci/print-webclients-backup-paths-pattern.ts) # split paths string to array WEBCLIENT_DIST_PATHS=$(IFS=';';echo $WEBCLIENT_DIST_PATHS_STRING); diff --git a/scripts/ci/github/install-node-gyp.ps1 b/scripts/ci/github/install-node-gyp.ps1 new file mode 100644 index 000000000..3e5c5184f --- /dev/null +++ b/scripts/ci/github/install-node-gyp.ps1 @@ -0,0 +1,6 @@ +npm install -g node-gyp +npx node-gyp --verbose list +npx node-gyp --verbose install $(node -v) +npx node-gyp --verbose list +# TODO resolve "node-gyp" cache path or set it for "node-gyp" explicitly rather than hardcoding the value +ls "C:\Users\runneradmin\AppData\Local\node-gyp\Cache\$($(node -v).TrimStart('v'))\include\node" diff --git a/scripts/ci/github/install-node-modules.ps1 b/scripts/ci/github/install-node-modules.ps1 deleted file mode 100644 index 1c8666292..000000000 --- a/scripts/ci/github/install-node-modules.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -# running this script is omitted for the packaging job since cached dependencies used, see "github actions" config for details - -echo "::group::system setup" -./scripts/ci/github/system-setup.ps1 -IncludeWin81Sdk $true -npm config set msvs_version 2017 -echo "::endgroup::" - -echo "::group::install node modules" -yarn --pure-lockfile --network-timeout 60000 -echo "::endgroup::" diff --git a/scripts/ci/github/system-setup.ps1 b/scripts/ci/github/install-vs-build-tools.ps1 similarity index 100% rename from scripts/ci/github/system-setup.ps1 rename to scripts/ci/github/install-vs-build-tools.ps1 diff --git a/scripts/ci/github/package-app.ps1 b/scripts/ci/github/package-app.ps1 index 2865e2407..b9e3a9987 100644 --- a/scripts/ci/github/package-app.ps1 +++ b/scripts/ci/github/package-app.ps1 @@ -1,21 +1,14 @@ -# enable debug output printing for "electron-builder" / "node-gyp" like modules -$env:DEBUG = "true" - -echo "::group::system setup" -./scripts/ci/github/system-setup.ps1 -IncludeWin81Sdk $true +echo "::group::vs build tools setup" +./scripts/ci/github/install-vs-build-tools.ps1 -IncludeWin81Sdk $true npm config set msvs_version 2017 echo "::endgroup::" -echo "::group::build native modules (electron-rebuild)" -npm run postinstall:remove:prebuild-install -npm run clean:prebuilds -npm exec --package=electron-rebuild -- electron-rebuild --version $((Get-Content ./package.json | ConvertFrom-Json).devDependencies.electron) -echo "::endgroup::" - echo "::group::build native modules" npm run postinstall:remove:prebuild-install npm run clean:prebuilds +# $env:DEBUG = "*" npm exec --package=electron-builder -- electron-builder install-app-deps --arch=x64 +# $env:DEBUG = $null echo "::endgroup::" echo "::group::test:e2e" diff --git a/scripts/ci/print-webclients-dist-paths-pattern.ts b/scripts/ci/print-webclients-backup-paths-pattern.ts similarity index 100% rename from scripts/ci/print-webclients-dist-paths-pattern.ts rename to scripts/ci/print-webclients-backup-paths-pattern.ts diff --git a/scripts/prepare-webclient/index.ts b/scripts/prepare-webclient/index.ts index a44a67ce4..e6fddf7de 100644 --- a/scripts/prepare-webclient/index.ts +++ b/scripts/prepare-webclient/index.ts @@ -4,7 +4,7 @@ import pathIsInside from "path-is-inside"; import {CWD_ABSOLUTE_DIR} from "scripts/const"; import {buildProtonClients} from "scripts/prepare-webclient/protonmail"; import {catchTopLeventAsync} from "scripts/lib"; -import {resolveProtonMetadata} from "scripts/prepare-webclient/monaco-editor-dts"; +import {generateDtsForMonacoEditor} from "scripts/prepare-webclient/monaco-editor-dts"; const [, , appDestDir] = process.argv; @@ -18,5 +18,5 @@ if (!pathIsInside(path.resolve(CWD_ABSOLUTE_DIR, appDestDir), CWD_ABSOLUTE_DIR)) catchTopLeventAsync(async () => { await buildProtonClients({destDir: path.join(appDestDir, "./webclient")}); - await resolveProtonMetadata({destDir: path.join(appDestDir)}); + await generateDtsForMonacoEditor({destDir: appDestDir}); }); diff --git a/scripts/prepare-webclient/protonmail-lib.ts b/scripts/prepare-webclient/lib.ts similarity index 96% rename from scripts/prepare-webclient/protonmail-lib.ts rename to scripts/prepare-webclient/lib.ts index 7eb40aaf6..a6d8df14e 100644 --- a/scripts/prepare-webclient/protonmail-lib.ts +++ b/scripts/prepare-webclient/lib.ts @@ -148,29 +148,20 @@ export async function executeBuildFlow => { - if (fsExtra.pathExistsSync(path.resolve(repoDir, "node_modules"))) { - CONSOLE_LOG("Skip dependencies installing"); - } else if (flow.install) { - await flow.install(flowOptions); - } else { - await execShell(["npm", ["ci"], {cwd: repoDir}]); - } - }; - if (fsExtra.pathExistsSync(repoDistBackupDir)) { // taking dist from the backup const src = repoDistBackupDir; const dest = repoDistDir; CONSOLE_LOG(`Copying backup ${src} to ${dest}`); await fsExtra.copy(src, dest); - - if (repoType === "proton-mail") { - // installing modules since required by "./scripts/prepare-webclient/monaco-editor-dts.ts" - await installModules(); - } } else { // executing the build - await installModules(); + if (fsExtra.pathExistsSync(path.resolve(repoDir, "node_modules"))) { + CONSOLE_LOG("Skip dependencies installing"); + } else if (flow.install) { + await flow.install(flowOptions); + } else { + await execShell(["npm", ["ci"], {cwd: repoDir}]); + } if (shouldFailOnBuild) { throw new Error(`Halting since "${shouldFailOnBuildEnvVarName}" env var has been enabled`); diff --git a/scripts/prepare-webclient/monaco-editor-dts.ts b/scripts/prepare-webclient/monaco-editor-dts.ts index b560b4560..49ed6a679 100644 --- a/scripts/prepare-webclient/monaco-editor-dts.ts +++ b/scripts/prepare-webclient/monaco-editor-dts.ts @@ -3,7 +3,6 @@ import fsExtra from "fs-extra"; import path from "path"; import {CONSOLE_LOG} from "scripts/lib"; -import {IpcMainServiceScan} from "src/shared/api/main"; import {PROTON_MONACO_EDITOR_DTS_ASSETS_LOCATION} from "src/shared/constants"; import {PROTON_SHARED_MESSAGE_INTERFACE} from "src/shared/proton-apps-constants"; @@ -12,9 +11,9 @@ export const dtsGenerator: { // eslint-disable-line @typescript-eslint/no-unsafe default: (options: { baseDir: string, files: string[], out: string }) => Promise } = require("dts-generator"); // eslint-disable-line @typescript-eslint/no-var-requires -export const resolveProtonMetadata = async ( +export const generateDtsForMonacoEditor = async ( {destDir}: { destDir: string }, -): Promise => { +): Promise => { const options = { system: { base: "./node_modules/typescript/lib", @@ -30,16 +29,25 @@ export const resolveProtonMetadata = async ( // TODO replace "dts-generator" dependency with something capable to combine "./node_modules/typescript/lib/lib.esnext.d.ts" for (const key of [/* "system", */ "protonMessage"] as const) { - const sourceFile = options[key].in; + const {in: sourceFile, out: destFile, base: baseDir} = options[key]; + if (fsExtra.pathExistsSync(destFile)) { + CONSOLE_LOG(`The "${destFile}" file already exists.`); + continue; + } if (!fsExtra.pathExistsSync(sourceFile)) { throw new Error(`The source "${sourceFile}" file doesn't exits.`); } - await dtsGenerator.default({baseDir: options[key].base, files: [sourceFile], out: options[key].out}); - CONSOLE_LOG(`Merged "${options[key].in}" to "${options[key].out}"`); + await dtsGenerator.default({baseDir, files: [sourceFile], out: destFile}); + CONSOLE_LOG(`Merged "${sourceFile}" to "${destFile}"`); } // TODO drop custom "./node_modules/typescript/lib/lib.esnext.d.ts" combining when "dts-generator" gets replaced { + const {in: sourceFile, out: destFile} = options.system; + if (fsExtra.pathExistsSync(destFile)) { + CONSOLE_LOG(`The "${destFile}" file already exists.`); + return; + } const referenceTagRe = /\/\/\/[\s\t]+/; const mergedFiles: string[] = []; const extractContent = (file: string): string => { @@ -59,23 +67,8 @@ export const resolveProtonMetadata = async ( mergedFiles.push(file); return resultLines.join("\n"); }; - fsExtra.ensureDirSync(path.dirname(options.system.out)); - fs.writeFileSync(options.system.out, extractContent(options.system.in)); - CONSOLE_LOG(`Merged to "${options.system.out}" files:`, mergedFiles); + fsExtra.ensureDirSync(path.dirname(destFile)); + fs.writeFileSync(destFile, extractContent(sourceFile)); + CONSOLE_LOG(`Merged to "${destFile}" files:`, mergedFiles); } - - return { - system: [ - fs.readFileSync(options.system.out).toString(), - `in-memory:${options.system.in}`, - ], - protonMessage: [ - ( - fs.readFileSync(options.protonMessage.out).toString() - + - `declare const mail: Omit & {Body: string};` - ), - `in-memory:${PROTON_SHARED_MESSAGE_INTERFACE.url}`, - ], - }; }; diff --git a/scripts/prepare-webclient/protonmail.ts b/scripts/prepare-webclient/protonmail.ts index a09e9ae94..b2e71b8fb 100644 --- a/scripts/prepare-webclient/protonmail.ts +++ b/scripts/prepare-webclient/protonmail.ts @@ -3,7 +3,7 @@ import path from "path"; import {BINARY_NAME} from "src/shared/constants"; import {CWD_ABSOLUTE_DIR} from "scripts/const"; -import {FolderAsDomainEntry, executeBuildFlow, printAndWriteFile} from "./protonmail-lib"; +import {FolderAsDomainEntry, executeBuildFlow, printAndWriteFile} from "./lib"; import {PROVIDER_REPO_MAP, PROVIDER_REPO_NAMES} from "src/shared/proton-apps-constants"; import {applyPatch, execShell} from "scripts/lib";