From c4268cfd23339e4143a0db4491e7ed8915278a09 Mon Sep 17 00:00:00 2001 From: Dane Pilcher Date: Mon, 16 Oct 2023 11:32:44 -0600 Subject: [PATCH] ci: unit test on windows --- .codebuild/build_windows.yml | 1 + .codebuild/e2e_workflow.yml | 8 + .codebuild/e2e_workflow_base.yml | 8 + .codebuild/pr_workflow.yml | 8 + .codebuild/release_workflow.yml | 8 + .codebuild/scripts-windows/load-e2e-cache.sh | 14 ++ .../scripts-windows/rename-packaged-cli.sh | 5 + .codebuild/scripts-windows/run-e2e-windows.sh | 24 +++ .../scripts-windows/save-build-windows.sh | 6 + .../scripts-windows/shared-scripts-windows.sh | 138 ++++++++++++++++++ .codebuild/scripts-windows/wait-windows.sh | 12 ++ .codebuild/test_windows.yml | 11 ++ 12 files changed, 243 insertions(+) create mode 100644 .codebuild/scripts-windows/load-e2e-cache.sh create mode 100644 .codebuild/scripts-windows/rename-packaged-cli.sh create mode 100644 .codebuild/scripts-windows/run-e2e-windows.sh create mode 100644 .codebuild/scripts-windows/save-build-windows.sh create mode 100644 .codebuild/scripts-windows/shared-scripts-windows.sh create mode 100644 .codebuild/scripts-windows/wait-windows.sh create mode 100644 .codebuild/test_windows.yml diff --git a/.codebuild/build_windows.yml b/.codebuild/build_windows.yml index a79f4a825..9ac7a0c72 100644 --- a/.codebuild/build_windows.yml +++ b/.codebuild/build_windows.yml @@ -5,6 +5,7 @@ phases: build: commands: - yarn run production-build + - bash ./codebuild_specs/scripts-windows/save-build-windows.sh artifacts: files: diff --git a/.codebuild/e2e_workflow.yml b/.codebuild/e2e_workflow.yml index e75a50bb1..1c420534c 100644 --- a/.codebuild/e2e_workflow.yml +++ b/.codebuild/e2e_workflow.yml @@ -16,6 +16,14 @@ batch: type: WINDOWS_SERVER_2019_CONTAINER compute-type: BUILD_GENERAL1_LARGE image: $WINDOWS_IMAGE_2019 + - identifier: test_windows + buildspec: .codebuild/test_windows.yml + env: + type: WINDOWS_SERVER_2019_CONTAINER + compute-type: BUILD_GENERAL1_LARGE + image: $WINDOWS_IMAGE_2019 + depend-on: + - build_windows - identifier: test buildspec: .codebuild/test.yml env: diff --git a/.codebuild/e2e_workflow_base.yml b/.codebuild/e2e_workflow_base.yml index 3e33fd18d..538530760 100644 --- a/.codebuild/e2e_workflow_base.yml +++ b/.codebuild/e2e_workflow_base.yml @@ -16,6 +16,14 @@ batch: type: WINDOWS_SERVER_2019_CONTAINER compute-type: BUILD_GENERAL1_LARGE image: $WINDOWS_IMAGE_2019 + - identifier: test_windows + buildspec: .codebuild/test_windows.yml + env: + type: WINDOWS_SERVER_2019_CONTAINER + compute-type: BUILD_GENERAL1_LARGE + image: $WINDOWS_IMAGE_2019 + depend-on: + - build_windows - identifier: test buildspec: .codebuild/test.yml env: diff --git a/.codebuild/pr_workflow.yml b/.codebuild/pr_workflow.yml index 56eb3ff7e..ebaa670fc 100644 --- a/.codebuild/pr_workflow.yml +++ b/.codebuild/pr_workflow.yml @@ -14,6 +14,14 @@ batch: type: WINDOWS_SERVER_2019_CONTAINER compute-type: BUILD_GENERAL1_LARGE image: $WINDOWS_IMAGE_2019 + - identifier: test_windows + buildspec: .codebuild/test_windows.yml + env: + type: WINDOWS_SERVER_2019_CONTAINER + compute-type: BUILD_GENERAL1_LARGE + image: $WINDOWS_IMAGE_2019 + depend-on: + - build_windows - identifier: test buildspec: .codebuild/test.yml depend-on: diff --git a/.codebuild/release_workflow.yml b/.codebuild/release_workflow.yml index e7d6b7857..50ce2d11e 100644 --- a/.codebuild/release_workflow.yml +++ b/.codebuild/release_workflow.yml @@ -14,6 +14,14 @@ batch: type: WINDOWS_SERVER_2019_CONTAINER compute-type: BUILD_GENERAL1_LARGE image: $WINDOWS_IMAGE_2019 + - identifier: test_windows + buildspec: .codebuild/test_windows.yml + env: + type: WINDOWS_SERVER_2019_CONTAINER + compute-type: BUILD_GENERAL1_LARGE + image: $WINDOWS_IMAGE_2019 + depend-on: + - build_windows - identifier: test buildspec: .codebuild/test.yml depend-on: diff --git a/.codebuild/scripts-windows/load-e2e-cache.sh b/.codebuild/scripts-windows/load-e2e-cache.sh new file mode 100644 index 000000000..593619b76 --- /dev/null +++ b/.codebuild/scripts-windows/load-e2e-cache.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +source ./codebuild_specs/scripts-windows/shared-scripts-windows.sh + +# export CODEBUILD_SOURCE_VERSION=e1dce3fd6083503048f8d6eec3c09cf1b5d8a007 +loadCache repo-windows $CODEBUILD_SRC_DIR +loadCache .cache-windows $HOME/AppData/Local/Yarn/Cache/v6 + +loadCache verdaccio-cache $CODEBUILD_SRC_DIR/../verdaccio-cache +loadCache all-binaries $CODEBUILD_SRC_DIR/out +loadCacheFile .amplify-pkg-version $CODEBUILD_SRC_DIR/.amplify-pkg-version +loadCacheFile UNIFIED_CHANGELOG.md $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md + +ls $CODEBUILD_SRC_DIR diff --git a/.codebuild/scripts-windows/rename-packaged-cli.sh b/.codebuild/scripts-windows/rename-packaged-cli.sh new file mode 100644 index 000000000..b519fdd04 --- /dev/null +++ b/.codebuild/scripts-windows/rename-packaged-cli.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source ./codebuild_specs/scripts-windows/shared-scripts-windows.sh + +_install_packaged_cli_win diff --git a/.codebuild/scripts-windows/run-e2e-windows.sh b/.codebuild/scripts-windows/run-e2e-windows.sh new file mode 100644 index 000000000..0e1304e59 --- /dev/null +++ b/.codebuild/scripts-windows/run-e2e-windows.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# set exit on error to true +set -e + +export MSYS_NO_PATHCONV=1 +export MSYS2_ARG_CONV_EXCL="*" + +export AMPLIFY_DIR=$CODEBUILD_SRC_DIR\\out +export AMPLIFY_PATH=$CODEBUILD_SRC_DIR\\out\\amplify.exe +export NODE_OPTIONS=--max-old-space-size=5120 + +source .circleci/local_publish_helpers_codebuild.sh +source ./codebuild_specs/scripts-windows/shared-scripts-windows.sh + +# source $BASH_ENV + +amplify version + +cd packages/amplify-e2e-tests + +_loadTestAccountCredentials + +retry runE2eTestCb diff --git a/.codebuild/scripts-windows/save-build-windows.sh b/.codebuild/scripts-windows/save-build-windows.sh new file mode 100644 index 000000000..3daceeea5 --- /dev/null +++ b/.codebuild/scripts-windows/save-build-windows.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# set exit on error to true +set -e + +source ./codebuild_specs/scripts-windows/shared-scripts-windows.sh && _saveBuild diff --git a/.codebuild/scripts-windows/shared-scripts-windows.sh b/.codebuild/scripts-windows/shared-scripts-windows.sh new file mode 100644 index 000000000..ae0100663 --- /dev/null +++ b/.codebuild/scripts-windows/shared-scripts-windows.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +# set exit on error to true +set -e + +# The flags address the issue here: https://github.com/boto/botocore/issues/1716 +export MSYS_NO_PATHCONV=1 +export MSYS2_ARG_CONV_EXCL="*" + +# We have custom caching for our CodeBuild pipelines +# which allows us to share caches with jobs in the same batch + +# storeCache +function storeCache { + localPath="$1" + alias="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo writing cache to $s3Path + # zip contents and upload to s3 + # if ! (cd $localPath && tar -czf cache.tar . && ls && aws s3 cp cache.tar $s3Path); then + # echo Something went wrong storing the cache. + # fi + if ! (cd $localPath && tar -czf cache.tar . && ls && aws s3 cp cache.tar $s3Path); then + echo Something went wrong storing the cache. + fi + echo done writing cache + cd $CODEBUILD_SRC_DIR +} +function storeCacheFile { + localFilePath="$1" + alias="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo writing cache to $s3Path + # zip contents and upload to s3 + if ! (aws s3 cp $localFilePath $s3Path); then + echo Something went wrong storing the cache. + fi + echo done writing cache + cd $CODEBUILD_SRC_DIR +} +# loadCache +function loadCache { + alias="$1" + localPath="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo loading cache from $s3Path + # create directory if it doesn't exist yet + mkdir -p $localPath + # check if cache exists in s3 + if ! aws s3 ls $s3Path > /dev/null; then + echo "Cache not found." + exit 0 + fi + # tar --help + # load cache and unzip it + # if ! (cd $localPath && aws s3 cp $s3Path cache.tar && ls && tar -xzkf cache.tar); then + # echo "Something went wrong fetching the cache. Continuing anyway." + # fi + if ! (cd $localPath && aws s3 cp $s3Path - | tar xzkf -); then + echo "Something went wrong fetching the cache. Continuing anyway." + fi + echo done loading cache + cd $CODEBUILD_SRC_DIR +} +function loadCacheFile { + alias="$1" + localFilePath="$2" + s3Path="s3://$CACHE_BUCKET_NAME/$CODEBUILD_SOURCE_VERSION/$alias" + echo loading cache file from $s3Path + # check if cache file exists in s3 + if ! aws s3 ls $s3Path > /dev/null; then + echo "Cache file not found." + exit 0 + fi + # load cache file + if ! (aws s3 cp $s3Path $localFilePath); then + echo "Something went wrong fetching the cache file. Continuing anyway." + fi + echo done loading cache + cd $CODEBUILD_SRC_DIR +} +function _loadTestAccountCredentials { + echo ASSUMING PARENT TEST ACCOUNT credentials + session_id=$((1 + $RANDOM % 10000)) + creds=$(aws sts assume-role --role-arn $TEST_ACCOUNT_ROLE --role-session-name testSession${session_id} --duration-seconds 3600) + if [ -z $(echo $creds | jq -c -r '.AssumedRoleUser.Arn') ]; then + echo "Unable to assume parent e2e account role." + return + fi + echo "Using account credentials for $(echo $creds | jq -c -r '.AssumedRoleUser.Arn')" + export AWS_ACCESS_KEY_ID=$(echo $creds | jq -c -r ".Credentials.AccessKeyId") + export AWS_SECRET_ACCESS_KEY=$(echo $creds | jq -c -r ".Credentials.SecretAccessKey") + export AWS_SESSION_TOKEN=$(echo $creds | jq -c -r ".Credentials.SessionToken") +} + + + +function _lsOut { + ls .. + ls ~ + ls $HOME + ls $HOME/.. + ls $HOME/../.. +} +function _build { + echo Windows Build + yarn run production-build + yarn build-tests +} +function _saveBuild { + _lsOut + storeCache $CODEBUILD_SRC_DIR repo-windows + storeCache $HOME/AppData/Local/Yarn/Cache/v6 .cache-windows +} +function _install_packaged_cli_win { + echo Install Amplify Packaged CLI to PATH + # rename the command to amplify + cd $CODEBUILD_SRC_DIR/out + cp amplify-pkg-win-x64.exe amplify.exe + + echo Move to CLI Binary to already existing PATH + # This is a Hack to make sure the Amplify CLI is in the PATH + + cp $CODEBUILD_SRC_DIR/out/amplify.exe $HOME/AppData/Local/Microsoft/WindowsApps + ls $HOME/AppData/Local/Microsoft/WindowsApps + + # reset working directory + cd $CODEBUILD_SRC_DIR +} + + +function _scanArtifacts { + if ! yarn ts-node .circleci/scan_artifacts_codebuild.ts; then + echo "Cleaning the repository" + git clean -fdx + exit 1 + fi +} diff --git a/.codebuild/scripts-windows/wait-windows.sh b/.codebuild/scripts-windows/wait-windows.sh new file mode 100644 index 000000000..e11087209 --- /dev/null +++ b/.codebuild/scripts-windows/wait-windows.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# set exit on error to true +set -e + +export MSYS_NO_PATHCONV=1 +export MSYS2_ARG_CONV_EXCL="*" + +export NODE_OPTIONS=--max-old-space-size=4096 + +source shared-scripts.sh +_waitForJobs $CODEBUILD_SRC_DIR/codebuild_specs/wait_windows.json requirePrevJobsToSucceed diff --git a/.codebuild/test_windows.yml b/.codebuild/test_windows.yml new file mode 100644 index 000000000..652871698 --- /dev/null +++ b/.codebuild/test_windows.yml @@ -0,0 +1,11 @@ +version: 0.2 +env: + shell: powershell.exe +phases: + build: + commands: + - bash ./codebuild_specs/scripts-windows/load-e2e-cache.sh + - yarn test-ci +artifacts: + files: + - 'shared-scripts.sh'