From b563b57f80b9ce9b92e9e4b1483c22d52a943b92 Mon Sep 17 00:00:00 2001 From: Marcin Maciaszczyk Date: Thu, 19 Dec 2024 15:24:50 +0100 Subject: [PATCH] test: `plural up` tests (#574) * retry adding plural to path * fix make targets name conflict * upsert domain if workspace.yml already exists * ensure branch * ensure branch * use template-branch for bootstrap * template new dir * setup gcloud in workflow * add jobs for all providers * simplify workflow * use built-in terraform binary * add gcloud setup/teardown * azure support * add crypto init and readme * azure teardown * git teardown * update makefile * add required arg check to test * update makefile * azure login * azure login auth type * update makefile * setup venom * update action * setup venom * add plural key variable * initial test setup * get timestamp * add docs to variables * get timestamp * get timestamp * get timestamp * get timestamp * get timestamp * get timestamp * update venom args * use common env vars where possible * fix plural up and improve required checks * fix skip * fix provider skip * fix skip * update azure region * add commit message to plural up * decode private key into a file * retry azure auth * run tests in ci * update testout path * bump venom version * update decode * upload log * run in debug mode * test * chmod * update actions * add workflow_dispatch to action * rename privateKeyPath to gitRepoPrivateKeyPath export common env vars * update gcp env vars * remove unused env var * fix private key embedding into context.yaml * replace default git key * fix ssh key setup * fix ssh key setup * fix git teardown * fix git teardown * start ssh-agent * fix typo * create plural dir * create plural dir * fix artifact upload * use custom step failure handling in tests * do not stop on all failures * fix exit code assertion configuration * refactor * refactor * update make e2e * do not ask for confirmation when deleting resource group * do not ask for confirmation when deleting resource group * rename key * artifact upload * fix azure storage account var * refactor action * update agent command * update step name * update tf creds * update gcp e2e ci * update tf creds * disable tenant id * retry with msi * test tf * test tf * test tf * test tf * test tf * test tf * test tf * bump aks module version * bump azidentity * setup slack ci action * update slack configuration * retest tf * update slack configuration * retest tf * retest tf * retest tf * retest tf * update slack configuration * retest tf * add az-test bin * retest tf * retest tf * add debug * update slack configuration * retest tf * decrease venom log lvl for gcp * retest tf * retest tf * add gcloud auth step * retest tf * fix gcp auth * remove debug * fix gcp auth * use google federated credentials * use google federated credentials * use google federated credentials * retest tf * cleanup gcp venom setup * upload only html report * post status only on slack * debug gcp e2e * debug gcp e2e * skip azure * skip azure * enable gcp * enable gcp * fix gcp e2e * update slack status * reenable slack upload for gcp * update slack status * update slack status * add timestamp * add timestamp * remove secrets block from venom * remove actor * enable azure * hide az outputs * restore log * attempt to fix azapi auth * add plural key secret to venom * attempt to fix azapi auth * attempt to fix azapi auth * update slack file handling * add export kubeconfig command * add test for deployment services * export CONSOLE_ACCESS_TOKEN * fix plural cd login * fix plural cd login * improve get kubeconfig * improve plural cd login * add kubeconfig dir * create only kube dir * check kubeconfig command * install gke-gcloud-auth-plugin * check az get-credentials * run all tests * add provider's kubeconfigs * fix nit * add retry for services * add aws workspace * fix checking services * add service tests * increase retry * increase retry and delay * gen mocks, fix linter * check service condition * add aws-nuke config * add github job and test aws * remove login aws * add aws login * add aws login * create and use profile * tune aws-nuke * add plural down step * ignore os file * tune aws-nuke * run plural down only for azure * tune aws-nuke * tune aws-nuke * tune aws-nuke * add profile * enable all providers * post-merge fix * final adjustments * rerun gcp e2e * set plural up to main branch by default --------- Co-authored-by: Lukasz Zajaczkowski Co-authored-by: Sebastian Florek Co-authored-by: test-cli-e2e-gcp Co-authored-by: test-cli-e2e-aws --- .DS_Store | Bin 0 -> 8196 bytes .github/workflows/e2e.yaml | 354 +++++++++++++++++++++++++++ .gitignore | 4 + Makefile | 50 +++- cmd/command/cd/cd_repositories.go | 35 +++ cmd/command/cd/cd_services.go | 8 +- go.mod | 4 +- go.sum | 8 +- hack/include/tools.mk | 16 +- nuke-config.yml | 128 ++++++++++ pkg/client/plural.go | 33 ++- pkg/console/console.go | 1 + pkg/console/repositories.go | 9 + pkg/test/mocks/ConsoleClient.go | 30 +++ pkg/up/context.go | 3 +- pkg/up/generate.go | 5 +- pkg/utils/jsonpath.go | 54 ++++ test.tf | 59 +++++ test/plural/lib/aws-teardown.yaml | 9 + test/plural/lib/azure-teardown.yml | 9 + test/plural/lib/check-required.yml | 81 ++++++ test/plural/lib/cluster-services.yml | 42 ++++ test/plural/lib/cluster-setup.yml | 16 ++ test/plural/lib/context-setup.yml | 23 ++ test/plural/lib/gcloud-setup.yml | 23 ++ test/plural/lib/gcloud-teardown.yml | 10 + test/plural/lib/git-setup.yml | 20 ++ test/plural/lib/git-teardown.yml | 12 + test/plural/lib/plural-login.yml | 30 +++ test/plural/lib/workspace-setup.yml | 74 ++++++ test/plural/up.yml | 156 ++++++++++++ 31 files changed, 1283 insertions(+), 23 deletions(-) create mode 100644 .DS_Store create mode 100644 .github/workflows/e2e.yaml create mode 100644 nuke-config.yml create mode 100644 pkg/utils/jsonpath.go create mode 100644 test.tf create mode 100644 test/plural/lib/aws-teardown.yaml create mode 100644 test/plural/lib/azure-teardown.yml create mode 100644 test/plural/lib/check-required.yml create mode 100644 test/plural/lib/cluster-services.yml create mode 100644 test/plural/lib/cluster-setup.yml create mode 100644 test/plural/lib/context-setup.yml create mode 100644 test/plural/lib/gcloud-setup.yml create mode 100644 test/plural/lib/gcloud-teardown.yml create mode 100644 test/plural/lib/git-setup.yml create mode 100644 test/plural/lib/git-teardown.yml create mode 100644 test/plural/lib/plural-login.yml create mode 100644 test/plural/lib/workspace-setup.yml create mode 100644 test/plural/up.yml diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..9928f224eac0855bbffe66ee173bde4f61202666 GIT binary patch literal 8196 zcmeHM&2AGh5dNG*ZAb*FIQ4?G95^--Z;-ahp%O)iK0qNqq@|l|luc>5<=%JT6%Y@@ z!*GIc#wOTKaza8#Ye)8McfZfG{${+Z*#a;px1*Q9A%G^k*n>8QElsXV-)J>=M#zSI zpo0=uXL&Ik*V&b|mJVa-FwV1a z6^c>sn_c1`h-_oz&ivh35W;$l?{M5!HppGG?7}+X+jrtm) zo?^rqhq%EQ3moyNAZyOCqo@e z!-SqXyr&{wS#7cdTjCbeaNgSY2;0CF-irz$u6`Qf_VEg5I7J_?ae;n>{D|pbD|{1?nK7qY3(gzI zTq5CP+dh`-%WVdP9#lV{^*mr*uJgHMPK(z$F4-44*O(b1_aa$1(yMpWGEz~a?t6Ig z_a5kP&X`4NY0mhbT6ptEtsODro-|Lq7(c>W=FTzq%Q>y&@s|A=FdnM{Gb-|l`YctA zkKCIDW}(&+^MH9k)RLS9$0Oa%B}Xc#`MCc6_xs6Um<<8HLEe-RxR3lW69QZdcl08A zzT}Fw-kU@z;IzpZlVP7W65%E8u?#(l;sJ{(m|9YxathS#iVvm!@9+NpUpZaUf|CMO zQouBN7rhg%*wxk>YnNI(V}HjkO>iq~mBK;Gaat|MY1jWS> $GITHUB_ENV + - name: Setup test repository SSH key + run: | + mkdir -p ${{ env.SSH_PATH }} + (base64 -d <<< ${{ secrets.E2E_REPO_PRIVATE_KEY }}) > ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + chmod 600 ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + eval "$(ssh-agent -s)" + ssh-add ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + - name: Install aws-nuke + run: | + wget -c https://github.com/ekristen/aws-nuke/releases/download/v3.34.0/aws-nuke-v3.34.0-linux-amd64.tar.gz -O - | tar -xz -C /usr/local/bin + chmod +x ${{ env.AWS_NUKE_PATH }} + aws-nuke version + - name: Setup Go + uses: actions/setup-go@v4.1.0 + with: + go-version-file: go.mod + - name: Add GOBIN to PATH + run: echo $GOBIN >> $GITHUB_PATH + - name: Setup Venom + run: | + curl https://github.com/ovh/venom/releases/download/v1.2.0/venom.linux-amd64 -L -o ${{ env.VENOM_PATH }} + chmod +x ${{ env.VENOM_PATH }} + venom version + - name: Setup Plural CLI + run: | + make install-cli + mkdir -p ${{ env.VENOM_VAR_pluralHome }} + plural version + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + - name: Print AWS CLI version + run: aws --version + - name: Create kube directory + run: | + mkdir -p $HOME/.kube + touch $HOME/.kube/config + chmod 755 $HOME/.kube/config + - name: Run tests + env: + VENOM_VAR_provider: aws + VENOM_VAR_region: eu-west-1 + VENOM_VAR_awsZoneA: eu-west-1a + VENOM_VAR_awsZoneB: eu-west-1b + VENOM_VAR_awsZoneC: eu-west-1c + VENOM_VAR_awsProject: ${{ secrets.E2E_AWS_PROJECT_ID }} + VENOM_VAR_awsBucket: e2e-tf-state-${{ env.TIMESTAMP }} + VENOM_VAR_project: ${{ secrets.E2E_AWS_PROJECT_ID }} + VENOM_VAR_branch: e2e-${{ env.TIMESTAMP }}-aws + VENOM_VAR_username: ${{ secrets.E2E_AWS_SA_USERNAME }} + VENOM_VAR_email: ${{ secrets.E2E_AWS_SA_EMAIL }} + VENOM_VAR_token: ${{ secrets.E2E_AWS_SA_TOKEN }} + TF_VAR_deletion_protection: false + AWS_PROFILE: aws-nuke + run: venom run -vv --html-report --format=json --output-dir ${{ env.TESTOUT_PATH }} test/plural + - name: Post status on Slack + id: slack_message + if: always() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook-type: incoming-webhook + webhook: ${{ secrets.SLACK_WEBHOOK }} + payload: | + blocks: + - type: section + text: + type: mrkdwn + text: "${{ github.workflow }} workflow finished." + - type: section + fields: + - type: mrkdwn + text: "*Repository*\n" + - type: mrkdwn + text: "*Job*\n`${{ github.job }}`" + - type: mrkdwn + text: "*Status*\n`${{ job.status }}`" + - type: mrkdwn + text: "*Pull request*\n<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>" + - name: Upload artifacts to Slack on failure + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + method: files.uploadV2 + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel_id: ${{ secrets.SLACK_CHANNEL_ID }} + thread_ts: "${{ steps.slack_message.outputs.ts }}" + file_uploads: + - file: ${{ env.TESTOUT_PATH }}/venom.log + filename: venom.log + - file: ${{ env.TESTOUT_PATH }}/test_results.html + filename: rest_results.html + plural-up-gcp: + name: plural up / GCP + permissions: + contents: 'read' + id-token: 'write' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Store test timestamp + run: echo "TIMESTAMP=$(date +'%s')" >> $GITHUB_ENV + - name: Setup test repository SSH key + run: | + mkdir -p ${{ env.SSH_PATH }} + (base64 -d <<< ${{ secrets.E2E_REPO_PRIVATE_KEY }}) > ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + chmod 600 ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + eval "$(ssh-agent -s)" + ssh-add ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + - name: GCloud Auth + uses: google-github-actions/auth@v2 + with: + service_account: ${{ secrets.E2E_GCP_EMAIL }} + workload_identity_provider: projects/657418122889/locations/global/workloadIdentityPools/github/providers/github + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + with: + version: '>= 363.0.0' + - name: Setup Go + uses: actions/setup-go@v4.1.0 + with: + go-version-file: go.mod + - name: Add GOBIN to PATH + run: echo $GOBIN >> $GITHUB_PATH + - name: Setup Venom + run: | + curl https://github.com/ovh/venom/releases/download/v1.2.0/venom.linux-amd64 -L -o ${{ env.VENOM_PATH }} + chmod +x ${{ env.VENOM_PATH }} + venom version + - name: Setup Plural CLI + run: | + make install-cli + mkdir -p ${{ env.VENOM_VAR_pluralHome }} + plural version + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + - name: Print Google Cloud CLI version + run: gcloud --version + - name: Create kube directory + run: | + mkdir -p $HOME/.kube + touch $HOME/.kube/config + chmod 755 $HOME/.kube/config + - name: Run tests + env: + VENOM_VAR_provider: gcp + VENOM_VAR_region: us-central1 + VENOM_VAR_gcpOrgID: ${{ secrets.E2E_GCP_ORG_ID }} + VENOM_VAR_gcpBillingID: ${{ secrets.E2E_GCP_BILLING_ID }} + VENOM_VAR_project: e2e-${{ env.TIMESTAMP }} + VENOM_VAR_branch: e2e-${{ env.TIMESTAMP }}-gcp + VENOM_VAR_username: ${{ secrets.E2E_GCP_SA_USERNAME }} + VENOM_VAR_email: ${{ secrets.E2E_GCP_SA_EMAIL }} + VENOM_VAR_token: ${{ secrets.E2E_GCP_SA_TOKEN }} + TF_VAR_network: plural-e2e-network-${{ env.TIMESTAMP }} + TF_VAR_subnetwork: plural-e2e-subnet-${{ env.TIMESTAMP }} + run: venom run -vv --html-report --format=json --output-dir ${{ env.TESTOUT_PATH }} test/plural + - name: Post status on Slack + id: slack_message + if: always() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook-type: incoming-webhook + webhook: ${{ secrets.SLACK_WEBHOOK }} + payload: | + blocks: + - type: section + text: + type: mrkdwn + text: "${{ github.workflow }} workflow finished." + - type: section + fields: + - type: mrkdwn + text: "*Repository*\n" + - type: mrkdwn + text: "*Job*\n`${{ github.job }}`" + - type: mrkdwn + text: "*Status*\n`${{ job.status }}`" + - type: mrkdwn + text: "*Pull request*\n<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>" + - name: Upload artifacts to Slack on failure + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + method: files.uploadV2 + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel_id: ${{ secrets.SLACK_CHANNEL_ID }} + thread_ts: "${{ steps.slack_message.outputs.ts }}" + file_uploads: + - file: ${{ env.TESTOUT_PATH }}/venom.log + filename: venom.log + - file: ${{ env.TESTOUT_PATH }}/test_results.html + filename: rest_results.html + plural-up-azure: + name: plural up / Azure + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Store test timestamp + run: echo "TIMESTAMP=$(date +'%s')" >> $GITHUB_ENV + - name: Setup test repository SSH key + run: | + mkdir -p ${{ env.SSH_PATH }} + (base64 -d <<< ${{ secrets.E2E_REPO_PRIVATE_KEY }}) >> ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + chmod 600 ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + eval "$(ssh-agent -s)" + ssh-add ${{ env.VENOM_VAR_gitRepoPrivateKeyPath }} + - name: Setup Go + uses: actions/setup-go@v4.1.0 + with: + go-version-file: go.mod + - name: Add GOBIN to PATH + run: echo $GOBIN >> $GITHUB_PATH + - name: Setup Venom + run: | + curl https://github.com/ovh/venom/releases/download/v1.2.0/venom.linux-amd64 -L -o ${{ env.VENOM_PATH }} + chmod +x ${{ env.VENOM_PATH }} + venom version + - name: Setup Plural CLI + run: | + make install-cli + mkdir -p ${{ env.VENOM_VAR_pluralHome }} + plural version + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + - name: Print Azure CLI version + run: az --version + - name: Create kube directory + run: | + mkdir -p $HOME/.kube + touch $HOME/.kube/config + chmod 755 $HOME/.kube/config + - name: Login to Azure + uses: azure/login@v2 + with: + client-id: ${{ secrets.E2E_AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.E2E_AZURE_TENANT_ID }} + subscription-id: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }} + - name: Run tests + env: + ARM_USE_AKS_WORKLOAD_IDENTITY: true + ARM_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }} + ARM_CLIENT_ID: ${{ secrets.E2E_AZURE_CLIENT_ID }} + ARM_TENANT_ID: ${{ secrets.E2E_AZURE_TENANT_ID }} + VENOM_VAR_provider: azure + VENOM_VAR_region: polandcentral + VENOM_VAR_azureTenantId: ${{ secrets.E2E_AZURE_TENANT_ID }} + VENOM_VAR_azureSubscriptionId: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }} + VENOM_VAR_azureStorageAccount: e2e${{ env.TIMESTAMP }} + VENOM_VAR_project: e2e-${{ env.TIMESTAMP }} + VENOM_VAR_branch: e2e-${{ env.TIMESTAMP }}-azure + VENOM_VAR_username: ${{ secrets.E2E_AZURE_SA_USERNAME }} + VENOM_VAR_email: ${{ secrets.E2E_AZURE_SA_EMAIL }} + VENOM_VAR_token: ${{ secrets.E2E_AZURE_SA_TOKEN }} + run: | + # Create resource group and storage account + az group create --name ${{ env.VENOM_VAR_project }} --location ${{ env.VENOM_VAR_region }} --output none + az storage account create --name ${{ env.VENOM_VAR_azureStorageAccount }} --resource-group ${{ env.VENOM_VAR_project }} --location ${{ env.VENOM_VAR_region }} --sku Standard_LRS --kind StorageV2 --output none + + # Export access key that is required to authenticate to Terraform azurerm backend + export ARM_ACCESS_KEY=$(az storage account keys list --resource-group ${{ env.VENOM_VAR_project }} --account-name ${{ env.VENOM_VAR_azureStorageAccount }} --query '[0].value' -o tsv) + + # Run tests + venom run -vv --html-report --format=json --output-dir ${{ env.TESTOUT_PATH }} test/plural + - name: Post status on Slack + if: always() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook-type: incoming-webhook + webhook: ${{ secrets.SLACK_WEBHOOK }} + payload: | + blocks: + - type: section + text: + type: mrkdwn + text: "${{ github.workflow }} workflow finished." + - type: section + fields: + - type: mrkdwn + text: "*Repository*\n" + - type: mrkdwn + text: "*Job*\n`${{ github.job }}`" + - type: mrkdwn + text: "*Status*\n`${{ job.status }}`" + - type: mrkdwn + text: "*Pull request*\n<${{ github.event.pull_request.html_url }}|${{ github.event.pull_request.title }}>" + - name: Upload artifacts to Slack on failure + id: slack_message + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + method: files.uploadV2 + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel_id: ${{ secrets.SLACK_CHANNEL_ID }} + thread_ts: "${{ steps.slack_message.outputs.ts }}" + file_uploads: + - file: ${{ env.TESTOUT_PATH }}/venom.log + filename: venom.log + - file: ${{ env.TESTOUT_PATH }}/test_results.html + filename: rest_results.html diff --git a/.gitignore b/.gitignore index b284f7980..83bc48fe2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .idea/ .vscode/ +# OS files +.DS_Store + # Testing files context.yaml workspace.yaml @@ -9,6 +12,7 @@ workspace.yaml # Build files and binaries dist/ build/ +testout/ *.exe *.exe~ *.dll diff --git a/Makefile b/Makefile index 21389767e..91d15f8dd 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ APP_NAME ?= plural-cli APP_VSN ?= $(shell git describe --tags --always --dirty) APP_DATE ?= $(shell date -u +"%Y-%m-%dT%H:%M:%S%z") BUILD ?= $(shell git rev-parse --short HEAD) +TIMESTAMP ?= $(shell date +%s) DKR_HOST ?= dkr.plural.sh GOOS ?= darwin GOARCH ?= arm64 @@ -34,7 +35,10 @@ git-push: git push .PHONY: install -install: +install: install-cli + +.PHONY: install-cli +install-cli: go build -ldflags '$(LDFLAGS)' -o $(GOBIN)/plural ./cmd/plural .PHONY: build-cli @@ -68,8 +72,8 @@ build: ## Build the Docker image -t gcr.io/$(GCP_PROJECT)/$(APP_NAME):$(APP_VSN) \ -t $(DKR_HOST)/plural/$(APP_NAME):$(APP_VSN) . -.PHONY: build-cloud -build-cloud: ## build the cloud docker image +.PHONY: build-cloud-image +build-cloud-image: ## build the cloud docker image docker build --build-arg APP_NAME=$(APP_NAME) \ --build-arg APP_VSN=$(APP_VSN) \ --build-arg APP_DATE=$(APP_DATE) \ @@ -79,8 +83,8 @@ build-cloud: ## build the cloud docker image -t gcr.io/$(GCP_PROJECT)/$(APP_NAME)-cloud:$(APP_VSN) \ -t $(DKR_HOST)/plural/$(APP_NAME)-cloud:$(APP_VSN) -f dockerfiles/Dockerfile.cloud . -.PHONY: build-dind -build-dind: ## build the dind docker image +.PHONY: build-dind-image +build-dind-image: ## build the dind docker image docker build --build-arg APP_NAME=$(APP_NAME) \ --build-arg APP_VSN=$(APP_VSN) \ --build-arg APP_DATE=$(APP_DATE) \ @@ -118,7 +122,7 @@ pull: ## pulls new server image docker-compose pull .PHONY: serve -serve: build-cloud ## build cloud version of plural-cli and start plural serve in docker +serve: build-cloud-image ## build cloud version of plural-cli and start plural serve in docker docker kill plural-cli || true docker run --rm --name plural-cli -p 8080:8080 -d plural-cli:latest-cloud @@ -138,6 +142,40 @@ setup-tests: test: setup-tests gotestsum --format testname -- -v -race ./pkg/... ./cmd/command/... +.PHONY: e2e +e2e: --ensure-venom + @rm -rf testout ;\ + VENOM_VAR_branch=e2e-${PLRL_CLI_E2E_PROVIDER}-${TIMESTAMP} \ + VENOM_VAR_directory=../../testout/${PLRL_CLI_E2E_PROVIDER} \ + VENOM_VAR_email=${PLRL_CLI_E2E_SA_EMAIL} \ + VENOM_VAR_gitRepo=${PLRL_CLI_E2E_GIT_REPO} \ + VENOM_VAR_gitRepoPrivateKeyPath=${PLRL_CLI_E2E_PRIVATE_KEY_PATH} \ + VENOM_VAR_username=${PLRL_CLI_E2E_SA_USERNAME} \ + VENOM_VAR_token=${PLRL_CLI_E2E_SA_TOKEN} \ + VENOM_VAR_pluralHome=${HOME}/.plural \ + VENOM_VAR_pluralKey=${PLRL_CLI_E2E_PLURAL_KEY} \ + VENOM_VAR_project=${PLRL_CLI_E2E_PROJECT}-${TIMESTAMP} \ + VENOM_VAR_provider=${PLRL_CLI_E2E_PROVIDER} \ + VENOM_VAR_region=${PLRL_CLI_E2E_REGION} \ + VENOM_VAR_azureSubscriptionId=${PLRL_CLI_E2E_AZURE_SUBSCRIPTION_ID} \ + VENOM_VAR_azureTenantId=${PLRL_CLI_E2E_AZURE_TENANT_ID} \ + VENOM_VAR_azureStorageAccount=${PLRL_CLI_E2E_AZURE_STORAGE_ACCOUNT}${TIMESTAMP} \ + VENOM_VAR_gcpOrgID=${PLRL_CLI_E2E_GCLOUD_ORG_ID} \ + VENOM_VAR_gcpBillingID=${PLRL_CLI_E2E_GCLOUD_BILLING_ID} \ + VENOM_VAR_awsZoneA=${PLRL_CLI_E2E_AWS_ZONE_A} \ + VENOM_VAR_awsZoneB=${PLRL_CLI_E2E_AWS_ZONE_B} \ + VENOM_VAR_awsZoneC=${PLRL_CLI_E2E_AWS_ZONE_C} \ + VENOM_VAR_awsProject=${PLRL_CLI_E2E_PROJECT} \ + VENOM_VAR_awsBucket=e2e-tf-state-${TIMESTAMP} \ + PLURAL_LOGIN_AFFIRM_CURRENT_USER=true \ + PLURAL_UP_AFFIRM_DEPLOY=true \ + PLURAL_DOWN_AFFIRM_DESTROY=true \ + PLURAL_CD_USE_EXISTING_CREDENTIALS=true \ + TF_VAR_network=plural-e2e-network-${TIMESTAMP} \ + TF_VAR_subnetwork=plural-e2e-subnet-${TIMESTAMP} \ + TF_VAR_deletion_protection=false \ + venom run -vv --html-report --format=json --output-dir testout test/plural + .PHONY: format format: ## formats all go code to prep for linting docker run --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.62.2 golangci-lint run --fix diff --git a/cmd/command/cd/cd_repositories.go b/cmd/command/cd/cd_repositories.go index 7790a1ea1..aec1fccd0 100644 --- a/cmd/command/cd/cd_repositories.go +++ b/cmd/command/cd/cd_repositories.go @@ -2,6 +2,7 @@ package cd import ( "fmt" + "strings" "github.com/pluralsh/plural-cli/pkg/common" @@ -26,6 +27,13 @@ func (p *Plural) cdRepositoriesCommands() []cli.Command { Action: common.LatestVersion(p.handleListCDRepositories), Usage: "list repositories", }, + { + Name: "get", + Action: common.LatestVersion(common.RequireArgs(p.handleGetCDRepository, []string{"ID"})), + Usage: "get repository", + ArgsUsage: "{id}", + Flags: []cli.Flag{cli.StringFlag{Name: "o", Usage: "output format"}}, + }, { Name: "create", Action: common.LatestVersion(p.handleCreateCDRepository), @@ -35,6 +43,7 @@ func (p *Plural) cdRepositoriesCommands() []cli.Command { cli.StringFlag{Name: "passphrase", Usage: "git repo passphrase"}, cli.StringFlag{Name: "username", Usage: "git repo username"}, cli.StringFlag{Name: "password", Usage: "git repo password"}, + cli.StringFlag{Name: "o", Usage: "output format"}, }, Usage: "create repository", }, @@ -76,6 +85,27 @@ func (p *Plural) handleListCDRepositories(_ *cli.Context) error { } +func (p *Plural) handleGetCDRepository(c *cli.Context) error { + if err := p.InitConsoleClient(consoleToken, consoleURL); err != nil { + return err + } + + repo, err := p.ConsoleClient.GetRepository(c.Args().Get(0)) + if err != nil { + return err + } + + output := c.String("o") + if strings.HasPrefix(output, "jsonpath=") { + return utils.ParseJSONPath(output, repo.GetGitRepository()) + } + + headers := []string{"ID", "URL"} + return utils.PrintTable([]gqlclient.GitRepositoryFragment{*repo.GitRepository}, headers, func(r gqlclient.GitRepositoryFragment) ([]string, error) { + return []string{r.ID, r.URL}, nil + }) +} + func (p *Plural) handleCreateCDRepository(c *cli.Context) error { if err := p.InitConsoleClient(consoleToken, consoleURL); err != nil { return err @@ -87,6 +117,11 @@ func (p *Plural) handleCreateCDRepository(c *cli.Context) error { return err } + output := c.String("o") + if strings.HasPrefix(output, "jsonpath=") { + return utils.ParseJSONPath(output, repo.CreateGitRepository) + } + headers := []string{"ID", "URL"} return utils.PrintTable([]gqlclient.GitRepositoryFragment{*repo.CreateGitRepository}, headers, func(r gqlclient.GitRepositoryFragment) ([]string, error) { return []string{r.ID, r.URL}, nil diff --git a/cmd/command/cd/cd_services.go b/cmd/command/cd/cd_services.go index c36f47af0..d4458b130 100644 --- a/cmd/command/cd/cd_services.go +++ b/cmd/command/cd/cd_services.go @@ -465,13 +465,17 @@ func (p *Plural) handleDescribeClusterService(c *cli.Context) error { return fmt.Errorf("existing service deployment is empty") } output := c.String("o") - if output == "json" { + switch { + case output == "json": utils.NewJsonPrinter(existing).PrettyPrint() return nil - } else if output == "yaml" { + case output == "yaml": utils.NewYAMLPrinter(existing).PrettyPrint() return nil + case strings.HasPrefix(output, "jsonpath="): + return utils.ParseJSONPath(output, existing) } + desc, err := console.DescribeService(existing) if err != nil { return err diff --git a/go.mod b/go.mod index f1f076aa9..2406df3e0 100644 --- a/go.mod +++ b/go.mod @@ -40,8 +40,8 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/packethost/packngo v0.29.0 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c - github.com/pluralsh/console/go/client v1.23.0 - github.com/pluralsh/console/go/controller v0.0.0-20241106170618-0c255ee72c4c + github.com/pluralsh/console/go/client v1.24.1 + github.com/pluralsh/console/go/controller v0.0.0-20241128121629-07f05ca53ca4 github.com/pluralsh/gqlclient v1.12.2 github.com/pluralsh/plural-operator v0.5.5 github.com/pluralsh/polly v0.1.10 diff --git a/go.sum b/go.sum index f5138c410..79e538e9e 100644 --- a/go.sum +++ b/go.sum @@ -587,10 +587,10 @@ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjL github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pluralsh/console/go/client v1.23.0 h1:aS6CuC0o5ONSoRKFbm/rb6JVDD858KIs0t+HLLMMX/w= -github.com/pluralsh/console/go/client v1.23.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k= -github.com/pluralsh/console/go/controller v0.0.0-20241106170618-0c255ee72c4c h1:1bSNjddgWG/C0Zgm0BoOOngTlv+sFPvBjxo18I7jEdw= -github.com/pluralsh/console/go/controller v0.0.0-20241106170618-0c255ee72c4c/go.mod h1:MN71cuC4jWgH2Lk3SYNNno2FZP22u0uGsL/MqyreBs0= +github.com/pluralsh/console/go/client v1.24.1 h1:PaMJI7CjMooeHTKLAEH2Ha885Y4zz5LcoAog6T+LahM= +github.com/pluralsh/console/go/client v1.24.1/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k= +github.com/pluralsh/console/go/controller v0.0.0-20241128121629-07f05ca53ca4 h1:mYmTxes8JvIs+inp+pFyk1eukkDTxAm4lGbPRJkqs1M= +github.com/pluralsh/console/go/controller v0.0.0-20241128121629-07f05ca53ca4/go.mod h1:MN71cuC4jWgH2Lk3SYNNno2FZP22u0uGsL/MqyreBs0= github.com/pluralsh/controller-reconcile-helper v0.0.4 h1:1o+7qYSyoeqKFjx+WgQTxDz4Q2VMpzprJIIKShxqG0E= github.com/pluralsh/controller-reconcile-helper v0.0.4/go.mod h1:AfY0gtteD6veBjmB6jiRx/aR4yevEf6K0M13/pGan/s= github.com/pluralsh/gqlclient v1.12.2 h1:BrEFAASktf4quFw57CIaLAd+NZUTLhG08fe6tnhBQN4= diff --git a/hack/include/tools.mk b/hack/include/tools.mk index a875f070d..f221bd0fc 100644 --- a/hack/include/tools.mk +++ b/hack/include/tools.mk @@ -1,11 +1,14 @@ +VENOM_BINARY := $(shell which venom) +VENOM_VERSION := v1.2.0 + GO_BINARY := $(shell which go) GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1) GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2) MIN_GO_MAJOR_VERSION = 1 -MIN_GO_MINOR_VERSION = 19 +MIN_GO_MINOR_VERSION = 22 .PHONY: install-tools -install-tools: --ensure-go ## Install required dependencies to run make targets +install-tools: --ensure-go --ensure-venom ## Install required dependencies to run make targets .PHONY: --ensure-go --ensure-go: @@ -19,3 +22,12 @@ endif elif [ $(GO_MINOR_VERSION) -lt $(MIN_GO_MINOR_VERSION) ] ; then \ exit 1; \ fi + +.PHONY: --ensure-venom +--ensure-venom: +ifndef VENOM_BINARY + @echo "[tools] downloading venom..." + @go install github.com/ovh/venom/cmd/venom@${VENOM_VERSION} +else + @echo "[tools] venom already exists" +endif diff --git a/nuke-config.yml b/nuke-config.yml new file mode 100644 index 000000000..b14b900ab --- /dev/null +++ b/nuke-config.yml @@ -0,0 +1,128 @@ +regions: + - eu-west-1 + - global +accounts: + 911167907168: + filters: + EC2InternetGatewayAttachment: + - property: DefaultVPC + value: "true" + IAMRole: + - property: Name + value: "OrganizationAccountAccessRole" + - property: Name + value: "GitHubAction-AssumeRoleWithAction" + IAMRolePolicyAttachment: + - property: RoleName + value: "OrganizationAccountAccessRole" + - property: PolicyName + value: "AdministratorAccess" + EC2InternetGateway: + - property: DefaultVPC + value: "true" + EC2DHCPOption: + - property: DefaultVPC + value: "true" + EC2Subnet: + - property: DefaultForAz + value: "true" + EC2VPC: + - property: IsDefault + value: "true" + EC2SecurityGroup: + - property: Name + value: default + - property: tag:Name + type: contains + value: "Default VPC" + +blocklist: + - 312272277431 + - 654897662046 + +resource-types: + # Specifying this in the configuration will ensure that only these three + # resources are targeted by aws-nuke during it's run. + # includes: + # - S3Object + # - S3Bucket + # - IAMRole + # - EC2VPC + excludes: + - EC2DefaultSecurityGroupRule + - IAMAccountSettingPasswordPolicy + - IAMGroup + - IAMGroupPolicy + - IAMGroupPolicyAttachment + - IAMInstanceProfile + - IAMInstanceProfileRole + - IAMLoginProfile + - IAMOpenIDConnectProvider + - IAMPolicy +# - IAMRole + - IAMRolePolicy +# - IAMRolePolicyAttachment +# - IAMRolesAnywhereCRL +# - IAMRolesAnywhereProfile +# - IAMRolesAnywhereTrustAnchor + - IAMSAMLProvider + - IAMServerCertificate + - IAMServiceSpecificCredential + - IAMSigningCertificate + - IAMUser + - IAMUserAccessKey + - IAMUserGroupAttachment + - IAMUserHTTPSGitCredential + - IAMUserMFADevice + - IAMUserPolicy + - IAMUserPolicyAttachment + - IAMUserSSHPublicKey + - IAMVirtualMFADevice + - CloudWatchLogsLogGroup + - MachineLearningMLModel + - MachineLearningBranchPrediction + - MachineLearningEvaluation + - OpsWorksCMBackup + - OpsWorksUserProfile + - RoboMakerSimulationJob + - FMSPolicy + - OpsWorksCMServerState + - CloudSearchDomain + - BedrockPrompt + - OpsWorksCMServer + - RoboMakerRobotApplication + - RoboMakerSimulationApplication + - ElasticTranscoderPreset + - CodeStarProject + - IoTAuthorizer + - IoTCACertificate + - IoTCertificate + - IoTJob + - IoTOTAUpdate + - IoTPolicy + - IoTRoleAlias + - IoTSiteWiseAccessPolicy + - IoTSiteWiseAsset + - IoTSiteWiseAssetModel + - IoTSiteWiseDashboard + - IoTSiteWiseGateway + - IoTSiteWisePortal + - IoTSiteWiseProject + - IoTStream + - IoTThing + - IoTThingGroup + - IoTThingType + - IoTThingTypeState + - IoTTopicRule + - IoTTwinMakerComponentType + - IoTTwinMakerEntity + - IoTTwinMakerScene + - IoTTwinMakerSyncJob + - IoTTwinMakerWorkspace + - FMSNotificationChannel + - MachineLearningDataSource + - OpsWorksLayer + - Cloud9Environment + - ElasticTranscoderPipeline + - OpsWorksApp + - OpsWorksInstance \ No newline at end of file diff --git a/pkg/client/plural.go b/pkg/client/plural.go index b65dd7218..538405a08 100644 --- a/pkg/client/plural.go +++ b/pkg/client/plural.go @@ -4,11 +4,14 @@ import ( "fmt" "os" + "github.com/pluralsh/plural-cli/pkg/utils/git" + + "github.com/urfave/cli" + "github.com/pluralsh/plural-cli/pkg/common" "github.com/pluralsh/plural-cli/pkg/crypto" "github.com/pluralsh/plural-cli/pkg/scm" "github.com/pluralsh/plural-cli/pkg/wkspace" - "github.com/urfave/cli" "github.com/pluralsh/plural-cli/pkg/api" "github.com/pluralsh/plural-cli/pkg/config" @@ -95,9 +98,33 @@ func (p *Plural) AssumeServiceAccount(conf config.Config, man *manifest.ProjectM func (p *Plural) HandleInit(c *cli.Context) error { gitCreated := false repo := "" + p.InitPluralClient() if utils.Exists("./workspace.yaml") { - utils.Highlight("Found workspace.yaml, skipping init as this repo has already been initialized...\n") + utils.Highlight("Found workspace.yaml, skipping init as this repo has already been initialized\n") + utils.Highlight("Checking domain...\n") + proj, err := manifest.FetchProject() + if err != nil { + return err + } + if proj.Network.PluralDns { + if err := p.Client.CreateDomain(proj.Network.Subdomain); err != nil { + return err + } + } + utils.Highlight("Domain OK \n") + branch, err := git.CurrentBranch() + if err != nil { + return err + } + proj.Context["Branch"] = branch + if err := proj.Flush(); err != nil { + return err + } + if err := common.CryptoInit(c); err != nil { + return err + } + _ = wkspace.DownloadReadme() return nil } @@ -110,8 +137,6 @@ func (p *Plural) HandleInit(c *cli.Context) error { return err } - p.InitPluralClient() - me, err := p.Me() if err != nil { return api.GetErrorResponse(err, "Me") diff --git a/pkg/console/console.go b/pkg/console/console.go index 16caf9103..45726ded7 100644 --- a/pkg/console/console.go +++ b/pkg/console/console.go @@ -32,6 +32,7 @@ type ConsoleClient interface { DetachCluster(id string) error ListClusterServices(clusterId, handle *string) ([]*consoleclient.ServiceDeploymentEdgeFragment, error) CreateRepository(url string, privateKey, passphrase, username, password *string) (*consoleclient.CreateGitRepository, error) + GetRepository(id string) (*consoleclient.GetGitRepository, error) ListRepositories() (*consoleclient.ListGitRepositories, error) UpdateRepository(id string, attrs consoleclient.GitAttributes) (*consoleclient.UpdateGitRepository, error) CreateClusterService(clusterId, clusterName *string, attr consoleclient.ServiceDeploymentAttributes) (*consoleclient.ServiceDeploymentExtended, error) diff --git a/pkg/console/repositories.go b/pkg/console/repositories.go index cb943f1dd..caea3db09 100644 --- a/pkg/console/repositories.go +++ b/pkg/console/repositories.go @@ -37,3 +37,12 @@ func (c *consoleClient) UpdateRepository(id string, attrs gqlclient.GitAttribute } return res, nil } + +func (c *consoleClient) GetRepository(id string) (*gqlclient.GetGitRepository, error) { + + res, err := c.client.GetGitRepository(c.ctx, &id, nil) + if err != nil { + return nil, api.GetErrorResponse(err, "GetGitRepository") + } + return res, nil +} diff --git a/pkg/test/mocks/ConsoleClient.go b/pkg/test/mocks/ConsoleClient.go index 51f4875ee..bc8dd96fa 100644 --- a/pkg/test/mocks/ConsoleClient.go +++ b/pkg/test/mocks/ConsoleClient.go @@ -633,6 +633,36 @@ func (_m *ConsoleClient) GetProject(name string) (*client.ProjectFragment, error return r0, r1 } +// GetRepository provides a mock function with given fields: id +func (_m *ConsoleClient) GetRepository(id string) (*client.GetGitRepository, error) { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for GetRepository") + } + + var r0 *client.GetGitRepository + var r1 error + if rf, ok := ret.Get(0).(func(string) (*client.GetGitRepository, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(string) *client.GetGitRepository); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*client.GetGitRepository) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetServiceContext provides a mock function with given fields: name func (_m *ConsoleClient) GetServiceContext(name string) (*client.ServiceContextFragment, error) { ret := _m.Called(name) diff --git a/pkg/up/context.go b/pkg/up/context.go index a4c290559..bf90e743b 100644 --- a/pkg/up/context.go +++ b/pkg/up/context.go @@ -6,13 +6,14 @@ import ( "strings" "github.com/AlecAivazis/survey/v2" + "github.com/samber/lo" + "github.com/pluralsh/plural-cli/pkg/bundle" "github.com/pluralsh/plural-cli/pkg/config" "github.com/pluralsh/plural-cli/pkg/manifest" "github.com/pluralsh/plural-cli/pkg/provider" "github.com/pluralsh/plural-cli/pkg/utils" "github.com/pluralsh/plural-cli/pkg/utils/git" - "github.com/samber/lo" "github.com/mitchellh/go-homedir" ) diff --git a/pkg/up/generate.go b/pkg/up/generate.go index c41a50c9a..7ae2dff86 100644 --- a/pkg/up/generate.go +++ b/pkg/up/generate.go @@ -88,8 +88,9 @@ func (ctx *Context) Generate() (dir string, err error) { ctx.changeDelims() overwrites := []templatePair{ - {from: "bootstrap/setup.yaml", to: "bootstrap/setup.yaml"}, - {from: "bootstrap/pr-automation", to: "bootstrap/pr-automation"}, + {from: "resources/monitoring/services", to: "resources/monitoring/services"}, + {from: "resources/policy/services", to: "resources/policy/services"}, + {from: "bootstrap", to: "bootstrap"}, } for _, tpl := range overwrites { diff --git a/pkg/utils/jsonpath.go b/pkg/utils/jsonpath.go new file mode 100644 index 000000000..7859a7ebf --- /dev/null +++ b/pkg/utils/jsonpath.go @@ -0,0 +1,54 @@ +package utils + +import ( + "bytes" + "fmt" + "regexp" + "strings" + + "k8s.io/client-go/util/jsonpath" +) + +var jsonRegexp = regexp.MustCompile(`^\{\.?([^{}]+)\}$|^\.?([^{}]+)$`) + +func ParseJSONPath(input string, data interface{}) error { + after, ok := strings.CutPrefix(input, "jsonpath=") + if !ok { + return fmt.Errorf("invalid jsonpath format: %s", input) + } + field, err := RelaxedJSONPathExpression(after) + if err != nil { + return err + } + parser := jsonpath.New("parsing").AllowMissingKeys(true) + err = parser.Parse(field) + if err != nil { + return fmt.Errorf("parsing error: %w", err) + } + buf := new(bytes.Buffer) + if err := parser.Execute(buf, data); err != nil { + return err + } + fmt.Print(buf.String()) + return nil +} + +func RelaxedJSONPathExpression(pathExpression string) (string, error) { + if len(pathExpression) == 0 { + return pathExpression, nil + } + submatches := jsonRegexp.FindStringSubmatch(pathExpression) + if submatches == nil { + return "", fmt.Errorf("unexpected path string, expected a 'name1.name2' or '.name1.name2' or '{name1.name2}' or '{.name1.name2}'") + } + if len(submatches) != 3 { + return "", fmt.Errorf("unexpected submatch list: %v", submatches) + } + var fieldSpec string + if len(submatches[1]) != 0 { + fieldSpec = submatches[1] + } else { + fieldSpec = submatches[2] + } + return fmt.Sprintf("{.%s}", fieldSpec), nil +} diff --git a/test.tf b/test.tf new file mode 100644 index 000000000..91aaaff4d --- /dev/null +++ b/test.tf @@ -0,0 +1,59 @@ +terraform { + required_version = ">=1.3" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "3.106.1" # 3.40.0 doesn't work + } + azapi = { + source = "azure/azapi" + } + } +} + +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } +} + +provider "azapi" { +} + +resource "random_string" "random" { + length = 5 + upper = false + special = false +} + +resource "azurerm_resource_group" "group" { + name = "test-${random_string.random.result}" + location = "polandcentral" +} + +module "aks" { + source = "Azure/aks/azurerm" + version = "7.5.0" + + kubernetes_version = "1.22" + cluster_name = "marcin" + resource_group_name = azurerm_resource_group.group.name + prefix = "marcin" + os_disk_size_gb = 60 + sku_tier = "Standard" + rbac_aad = false + # vnet_subnet_id = azurerm_subnet.network.id + # node_pools = {for name, pool in var.node_pools : name => merge(pool, {name = name, vnet_subnet_id = azurerm_subnet.network.id})} + + ebpf_data_plane = "cilium" + network_plugin_mode = "overlay" + network_plugin = "azure" + + role_based_access_control_enabled = true + + workload_identity_enabled = true + oidc_issuer_enabled = true +} diff --git a/test/plural/lib/aws-teardown.yaml b/test/plural/lib/aws-teardown.yaml new file mode 100644 index 000000000..1d9921065 --- /dev/null +++ b/test/plural/lib/aws-teardown.yaml @@ -0,0 +1,9 @@ +executor: aws-teardown +input: + resourceGroup: '' +steps: + - script: aws-nuke run --config ../../nuke-config.yml --force --profile aws-nuke --no-dry-run --no-alias-check --max-wait-retries 20 + retry: 5 + retry_if: + - result.code ShouldNotEqual 0 + delay: 20 \ No newline at end of file diff --git a/test/plural/lib/azure-teardown.yml b/test/plural/lib/azure-teardown.yml new file mode 100644 index 000000000..e1ad0d982 --- /dev/null +++ b/test/plural/lib/azure-teardown.yml @@ -0,0 +1,9 @@ +executor: azure-teardown +input: + resourceGroup: '' +steps: + - script: az group delete --yes --no-wait --name {{ .input.resourceGroup }} + retry: 3 + retry_if: + - result.code ShouldNotEqual 0 + delay: 5 diff --git a/test/plural/lib/check-required.yml b/test/plural/lib/check-required.yml new file mode 100644 index 000000000..ad2d3b93c --- /dev/null +++ b/test/plural/lib/check-required.yml @@ -0,0 +1,81 @@ +executor: check-required +input: + branch: '' + directory: '' + email: '' + gitRepo: '' + gitRepoPrivateKeyPath: '' + gitRepoPrivateKey: '' + username: '' + token: '' + pluralHome: '' + project: '' + provider: '' + region: '' + +steps: + - name: Branch + script: "echo Branch {{ .input.branch }}" + assertions: + - result.systemout ShouldMatchRegex '^Branch .+$' + - result.code ShouldEqual 0 + + - name: Directory + script: "echo Directory {{ .input.directory }}" + assertions: + - result.systemout ShouldMatchRegex '^Directory .+$' + - result.code ShouldEqual 0 + + - name: Email + script: "echo Email {{ .input.email }}" + assertions: + - result.systemout ShouldMatchRegex '^Email .+$' + - result.code ShouldEqual 0 + + - name: Git Repo + script: "echo Git Repo {{ .input.gitRepo }}" + assertions: + - result.systemout ShouldMatchRegex '^Git Repo .+$' + - result.code ShouldEqual 0 + + - name: Private Key Path + script: "echo Private Key Path {{ .input.gitRepoPrivateKeyPath }}" + assertions: + - result.systemout ShouldMatchRegex '^Private Key Path .+$' + - result.code ShouldEqual 0 + + - name: Username + script: "echo Username {{ .input.username }}" + assertions: + - result.systemout ShouldMatchRegex '^Username .+$' + - result.code ShouldEqual 0 + + - name: Token + script: "echo Token {{ .input.token }}" + assertions: + - result.systemout ShouldMatchRegex '^Token .+$' + - result.code ShouldEqual 0 + + - name: Plural Home + script: "echo Plural Home {{ .input.pluralHome }}" + assertions: + - result.systemout ShouldMatchRegex '^Plural Home .+$' + - result.code ShouldEqual 0 + + - name: Project + script: "echo Project {{ .input.project }}" + assertions: + - result.systemout ShouldMatchRegex '^Project .+$' + - result.code ShouldEqual 0 + + - name: Provider + script: "echo Provider {{ .input.provider }}" + assertions: + - result.systemout ShouldMatchRegex '^Provider .+$' + - result.code ShouldEqual 0 + + - name: Region + script: "echo Region {{ .input.region }}" + assertions: + - result.systemout ShouldMatchRegex '^Region .+$' + - result.code ShouldEqual 0 diff --git a/test/plural/lib/cluster-services.yml b/test/plural/lib/cluster-services.yml new file mode 100644 index 000000000..919a4378f --- /dev/null +++ b/test/plural/lib/cluster-services.yml @@ -0,0 +1,42 @@ +executor: cluster-services + +steps: + - script: plural cd services describe @mgmt/deploy-operator -o jsonpath='{.status}' + assertions: + - result.systemout ShouldBeIn HEALTHY STALE + + - name: create_repository + script: plural cd repositories create --url https://github.com/argoproj/argocd-example-apps.git -o jsonpath='{.id}' + vars: + repo_id: + from: result.systemout + assertions: + - result.code ShouldEqual 0 + + - name: display_id + info: "the value of repo id is {{.repo_id}}" + + - name: get_repository + script: plural cd repositories get {{.repo_id}} -o jsonpath='{.health}' + retry: 6 + retry_if: + - result.systemout ShouldNotEqual PULLABLE + delay: 5 + assertions: + - result.systemout ShouldEqual PULLABLE + + + - name: list_repositories + script: plural cd repositories list + assertions: + - result.code ShouldEqual 0 + + + - name: create_service + script: plural cd services create --name test --repo-id {{.repo_id}} --git-ref master --git-folder helm-guestbook @mgmt + assertions: + - result.code ShouldEqual 0 + + - script: plural cd services describe @mgmt/test -o jsonpath='{.status}' + assertions: + - result.systemout ShouldBeIn HEALTHY STALE \ No newline at end of file diff --git a/test/plural/lib/cluster-setup.yml b/test/plural/lib/cluster-setup.yml new file mode 100644 index 000000000..a77ef5c93 --- /dev/null +++ b/test/plural/lib/cluster-setup.yml @@ -0,0 +1,16 @@ +executor: cluster-setup +input: + name: '' + directory: '' + +steps: + - script: | + cd {{ .input.directory }} ;\ + plural mgmt cluster + assertions: + - result.code ShouldEqual 0 + - script: | + export PLURAL_CONSOLE_TOKEN=$(kubectl get secret console-auth-token -n plrl-console -o jsonpath='{.data.access-token}' | base64 --decode) ;\ + plural cd login --url https://console.{{ .input.name }}.onplural.sh/gql --token $PLURAL_CONSOLE_TOKEN + assertions: + - result.code ShouldEqual 0 \ No newline at end of file diff --git a/test/plural/lib/context-setup.yml b/test/plural/lib/context-setup.yml new file mode 100644 index 000000000..320361688 --- /dev/null +++ b/test/plural/lib/context-setup.yml @@ -0,0 +1,23 @@ +executor: context-setup +input: + directory: '' + gitRepo: '' + gitRepoPrivateKeyPath: '' +steps: + - script: | + PRIVATE_KEY=$(cat {{ .input.gitRepoPrivateKeyPath }} | sed 's/^/ /') ;\ + cat << EOF > {{ .input.directory }}/context.yaml + apiVersion: plural.sh/v1alpha1 + kind: Context + spec: + bundles: [] + buckets: [] + domains: [] + configuration: + console: + repo_url: {{ .input.gitRepo }} + private_key: |- + $PRIVATE_KEY + EOF + assertions: + - result.code ShouldEqual 0 diff --git a/test/plural/lib/gcloud-setup.yml b/test/plural/lib/gcloud-setup.yml new file mode 100644 index 000000000..d0119de2a --- /dev/null +++ b/test/plural/lib/gcloud-setup.yml @@ -0,0 +1,23 @@ +executor: gcloud-setup +input: + orgID: '' + project: '' + billingID: '' + +steps: + - script: | + gcloud projects describe {{ .input.project }} ;\ + if [ $? -eq 1 ]; then \ + echo "Project does not exist. Creating..." ;\ + gcloud -q projects create {{ .input.project }} --name="{{ .input.project }}" --organization={{ .input.orgID }} --labels=type=e2e ;\ + gcloud -q config set project {{ .input.project }} ;\ + gcloud -q services enable cloudbilling.googleapis.com ;\ + gcloud -q billing projects link {{ .input.project }} --billing-account={{ .input.billingID }} ;\ + else + echo "Project already exists." ;\ + fi ;\ + assertions: + - result.code ShouldEqual 0 + - script: gcloud -q components install gke-gcloud-auth-plugin + assertions: + - result.code ShouldEqual 0 \ No newline at end of file diff --git a/test/plural/lib/gcloud-teardown.yml b/test/plural/lib/gcloud-teardown.yml new file mode 100644 index 000000000..287ee5338 --- /dev/null +++ b/test/plural/lib/gcloud-teardown.yml @@ -0,0 +1,10 @@ +executor: gcloud-teardown +input: + project: '' + +steps: + - script: gcloud -q projects delete {{ .input.project }} + retry: 3 + retry_if: + - result.code ShouldNotEqual 0 + delay: 5 diff --git a/test/plural/lib/git-setup.yml b/test/plural/lib/git-setup.yml new file mode 100644 index 000000000..d9f8ce2b7 --- /dev/null +++ b/test/plural/lib/git-setup.yml @@ -0,0 +1,20 @@ +executor: git-setup +input: + branch: '' + directory: '' + email: '' + gitRepo: '' + gitRepoPrivateKeyPath: '' + username: '' +steps: + - script: | + git -c core.sshCommand="ssh -i {{ .input.gitRepoPrivateKeyPath }}" clone {{ .input.gitRepo }} "{{ .input.directory }}" ;\ + cd {{ .input.directory }} ;\ + git config --local user.email {{ .input.email }} ;\ + git config --local user.name {{ .input.username }} ;\ + git checkout -b {{ .input.branch }} ;\ + GIT_SSH_COMMAND='ssh -i {{ .input.gitRepoPrivateKeyPath }} -o IdentitiesOnly=yes -F /dev/null' git push -u origin {{ .input.branch }} + info: + - 'Git Repository: {{ .input.gitRepo }}' + assertions: + - result.code ShouldEqual 0 diff --git a/test/plural/lib/git-teardown.yml b/test/plural/lib/git-teardown.yml new file mode 100644 index 000000000..85e070b4d --- /dev/null +++ b/test/plural/lib/git-teardown.yml @@ -0,0 +1,12 @@ +executor: git-teardown +input: + branch: '' + directory: '' +steps: + - script: | + cd {{ .input.directory }} ;\ + git push -d origin {{ .input.branch }} + retry: 3 + retry_if: + - result.code ShouldNotEqual 0 + delay: 5 diff --git a/test/plural/lib/plural-login.yml b/test/plural/lib/plural-login.yml new file mode 100644 index 000000000..feb497dc2 --- /dev/null +++ b/test/plural/lib/plural-login.yml @@ -0,0 +1,30 @@ +executor: plural-login +input: + email: '' + name: '' + pluralHome: '' + token: '' + key: '' +steps: + - name: Setup config.yml + script: | + cat << EOF > {{ .input.pluralHome }}/config.yml + apiVersion: platform.plural.sh/v1alpha1 + kind: Config + metadata: + name: {{ .input.name }} + spec: + email: {{ .input.email }} + token: {{ .input.token }} + namespacePrefix: "" + endpoint: "" + lockProfile: "" + reportErrors: true + assertions: + - result.code ShouldEqual 0 + - name: Setup key + script: | + cat << EOF > {{ .input.pluralHome }}/key + key: {{ .input.key }} + assertions: + - result.code ShouldEqual 0 diff --git a/test/plural/lib/workspace-setup.yml b/test/plural/lib/workspace-setup.yml new file mode 100644 index 000000000..9ae410701 --- /dev/null +++ b/test/plural/lib/workspace-setup.yml @@ -0,0 +1,74 @@ +executor: workspace-setup +input: + name: '' + directory: '' + provider: '' + region: '' + email: '' + project: '' + + # Azure variables + azureSubscriptionId: '' + azureTenantId: '' + azureStorageAccount: '' + + # AWS variables + awsZoneA: '' + awsZoneB: '' + awsZoneC: '' + awsProject: '' + awsBucket: '' + +steps: + - script: | + cat << EOF > {{ .input.directory }}/workspace.yaml + apiVersion: plural.sh/v1alpha1 + kind: ProjectManifest + metadata: + name: {{ .input.name }} + spec: + cluster: {{ .input.name }} + bucket: {{ .input.project }}-tf-state + project: {{ .input.project }} + provider: {{ .input.provider }} + region: {{ .input.region }} + owner: + email: {{ .input.email }} + network: + subdomain: {{ .input.name }}.onplural.sh + pluraldns: true + availabilityzones: [] + bucketPrefix: {{ .input.project }} + context: + StorageAccount: {{ .input.azureStorageAccount }} + SubscriptionId: {{ .input.azureSubscriptionId }} + TenantId: {{ .input.azureTenantId }} + assertions: + - result.code ShouldEqual 0 + - skip: + - provider ShouldEqual aws + script: | + cat << EOF > {{ .input.directory }}/workspace.yaml + apiVersion: plural.sh/v1alpha1 + kind: ProjectManifest + metadata: + name: {{ .input.name }} + spec: + cluster: {{ .input.name }} + bucket: {{ .input.awsBucket }} + project: {{ .input.awsProject }} + provider: {{ .input.provider }} + region: {{ .input.region }} + owner: + email: {{ .input.email }} + network: + subdomain: {{ .input.name }}.onplural.sh + pluraldns: true + availabilityzones: + - {{ .input.awsZoneA }} + - {{ .input.awsZoneB }} + - {{ .input.awsZoneC }} + bucketPrefix: e2e-tf-state + context: {} + assertions: + - result.code ShouldEqual 0 \ No newline at end of file diff --git a/test/plural/up.yml b/test/plural/up.yml new file mode 100644 index 000000000..d3a168615 --- /dev/null +++ b/test/plural/up.yml @@ -0,0 +1,156 @@ +name: Plural up +description: TODO + +vars: + ### Core variables + # Branch that will be created to run 'plural up' + branch: '' + # Local directory used to set up git repository (clone, checkout) + directory: '' + # Plural user email to run 'plural up' with + email: '' + # Git repository address used to run 'plural up' and store generated files + gitRepo: '' + # SSH key path used to access git repository + gitRepoPrivateKeyPath: '' + # Plural user name + username: '' + # Plural user acess token + token: '' + # Local plural home directory used to store plural files + pluralHome: '' + # Provider-specific project name/id + project: '' + # Provider name: gcp, azure, aws + provider: '' + # Provider region used to spin up the cluster + region: '' + # Plural aes key used to encrypt/decrypt git repository files + pluralKey: '' + + # Azure variables + azureSubscriptionId: '' + azureTenantId: '' + azureStorageAccount: '' + + ### GCP variables + # Google organization ID + gcpOrgID: '' + # Google billing account ID + gcpBillingID: '' + + # AWS variables + awsZoneA: '' + awsZoneB: '' + awsZoneC: '' + awsProject: '' + awsBucket: '' + +#secrets: +# - pluralKey +# - token +# - gcpOrgID +# - gcpBillingID +# - azureSubscriptionId +# - azureTenantId +# - azureStorageAccount + +testcases: + - name: Check required arguments + steps: + - type: check-required + + - name: Setup Git + steps: + - type: git-setup + + - name: Setup workspace file + steps: + - type: workspace-setup + name: {{ .username }} + directory: {{ .directory }} + provider: {{ .provider }} + region: {{ .region }} + email: {{ .email }} + project: {{ .project }} + azureSubscriptionId: {{ .azureSubscriptionId }} + azureTenantId: {{ .azureTenantId }} + azureStorageAccount: {{ .azureStorageAccount }} + awsZoneA: {{ .awsZoneA }} + awsZoneB: {{ .awsZoneB }} + awsZoneC: {{ .awsZoneC }} + awsProject: {{ .awsProject }} + awsBucket: {{ .awsBucket }} + + - name: Setup context file + steps: + - type: context-setup + + - name: Plural login + steps: + - type: plural-login + email: {{ .email }} + name: {{ .username }} + pluralHome: {{ .pluralHome }} + token: {{ .token }} + key: {{ .pluralKey }} + + - name: Google Cloud Setup + skip: + - provider ShouldEqual gcp + steps: + - type: gcloud-setup + orgID: {{ .gcpOrgID }} + billingID: {{ .gcpBillingID }} + project: {{ .project }} + + - name: Plural up + steps: + - script: | + cd {{ .directory }} ;\ + plural up --commit "Plural up e2e cluster" + retry: 3 + delay: 5 + + - name: Test cluster + steps: + - type: cluster-setup + name: {{ .username }} + directory: {{ .directory }} + - type: cluster-services + + - name: Plural down + skip: + - provider ShouldEqual azure + steps: + - script: | + cd {{ .directory }} ;\ + plural down + retry: 3 + delay: 5 + + - name: Azure teardown + skip: + - provider ShouldEqual azure + steps: + - type: azure-teardown + resourceGroup: {{ .project }} + + - name: Google Cloud teardown + skip: + - provider ShouldEqual gcp + steps: + - type: gcloud-teardown + project: {{ .project }} + + - name: AWS Cloud teardown + skip: + - provider ShouldEqual aws + steps: + - type: aws-teardown + + - name: Git teardown + steps: + - type: git-teardown + branch: {{ .branch }} + directory: {{ .directory }}