feat: add support for embedded cluster updates #15072

Workflow file for this run

.github/workflows/build-test.yaml at 31f2291

	name: build-test

	on:
	pull_request:
	# pull_request_target:
	# # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
	# # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
	# types: [ opened, synchronize, reopened, labeled ]

	concurrency:
	group: ${{ github.head_ref }}
	cancel-in-progress: true

	jobs:
	can-run-ci:
	runs-on: ubuntu-20.04
	# if the event is pull_request and:
	# - it is not a fork OR it is from the dependabot
	# - Then must have the label '@actions/safe-to-test'
	#
	# The 'pull_request_target' workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
	# Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
	if: >
	( github.event_name == 'pull_request' &&
	( github.event.pull_request.head.repo.full_name == github.repository \|\|
	contains(github.event.pull_request.labels.*.name, '@actions/safe-to-test' )
	)
	)
	steps:
	- name: ok
	run: echo "yes"

	# Use this to disable tests when iteratig on a specific test to save time
	enable-tests:
	runs-on: ubuntu-20.04
	steps:
	- name: ok
	# change 0 to a positive interger to prevent all tests from running
	run: exit 0


	generate-tag:
	runs-on: ubuntu-20.04
	outputs:
	tag: ${{ steps.get_tag.outputs.GIT_TAG }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Get tags
	id: get_tag
	uses: ./.github/actions/version-tag


	test-okteto-env:
	runs-on: ubuntu-latest
	needs: [ can-run-ci ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Test Okteto development environment
	uses: replicatedhq/action-okteto-test@main
	with:
	token: ${{ secrets.OKTETO_TOKEN }}
	branch: ${{ github.head_ref }}

	deps-web:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- name: Setup Node.js environment
	uses: actions/setup-node@v4
	with:
	node-version: '18.x'
	cache: yarn
	cache-dependency-path: web/yarn.lock

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: ${{ github.workspace }}/web/node_modules
	key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

	- name: Install web deps
	run: make -C web deps


	lint-web:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, deps-web ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- name: Setup Node.js environment
	uses: actions/setup-node@v4
	with:
	node-version: '18.x'
	cache: yarn
	cache-dependency-path: web/yarn.lock

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: ${{ github.workspace }}/web/node_modules
	key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

	- name: Lint
	run: make -C web lint


	unit-test-web:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, deps-web ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- name: Setup Node.js environment
	uses: actions/setup-node@v4
	with:
	node-version: '18.x'
	cache: yarn
	cache-dependency-path: web/yarn.lock

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: ${{ github.workspace }}/web/node_modules
	key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

	- name: Unit test
	run: make -C web test-unit


	build-web:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, deps-web, generate-tag ]
	steps:
	# This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
	# Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
	# this action checks out the remote branch and runs CI
	- name: Checkout
	uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- name: Setup Node.js environment
	uses: actions/setup-node@v4
	with:
	node-version: '18.x'
	cache: yarn
	cache-dependency-path: web/yarn.lock

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: ${{ github.workspace }}/web/node_modules
	key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

	- name: Build web
	env:
	GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
	run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C web build-kotsadm

	- name: Upload web artifact
	uses: actions/upload-artifact@v3
	with:
	name: web
	path: ./web/dist

	deps-kots:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'
	cache: true

	- run: go mod download

	vet-kots:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, deps-kots ]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'
	cache: true

	- name: vet
	run: make vet

	ci-test-kots:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, deps-kots ]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'
	cache: true

	- name: test
	run: make ci-test


	build-kots:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, build-web, deps-kots, generate-tag ]

	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'
	cache: true

	- name: Download web artifact
	uses: actions/download-artifact@v3
	with:
	name: web
	path: ./web/dist

	- name: Build kots
	env:
	GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
	run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make kots

	- uses: actions/upload-artifact@v3
	with:
	name: kots
	path: ./bin/kots


	build-kotsadm:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, generate-tag ]
	permissions:
	id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages.
	steps:
	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'
	cache: true
	- name: setup env
	shell: bash
	run: \|
	echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
	echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}
	- uses: ./.github/actions/build-push-kotsadm-image
	with:
	chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
	chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }}
	chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}
	image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
	git-tag: ${{ needs.generate-tag.outputs.tag }}


	build-kots-helm:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]

	steps:
	- uses: actions/checkout@v4
	with:
	ref: main
	repository: replicatedhq/kots-helm

	- name: Build KOTS Helm chart
	env:
	GIT_COMMIT: ${{ github.sha }}
	run: \|
	curl -O -L "https://raw.githubusercontent.com/replicatedhq/kots/${GIT_COMMIT}/.image.env"
	mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}"

	export CHART_VERSION=0.0.${{ github.run_id }}-automated
	export KOTS_VERSION=24h
	export KOTS_TAG=24h
	export KOTSADM_REGISTRY=ttl.sh/automated-${{ github.run_id }}

	envsubst < Chart.yaml.tmpl > Chart.yaml
	envsubst < values.yaml.tmpl > values.yaml

	CHART_NAME=$(helm package . \| rev \| cut -d/ -f1 \| rev)
	export CHART_NAME
	helm push "$CHART_NAME" oci://ttl.sh/automated-${{ github.run_id }}


	build-e2e:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'

	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- name: Cache Go modules
	uses: actions/cache@v3
	with:
	path: \|
	~/.cache/go-build
	~/go/pkg/mod
	key: ${{ runner.os }}-go-e2e-${{ hashFiles('**/go.sum') }}
	restore-keys: \|
	${{ runner.os }}-go-e2e-
	- run: make -C e2e build deps
	- run: docker save e2e-deps -o e2e/bin/e2e-deps.tar
	- uses: actions/upload-artifact@v3
	with:
	name: e2e
	path: e2e/bin/


	build-kurl-proxy:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, generate-tag ]
	steps:
	- uses: actions/setup-go@v4
	with:
	go-version: '^1.20.0'
	cache: true

	- name: setup env
	run: \|
	echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
	echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
	shell: bash

	# This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
	# Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
	# this action creates a branch based on remote branch and runs the tests
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- uses: ./.github/actions/build-push-kurl-proxy-image
	with:
	image-name: ttl.sh/automated-${{ github.run_id }}/kurl-proxy:24h
	git-tag: ${{ needs.generate-tag.outputs.tag }}


	build-migrations:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- name: setup env
	run: \|
	echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
	echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
	shell: bash

	# This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
	# Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
	# this action creates a branch based on remote branch and runs the tests
	- uses: actions/checkout@v4
	with:
	ref: ${{github.event.pull_request.head.ref}}
	repository: ${{github.event.pull_request.head.repo.full_name}}

	- name: load environment variables from .image.env
	uses: falti/dotenv-action@v1
	id: dotenv
	with:
	path: .image.env

	- name: build and push migrations for e2e
	uses: docker/build-push-action@v5
	with:
	tags: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h
	context: ./migrations
	file: ./migrations/deploy/Dockerfile
	push: true
	build-args: SCHEMAHERO_TAG=${{ steps.dotenv.outputs.schemahero_tag }}


	push-minio:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- uses: actions/checkout@v4

	- name: load environment variables from .image.env
	uses: falti/dotenv-action@v1
	id: dotenv
	with:
	path: .image.env

	- name: push minio for e2e
	run: \|
	docker pull minio/minio:${{ steps.dotenv.outputs.minio_tag }}
	docker tag minio/minio:${{ steps.dotenv.outputs.minio_tag }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }}
	docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }}


	push-rqlite:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- uses: actions/checkout@v4

	- name: load environment variables from .image.env
	uses: falti/dotenv-action@v1
	id: dotenv
	with:
	path: .image.env

	- name: push rqlite for CI
	run: \|
	docker pull rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
	docker tag rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
	docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}


	push-dex:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci ]
	steps:
	- uses: actions/checkout@v4

	- name: load environment variables from .image.env
	uses: falti/dotenv-action@v1
	id: dotenv
	with:
	path: .image.env

	- name: push dex for CI
	run: \|
	docker build --pull -f deploy/dex.Dockerfile -t ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} --build-arg TAG=${{ steps.dotenv.outputs.DEX_TAG }} .
	docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }}


	# only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**"
	kurl-addon-changes-filter:
	runs-on: ubuntu-20.04
	needs: [ can-run-ci, enable-tests ]
	outputs:
	ok-to-test: ${{ steps.filter.outputs.kurl-addon }}
	steps:
	- uses: actions/checkout@v4
	- uses: dorny/paths-filter@v2
	id: filter
	with:
	filters: \|
	kurl-addon:
	- 'deploy/kurl/kotsadm/template/**'
	- 'deploy/kurl/kotsadm/testgrid-os-spec.yaml'
	validate-kurl-addon:
	runs-on: ubuntu-20.04
	if: ${{ needs.kurl-addon-changes-filter.outputs.ok-to-test == 'true' }}
	needs: [ can-run-ci, enable-tests, generate-tag, kurl-addon-changes-filter, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-dex ]
	steps:
	- name: checkout
	uses: actions/checkout@v4
	- name: set outputs
	id: vars
	run: \|
	addon_version=${{ needs.generate-tag.outputs.tag }}
	echo "addon_version=${addon_version#v}" >> "$GITHUB_OUTPUT"
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- name: prepare kots binary executable
	run: \|
	chmod +x bin/*
	tar -C bin/ -czvf bin/kots.tar.gz kots
	- name: generate kurl add-on
	id: addon-generate
	uses: ./.github/actions/kurl-addon-kots-generate
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.KURL_ADDONS_AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.KURL_ADDONS_AWS_SECRET_ACCESS_KEY }}
	AWS_DEFAULT_REGION: us-east-1
	with:
	addon_version: ${{ steps.vars.outputs.addon_version }}
	kotsadm_image_registry: ttl.sh
	kotsadm_image_namespace: automated-${{ github.run_id }}
	kotsadm_image_tag: 24h
	kotsadm_binary_override: bin/kots.tar.gz
	- name: test kurl add-on
	id: addon-test
	uses: ./.github/actions/kurl-addon-kots-test
	with:
	addon_version: ${{ steps.vars.outputs.addon_version }}
	addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }}
	testgrid_api_token: ${{ secrets.TESTGRID_PROD_API_TOKEN }}
	- name: comment testgrid url
	uses: mshick/add-pr-comment@v2
	with:
	message: ${{ steps.addon-test.outputs.testgrid_run_message }}
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	allow-repeats: false


	cmx-versions:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci ]
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: get CMX versions to test
	id: cmx-versions-to-test
	uses: ./.github/actions/cmx-versions
	with:
	replicated-api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	outputs:
	versions-to-test: ${{ steps.cmx-versions-to-test.outputs.versions-to-test }}


	validate-existing-online-install-minimal:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: \|
	docker load -i e2e/bin/e2e-deps.tar
	chmod +x e2e/bin/*
	chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	kots-namespace: 'qakotsregression'
	test-focus: 'Regression'
	k8s-distribution: k3s
	k8s-version: v1.26
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-smoke-test:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
	continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'Smoke Test'
	kots-namespace: 'smoke-test'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-minimal-rbac:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0},
	{distribution: openshift, version: 4.13.0-okd}
	]
	env:
	APP_SLUG: minimal-rbac
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.MINIMAL_RBAC_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h \| tee output.txt

	if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
	echo "Expected to see an RBAC error for preflight checks, but did not"
	exit 1
	fi

	if ! grep -q 'The app was not deployed.' output.txt; then
	printf "Expected to see message about app not being deployed, but did not\n"
	exit 1
	fi

	if grep FAIL output.txt \| grep -q 'This application requires at least 100 nodes'; then
	printf "Did not expect to see a failure about number of nodes, but did\n"
	exit 1
	fi

	if ! kubectl get role -n "$APP_SLUG" \| grep -q kotsadm; then
	echo "kotsadm role not found in namespace $APP_SLUG"
	kubectl get role -n "$APP_SLUG"
	exit 1
	fi

	if ! kubectl get rolebinding -n "$APP_SLUG" \| grep -q kotsadm; then
	echo "kotsadm rolebinding not found in namespace $APP_SLUG"
	kubectl get rolebinding -n "$APP_SLUG"
	exit 1
	fi

	if kubectl get clusterrole \| grep -q kotsadm; then
	echo "found kotsadm clusterrole in minimal RBAC install"
	kubectl get clusterrole
	exit 1
	fi

	if kubectl get clusterrolebinding \| grep -q kotsadm; then
	echo "found kotsadm clusterrolebinding in minimal RBAC install"
	kubectl get clusterrolebinding
	exit 1
	fi

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-backup-and-restore:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
	continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'Backup and Restore'
	kots-namespace: 'backup-and-restore'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-no-required-config:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'No Required Config'
	kots-namespace: 'no-required-config'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-strict-preflight-checks:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: strict-preflight-checks
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.STRICT_PREFLIGHT_CHECKS_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--skip-preflights \
	--kotsadm-tag 24h \| tee output.txt

	if ! grep -q 'The app was not deployed.' output.txt; then
	printf "Expected to see message about app not being deployed, but did not\n"
	exit 1
	fi

	if ! grep FAIL output.txt \| grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then
	printf "Expected to see a failure about kubernetes version, but did not\n"
	exit 1
	fi

	if ! kubectl logs deploy/kotsadm -n "$APP_SLUG" \| grep -q "preflights will not be skipped, strict preflights are set to true"; then
	echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs"
	echo "kotsadm logs:"
	kubectl logs deploy/kotsadm -n "$APP_SLUG"
	exit 1
	fi

	# disable the strict preflight check and app should deploy successfully
	./bin/kots set config "$APP_SLUG" enable_failing_strict_analyzers="0" --namespace "$APP_SLUG" --deploy

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs deploy/kotsadm --tail=100 -n "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	printf "App is installed successfully and is ready\n\n"
	./bin/kots get apps --namespace "$APP_SLUG"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-config:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'Config Validation'
	kots-namespace: 'config-validation'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-version-history-pagination:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: k3s-local, version: v1.27.1-k3s1}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'Version History Pagination'
	kots-namespace: 'version-history-pagination'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-change-license:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'Change License'
	kots-namespace: 'change-license'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-minimal-rbac-override:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0},
	{distribution: openshift, version: 4.13.0-okd}
	]
	env:
	APP_SLUG: minimal-rbac
	APP_VERSION_LABEL: "0.0.1"
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' \|\| '' }}
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: minimal rbac override on command line
	run: \|
	set +e
	echo ${{ secrets.MINIMAL_RBAC_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--app-version-label "$APP_VERSION_LABEL" \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h \
	--use-minimal-rbac \| tee output.txt

	if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
	echo "Expected to see an RBAC error for preflight checks, but did not"
	exit 1
	fi

	if ! grep -q 'The app was not deployed.' output.txt; then
	printf "Expected to see message about app not being deployed, but did not\n"
	exit 1
	fi

	if grep FAIL output.txt \| grep -q 'This application requires at least 100 nodes'; then
	printf "Did not expect to see a failure about number of nodes, but did\n"
	exit 1
	fi

	if ! kubectl get role -n "$APP_SLUG" \| grep -q kotsadm; then
	echo "kotsadm role not found in namespace $APP_SLUG"
	kubectl get role -n "$APP_SLUG"
	exit 1
	fi

	if ! kubectl get rolebinding -n "$APP_SLUG" \| grep -q kotsadm; then
	echo "kotsadm rolebinding not found in namespace $APP_SLUG"
	kubectl get rolebinding -n "$APP_SLUG"
	exit 1
	fi

	if kubectl get clusterrole \| grep -q kotsadm; then
	echo "found kotsadm clusterrole in minimal RBAC install"
	kubectl get clusterrole
	exit 1
	fi

	if kubectl get clusterrolebinding \| grep -q kotsadm; then
	echo "found kotsadm clusterrolebinding in minimal RBAC install"
	kubectl get clusterrolebinding
	exit 1
	fi

	- name: create namespace and dockerhub secret
	run: \|
	kubectl delete ns "$APP_SLUG" --ignore-not-found
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: no minimal rbac override on command line
	run: \|
	set +e
	echo ${{ secrets.MINIMAL_RBAC_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--app-version-label "$APP_VERSION_LABEL" \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h \| tee output.txt

	if grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
	echo "Did not expect to see an RBAC error for preflight checks, but did"
	exit 1
	fi

	if ! grep -q 'The app was not deployed.' output.txt; then
	printf "Expected to see message about app not being deployed, but did not\n"
	exit 1
	fi

	if ! grep FAIL output.txt \| grep -q 'This application requires at least 100 nodes'; then
	printf "Expected to see a failure about number of nodes, but did not\n"
	exit 1
	fi

	if kubectl get role -n "$APP_SLUG" \| grep -q kotsadm; then
	echo "kotsadm role found in cluster scoped install"
	kubectl get role -n "$APP_SLUG"
	exit 1
	fi

	if kubectl get rolebinding -n "$APP_SLUG" \| grep -q kotsadm; then
	echo "kotsadm rolebinding found in cluster scoped install"
	kubectl get rolebinding -n "$APP_SLUG"
	exit 1
	fi

	if ! kubectl get clusterrole \| grep -q kotsadm; then
	echo "No kotsadm clusterrole found in cluster scoped install"
	kubectl get clusterrole
	exit 1
	fi

	if ! kubectl get clusterrolebinding \| grep -q kotsadm; then
	echo "No kotsadm clusterrolebinding found in cluster scoped install"
	kubectl get clusterrolebinding
	exit 1
	fi

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-multi-namespace:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
	continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
	env:
	APP_SLUG: multi-namespace-yeti
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- uses: azure/setup-helm@v3
	with:
	token: ${{ secrets.GITHUB_TOKEN }}

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' \|\| '' }}
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 180 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly

	if ! helm ls -n postgres-test \| awk 'NR>1{print $1}' \| grep -q postgresql; then
	printf "postgresql helm release not found in postgres-test namespace\n\n"
	helm ls -n postgres-test
	exit 1
	fi

	if ! helm ls -n "$APP_SLUG" \| awk 'NR>1{print $1}' \| grep -q private-chart; then
	printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG"
	helm ls -n "$APP_SLUG"
	exit 1
	fi

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-kots-pull:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
	continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
	env:
	APP_NAME: multi-namespace-yeti
	APP_SLUG: multi-namespace
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' \|\| '' }}
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_NAME"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_NAME"

	- name: run kots pull
	run: \|
	set +e
	echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} \| base64 -d > license.yaml
	./bin/kots pull "$APP_NAME/automated" \
	--license-file license.yaml \
	--shared-password password \
	--namespace "$APP_NAME" \
	--exclude-admin-console

	kubectl create ns "$APP_NAME"
	kubectl create ns nginx-test
	kubectl create ns redis-test
	kubectl create ns postgres-test

	# HACK: without operator, additonal namespaces don't get image pull secrets
	echo ${{ secrets.MULTI_NAMESPACE_REGISTRY_AUTH }} \| base64 -d > replicated-registry-auth.json
	kubectl -n nginx-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
	kubectl -n redis-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
	kubectl -n redis-test create secret generic multi-namespace-yeti-redis-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
	kubectl -n postgres-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
	kubectl -n default create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json

	kustomize build "$PWD/$APP_SLUG/overlays/midstream" \| kubectl apply -f -
	kustomize build "$PWD/$APP_SLUG/overlays/midstream/charts/redis" \| kubectl apply -f -

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "Failed to apply spec"
	echo "------pods:"
	kubectl get pods -A
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	echo "Waiting for pods to start"

	COUNTER=1
	while [ "$(kubectl get pods --no-headers \| grep -v Running \| grep -cv Completed)" -gt 0 ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for pods to start"
	kubectl get pods -A
	exit 1
	fi
	sleep 1
	done

	echo "All pods started"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_NAME"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-app-version-label:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: app-version-label
	APP_VERSION_LABEL: v1.0.0
	LATEST_APP_VERSION_LABEL: v1.0.1
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: test kots install with version label
	run: \|
	set +e
	echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--app-version-label "$APP_VERSION_LABEL" \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $3}')" != "$APP_VERSION_LABEL" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be installed with correct version label: $APP_VERSION_LABEL"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	printf "App is installed successfully with the correct version label: %s\n\n" "$APP_VERSION_LABEL"
	./bin/kots get apps --namespace "$APP_SLUG"

	# test setting DockerHub credentials

	set +e

	# TODO: deploy and check secrets are actually created and images are pulled
	./bin/kots docker ensure-secret --dockerhub-username replicatedtests --dockerhub-password ${{ secrets.DOCKERHUB_RATELIMIT_PASSWORD }} -n "$APP_SLUG"
	./bin/kots download -n "$APP_SLUG" --slug "$APP_SLUG"
	if grep "${APP_SLUG}-kotsadm-dockerhub" -w "./${APP_SLUG}/overlays/midstream/secret.yaml"; then
	echo "Found DockerHub secret in ${APP_SLUG} latest version"
	else
	echo "No DockerHub secret found in appication namespace"
	exit 1
	fi

	- name: remove the app
	run: \|
	set +e
	./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force

	- name: test kots install without version label
	run: \|
	set +e
	echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $3}')" != "$LATEST_APP_VERSION_LABEL" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be installed with latest version label: $LATEST_APP_VERSION_LABEL"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	printf "App is installed successfully with the correct version label: %s\n\n" "$LATEST_APP_VERSION_LABEL"
	./bin/kots get apps --namespace "$APP_SLUG"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-helm-install-order:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: helm-install-order
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' \|\| '' }}
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--skip-preflights \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	printf "App is installed successfully and is ready\n\n"
	./bin/kots get apps --namespace "$APP_SLUG"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-no-redeploy-on-restart:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: no-redeploy-on-restart
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.NO_REDEPLOY_ON_RESTART_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--skip-preflights \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	# wait for application job to be created
	COUNTER=1
	while [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found \| awk 'NR>1' \| wc -l)" == "0" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 60 ]; then
	echo "Timed out waiting for job to be created"
	exit 1
	fi
	sleep 1
	done

	# delete the application job and restart the admin console
	kubectl delete jobs -n "$APP_SLUG" --all
	kubectl delete pods -l app=kotsadm -n "$APP_SLUG"

	# wait for old kotsadm pod to terminate
	COUNTER=1
	while [ "$(kubectl get pods -l app=kotsadm -n "$APP_SLUG" \| awk 'NR>1' \| wc -l)" != "1" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 60 ]; then
	echo "More than 1 kotsadm pod found"
	exit 1
	fi
	sleep 1
	done

	# wait for new kotsadm pod to become ready
	kubectl wait --for=condition=ready pod -l app=kotsadm -n "$APP_SLUG" --timeout=60s

	# delay in case the app takes a bit to be deployed
	sleep 20

	# validate that the application wasn't re-deployed and the job wasn't re-created
	if [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found \| awk 'NR>1' \| wc -l)" != "0" ]; then
	echo "App should not be re-deployed after restart"
	exit 1
	fi

	printf "Success. App was not re-deployed after restart\n\n"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-kubernetes-installer-preflight:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: kubernetes-installer-preflight
	steps:
	- uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e

	echo ${{ secrets.KUBERNETES_INSTALLER_PREFLIGHT_LICENSE }} \| base64 -d > license.yaml

	# Fake kurl installation using the crd and installer spec from the application manifests:

	# Pull application manifests
	./bin/kots pull "$APP_SLUG/automated" --license-file license.yaml --shared-password password

	# Apply installer crd
	kubectl apply -f "$APP_SLUG/upstream/installer-crd.yaml"

	# Wait for crd to be created
	kubectl wait --for condition=established --timeout=60s crd/installers.cluster.kurl.sh

	# Seems that the above does not always guarantee the crd exists? So just in case...
	sleep 10

	# Apply installer
	kubectl apply -f "$APP_SLUG/upstream/installer.yaml"

	# Create kurl-config configmap in kube-system
	kubectl create cm kurl-config -n kube-system --from-literal=installer_id=7cc8094

	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	# validate that preflight checks ran
	JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}"
	if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" \| grep -c pending_preflight)" != "1" ]; then
	echo "Preflight checks did not run"
	exit 1
	fi

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# try get apps without namespace (using kubeconfig)
	# validate that output is the same as above
	mkdir -p /tmp/.kube
	sudo cp "$KUBECONFIG" /tmp/.kube/config
	sudo chmod -R 777 /tmp/.kube
	export KUBECONFIG=/tmp/.kube/config
	kubectl config set-context --current --namespace="$APP_SLUG"
	if [ "$(./bin/kots get apps \| awk 'NR>1{print $2}')" != "ready" ]; then
	echo "kots get apps output is not the same as above"
	exit 1
	fi

	printf "App is installed successfully and is ready\n\n"
	./bin/kots get apps --namespace "$APP_SLUG"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-kots-push-images-anonymous:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: run kots admin-console push-images
	run: \|
	set +e
	./bin/kots admin-console push-images ./hack/tests/small.airgap ttl.sh/automated-${{ github.run_id }}


	validate-kots-admin-console-generate-manifests:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: minimal-rbac
	BASE_KOTS_VERSION: v1.72.0
	NAMESPACE: generate-manifests
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: run kots admin-console generate-manifests without k8s context
	run: \|
	set +e
	./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password

	- name: validate that ./admin-console exists and is not empty
	run: \|
	set +e
	if [ ! -d ./admin-console ]; then
	echo "admin-console directory does not exist"
	exit 1
	fi
	if [ -z "$(ls -A ./admin-console)" ]; then
	echo "admin-console directory is empty"
	exit 1
	fi

	- name: remove admin-console directory
	run: rm -rf ./admin-console

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download base kots version
	run: \|
	curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
	&& tar zxvf kots_linux_amd64.tar.gz \
	&& mv kots "kots-$BASE_KOTS_VERSION"

	- name: create namespace
	run: \|
	set +e
	kubectl create namespace "$NAMESPACE"

	- name: run kots admin-console generate-manifests using base kots version with k8s context
	run: \|
	set +e
	"./kots-$BASE_KOTS_VERSION" admin-console generate-manifests -n "$NAMESPACE" --shared-password password

	- name: apply the generated manifests
	run: \|
	set +e
	kubectl apply -f ./admin-console -n "$NAMESPACE"

	- name: wait for the kotsadm-minio-0 pod to be created
	run: \|
	set +e
	COUNTER=1
	while ! kubectl get pods -n "$NAMESPACE" \| grep -q kotsadm-minio-0; do
	((COUNTER += 1))
	if [ $COUNTER -gt 30 ]; then
	echo "timed out waiting for kotsadm-minio-0 pod to be created"
	exit 1
	fi
	sleep 1
	done

	- name: wait for kotsadm-minio-0 pod to be ready
	run: \|
	set +e
	kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"

	- name: wait for the kotsadm deployment to be ready
	run: \|
	set +e
	kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"

	- name: run kots admin-console generate-manifests using new kots version with k8s context
	run: \|
	set +e
	./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	- name: validate that ./admin-console/minio-statefulset.yaml has initContainers since a migration is needed
	run: \|
	set +e
	if ! grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
	echo "admin-console/minio-statefulset.yaml does not have initContainers"
	exit 1
	fi

	- name: apply the generated manifests
	run: \|
	set +e
	kubectl apply -f ./admin-console -n "$NAMESPACE"

	- name: wait for kotsadm-minio-0 pod to be ready
	run: \|
	set +e
	sleep 10
	kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"

	- name: wait for the kotsadm deployment to be ready
	run: \|
	set +e
	kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"

	- name: run kots admin-console generate-manifests using new kots version with k8s context
	run: \|
	set +e
	./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	- name: validate that ./admin-console/minio-statefulset.yaml does not have initContainers since a migration is not needed
	run: \|
	set +e
	if grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
	echo "admin-console/minio-statefulset.yaml has initContainers"
	exit 1
	fi

	- name: apply the generated manifests
	run: \|
	set +e
	kubectl apply -f ./admin-console -n "$NAMESPACE"

	- name: wait for kotsadm-minio-0 pod to be ready
	run: \|
	set +e
	sleep 10
	kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"

	- name: wait for the kotsadm deployment to be ready
	run: \|
	set +e
	kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"

	- name: print pods and logs on failure
	if: failure()
	run: \|
	echo "------pods:"
	kubectl -n "$NAMESPACE" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$NAMESPACE"
	echo "------kotsadm-minio logs"
	kubectl logs -l app=kotsadm-minio --tail=100 --namespace "$NAMESPACE"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: automated-${{ github.run_id }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-min-kots-version:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
	env:
	APP_SLUG: min-kots-version
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: \|
	docker load -i e2e/bin/e2e-deps.tar
	chmod +x e2e/bin/*
	chmod +x bin/*
	cp ./bin/kots /usr/local/bin/kubectl-kots
	sudo apt-get update -y && sudo apt-get install jq -y
	- uses: ./.github/actions/kots-e2e
	id: kots-e2e
	with:
	test-focus: 'Min KOTS Version'
	kots-namespace: 'min-kots-version'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	kotsadm-image-registry: ttl.sh
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
	k8s-cluster-skip-teardown: true

	- name: validate that kots install fails early
	run: \|
	set +e

	result=$(kubectl kots install "$APP_SLUG/automated" --no-port-forward --namespace "$APP_SLUG" --shared-password password 2>&1 >/dev/null)
	echo "$result"

	if [[ "$result" == "requires" ]] && [[ "$result" == "10000.0.0" ]]; then
	exit 0
	else
	exit 1
	fi

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}


	validate-target-kots-version:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: \|
	docker load -i e2e/bin/e2e-deps.tar
	chmod +x e2e/bin/*
	chmod +x bin/*
	cp ./bin/kots /usr/local/bin/kubectl-kots
	sudo apt-get update -y && sudo apt-get install jq -y
	- uses: ./.github/actions/kots-e2e
	id: kots-e2e
	with:
	test-focus: 'Target KOTS Version'
	kots-namespace: 'target-kots-version'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
	k8s-cluster-skip-teardown: true

	- name: validate that kots install fails early
	run: \|
	set +e

	result=$(kubectl kots install target-kots-version/automated --no-port-forward --namespace target-kots-version --shared-password password 2>&1 >/dev/null)
	echo "$result"

	if [[ "$result" == "requires" ]] && [[ "$result" == "1.0.0" ]]; then
	exit 0
	else
	exit 1
	fi

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}


	validate-range-kots-version:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: \|
	docker load -i e2e/bin/e2e-deps.tar
	chmod +x e2e/bin/*
	chmod +x bin/*
	cp ./bin/kots /usr/local/bin/kubectl-kots
	sudo apt-get update -y && sudo apt-get install jq -y
	- uses: ./.github/actions/kots-e2e
	id: kots-e2e
	with:
	test-focus: 'Range KOTS Version'
	kots-namespace: 'range-kots-version'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
	k8s-cluster-skip-teardown: true

	- name: validate that kots install fails early
	run: \|
	set +e

	result=$(kubectl kots install range-kots-version/automated --no-port-forward --namespace range-kots-version --shared-password password 2>&1 >/dev/null)
	echo "$result"

	if [[ "$result" == "requires" ]] && [[ "$result" == "11000.0.0" ]]; then
	exit 0
	else
	exit 1
	fi

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}


	validate-kots-upgrade:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
	continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
	env:
	APP_SLUG: postgres-to-rqlite
	BASE_KOTS_VERSION: v1.57.0
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' \|\| '' }}
	export-kubeconfig: true

	- name: download base kots version
	run: \|
	curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
	&& tar zxvf kots_linux_amd64.tar.gz \
	&& mv kots "kots-$BASE_KOTS_VERSION"

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} \| base64 -d > license.yaml

	# install using the base KOTS version

	"./kots-$BASE_KOTS_VERSION" \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--port-forward=false \
	--namespace "$APP_SLUG" \
	--shared-password password

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	"./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# upgrade using the new KOTS version

	./bin/kots admin-console upgrade \
	--namespace "$APP_SLUG" \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	# verify that the postgres to rqlite migration was successful

	if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" \| grep -q "Migrated from Postgres to rqlite successfully"; then
	echo "Failed to find a successful migration log line"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG"
	exit 1
	fi

	# verify that the minio migration happened

	if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then
	echo "Failed to find initContainers in the kotsadm-minio statefulset"
	echo "kotsadm-minio statefulset:"
	kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml
	exit 1
	fi

	# make sure app is still installed and ready

	if [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; then
	echo "App is not ready after the upgrade"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi

	printf "App is still installed and is ready after the migration\n\n"
	./bin/kots get apps --namespace "$APP_SLUG"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-kots-helm-release-secret-migration:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: helm-release-secret-migration
	RELEASE_NAME: helm-release-chart
	RELEASE_NAMESPACE: helm-release
	BASE_KOTS_VERSION: v1.94.0
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download base kots version
	run: \|
	curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
	&& tar zxvf kots_linux_amd64.tar.gz \
	&& mv kots "kots-$BASE_KOTS_VERSION"

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.HELM_RELEASE_SECRET_MIGRATION_LICENSE }} \| base64 -d > license.yaml

	# install using the base KOTS version
	"./kots-$BASE_KOTS_VERSION" \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward=true \
	--namespace "$APP_SLUG" \
	--shared-password password

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	"./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# verify that the helm release secret is created in the kotsadm namespace
	releaseSecretName=$(kubectl get secret -n "$APP_SLUG" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
	if [ -z "$releaseSecretName" ]; then
	echo "Failed to find the helm release secret in the $APP_SLUG namespace"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi

	# if there are more than one helm release secrets, fail
	if [ "$(echo "$releaseSecretName" \| wc -l)" -gt 1 ]; then
	echo "Found more than one helm release secret in the $APP_SLUG namespace"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi

	# upgrade using the new KOTS version
	./bin/kots admin-console upgrade \
	--namespace "$APP_SLUG" \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	# make a config change and redeploy the app
	./bin/kots set config "$APP_SLUG" create_new_sequence=true --deploy --namespace "$APP_SLUG"

	# make sure app is still installed and ready
	if [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; then
	echo "App is not ready after the upgrade"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi

	COUNTER=1
	while [ "$(helm ls -n "$RELEASE_NAMESPACE" \| grep "$RELEASE_NAME" \| awk 'NR>0{print $3}')" != "2" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for $RELEASE_NAME to be upgraded to revision 2"
	helm ls -n "$RELEASE_NAMESPACE"
	exit 1
	fi
	sleep 1
	done

	# verify that the helm release secret is created in the helm relase namespace
	COUNT=1
	releaseSecret=""
	while [ -z "$releaseSecret" ]; do
	((COUNT += 1))
	if [ $COUNT -gt 10 ]; then
	echo "Timed out waiting for the helm release secret to be migrated"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	releaseSecret=$(kubectl get secret "$releaseSecretName" -n "$RELEASE_NAMESPACE")
	done

	# verify that the release secret in app namepspace is deleted
	oldReleaseSecret=$(kubectl get secret "$releaseSecretName" -n "$APP_SLUG")
	if [ -n "$oldReleaseSecret" ]; then
	echo "Found the helm release secret in the $APP_SLUG namespace"
	echo "$oldReleaseSecret"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi

	# verify that there are two helm release secrets in the helm release namespace
	releaseSecretNames=$(kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
	releaseSecretNamesCount=$(echo "$releaseSecretNames" \| wc -w)
	if [ "$releaseSecretNamesCount" -ne 2 ]; then
	echo "Found $releaseSecretNamesCount helm release secrets in the $RELEASE_NAMESPACE namespace(Want: 2)"
	kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi

	printf "Helm release secret migration test passed\n\n"
	./bin/kots get apps --namespace "$APP_SLUG"

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-multi-app-backup-and-restore:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'multi-app-backup-and-restore'
	kots-namespace: 'multi-app-backup-and-restore'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-multi-app-install:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'multi-app-install'
	kots-namespace: 'multi-app-install'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-airgap-smoke-test:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'airgap-smoke-test'
	kots-namespace: 'airgap-smoke-test'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	kots-airgap: true
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-remove-app:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0, instance-type: r1.medium}
	]
	env:
	APP_SLUG: remove-app
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	instance-type: ${{ matrix.cluster.instance-type }}
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e

	echo ${{ secrets.REMOVE_APP_LICENSE }} \| base64 -d > license.yaml
	HELM_APP_NAMESPACES=(postgres-test redis-test "$APP_SLUG")
	APP_NAMESPACES=(postgres-test redis-test nginx-test rabbitmq-system "$APP_SLUG")

	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# test that --undeploy deletes application resources from the cluster

	./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --undeploy

	if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json \| tr -d '\n')" != "[]" ]; then
	printf "App reference was not removed\n\n"
	exit 1
	fi

	if [ "$(kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found \| wc -l \| tr -d '\n')" != "0" ]; then
	printf "Application resources are still found in the cluster\n\n"
	kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG"
	exit 1
	fi

	if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found \| wc -l \| tr -d '\n')" != "0" ]; then
	printf "rabbitmqclusters.rabbitmq.com CRD was not removed\n\n"
	kubectl get crd rabbitmqclusters.rabbitmq.com
	exit 1
	fi

	for ns in "${HELM_APP_NAMESPACES[@]}"; do
	if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found \| wc -l \| tr -d '\n')" != "0" ]; then
	printf "Application Helm release secrets are still found in the namepspace %s\n\n" "$ns"
	kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
	exit 1
	fi
	done

	# re-install and test that running the remove command without --undeploy does _not_ delete any resources from the cluster

	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force

	if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json \| tr -d '\n')" != "[]" ]; then
	printf "App reference was not removed\n\n"
	exit 1
	fi

	for ns in "${APP_NAMESPACES[@]}"; do
	if [ "$(kubectl get all,secrets,configmap,pvc -n "$ns" -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found \| wc -l \| tr -d '\n')" == "0" ]; then
	printf "%s namespace does not contain any resources\n\n" "$ns"
	kubectl get all,secrets,configmap,pvc -n "$ns"
	exit 1
	fi
	done

	if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found \| wc -l \| tr -d '\n')" == "0" ]; then
	printf "rabbitmqclusters.rabbitmq.com CRD was not found\n\n"
	kubectl get crd rabbitmqclusters.rabbitmq.com
	exit 1
	fi

	if [ "$(kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system --ignore-not-found \| wc -l \| tr -d '\n')" == "0" ]; then
	printf "rabbitmq-system namespace does not contain rabbitmqclusters\n\n"
	kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system
	exit 1
	fi

	for ns in "${HELM_APP_NAMESPACES[@]}"; do
	if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found \| wc -l \| tr -d '\n')" == "0" ]; then
	printf "Application Helm release secrets are not found in namepspace %s\n\n" "$ns"
	kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
	exit 1
	fi
	done

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-registry-check:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: run the test
	run: \|
	set +e

	./bin/kots admin-console push-images \
	not-an-app.airgap localhost:1234/not-a-namespace \
	--registry-username not-a-username \
	--registry-password not-a-password > output.txt 2>&1

	if ! grep -q 'Failed to test access' output.txt; then
	printf "Expected registry validation to fail before pushing images, but did not.\n\n"
	cat output.txt
	exit 1
	fi

	rm output.txt

	./bin/kots install not-an-app \
	--airgap-bundle not-an-app.airgap \
	--kotsadm-registry localhost:1234/not-a-namespace \
	--registry-username not-a-username \
	--registry-password not-a-password > output.txt 2>&1

	if ! grep -q 'Failed to test access' output.txt; then
	printf "Expected registry validation to fail before installation, but did not.\n\n"
	cat output.txt
	exit 1
	fi

	rm output.txt

	./bin/kots upstream upgrade not-an-app \
	--airgap-bundle not-an-app.airgap \
	--kotsadm-registry localhost:1234/not-a-namespace \
	--registry-username not-a-username \
	--registry-password not-a-password \
	--namespace not-a-namespace > output.txt 2>&1

	if ! grep -q 'Failed to test access' output.txt; then
	printf "Expected registry validation to fail before upgrading, but did not.\n\n"
	cat output.txt
	exit 1
	fi

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-native-helm-v2:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: native-helm-v2
	INITIAL_VERSION: '0.1.1'
	UPGRADE_VERSION: '0.2.1'
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- uses: azure/setup-helm@v3
	with:
	token: ${{ secrets.GITHUB_TOKEN }}

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.NATIVE_HELM_V2_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--app-version-label "$INITIAL_VERSION" \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $3}')" != "$INITIAL_VERSION" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# validate that helm charts installed using both native helm workflow were deployed via the helm CLI correctly

	if ! helm ls -n "$APP_SLUG" \| awk 'NR>1{print $1}' \| grep -q my-chart-release; then
	printf "my-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
	helm ls -n "$APP_SLUG"
	exit 1
	fi

	if ! helm ls -n "$APP_SLUG" \| awk 'NR>1{print $1}' \| grep -q my-other-chart-release; then
	printf "my-other-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
	helm ls -n "$APP_SLUG"
	exit 1
	fi

	# validate that `helm get values` works for the v1beta2 chart

	if ! helm get values my-chart-release -n "$APP_SLUG" \| grep -q my-value; then
	printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
	helm get values my-chart-release -n "$APP_SLUG"
	exit 1
	fi

	# upgrade the app version

	./bin/kots upstream upgrade "$APP_SLUG" -n "$APP_SLUG" --deploy

	# wait for my-chart-release to be uninstalled and my-other-chart-release to be upgraded to revision 2

	COUNTER=1
	while [ "$(helm ls -n "$APP_SLUG" \| awk 'NR>1{print $1}' \| grep my-chart-release)" != "" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for my-chart-release to be uninstalled"
	helm ls -n "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	COUNTER=1
	while [ "$(helm ls -n "$APP_SLUG" \| grep my-other-chart-release \| awk 'NR>0{print $3}')" != "2" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for my-other-chart-release to be upgraded to revision 2"
	helm ls -n "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# validate that `helm get values` works for the upgraded v1beta2 chart
	if ! helm get values my-other-chart-release -n "$APP_SLUG" \| grep -q my-value; then
	printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
	helm get values my-chart-release -n "$APP_SLUG"
	exit 1
	fi

	# wait for the app to be ready again

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $3}')" != "$UPGRADE_VERSION" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# validate that the conditional chart is installed
	COUNTER=1
	while [ "$(helm ls -n "$APP_SLUG" \| awk 'NR>1{print $1}' \| grep my-conditional-chart-release)" == "" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for my-conditional-chart-release to be installed"
	helm ls -n "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# toggle the config option to exclude the conditional chart
	./bin/kots set config "$APP_SLUG" install_conditional_chart=0 --deploy --namespace "$APP_SLUG"

	# wait for my-conditional-chart-release to be uninstalled
	COUNTER=1
	while [ "$(helm ls -n "$APP_SLUG" \| awk 'NR>1{print $1}' \| grep my-conditional-chart-release)" != "" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for my-conditional-chart-release to be uninstalled"
	helm ls -n "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-deployment-orchestration:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	env:
	APP_SLUG: deployment-orchestration
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- uses: azure/setup-helm@v3
	with:
	token: ${{ secrets.GITHUB_TOKEN }}

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$APP_SLUG"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"

	- name: run the test
	run: \|
	set +e
	echo ${{ secrets.DEPLOYMENT_ORCHESTRATION_LICENSE }} \| base64 -d > license.yaml
	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$APP_SLUG" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$APP_SLUG" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit $EXIT_CODE
	fi

	function wait_for_log {
	local counter=1
	local timeout=30
	local log_pattern="$1"

	while ! kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" \| grep -i "$log_pattern"; do
	((counter += 1))
	if [ $counter -gt $timeout ]; then
	echo "Timed out waiting for log $log_pattern"
	exit 1
	fi
	sleep 1
	done
	}

	wait_for_log "applying phase -9999"

	wait_for_log "applying phase -3"
	wait_for_log "waiting for resource apiextensions.k8s.io/v1/CustomResourceDefinition/myresources.example.com in namespace $APP_SLUG to be ready"

	wait_for_log "applying phase -2"

	# validate that phase -1 has not deployed yet since we're waiting on the CR status fields
	if kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" \| grep -i "applying phase -1"; then
	printf "phase -1 was deployed before phase -2 completed"
	kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG"
	exit 1
	fi

	wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.extract=true"

	# set .status.tasks.extract=true
	kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"extract": true}}}' --type=merge

	wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.transform=true"

	# set .status.tasks.transform=true
	kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"transform": true}}}' --type=merge

	wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.load=true"

	# set .status.tasks.load=true
	kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"load": true}}}' --type=merge

	# now validate that the remaining phases are deployed (-1, 0, and 1)
	wait_for_log "applying phase -1"
	wait_for_log "waiting for resource apps/v1/Deployment/nginx-1 in namespace $APP_SLUG to be ready"

	wait_for_log "applying phase 0"
	wait_for_log "waiting for resource apps/v1/Deployment/nginx-2 in namespace $APP_SLUG to be ready"

	wait_for_log "applying phase 1"
	wait_for_log "waiting for resource /v1/Service/nginx-2 in namespace $APP_SLUG to be ready"

	# wait for the app to be ready
	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$APP_SLUG" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$APP_SLUG"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
	exit 1
	fi
	sleep 1
	done

	# remove the app
	./bin/kots remove "$APP_SLUG" -n "$APP_SLUG" --undeploy

	wait_for_log "deleting resources in phase -1"
	wait_for_log "deleting resources in phase 0"
	wait_for_log "deleting resources in phase 1"
	wait_for_log "deleting resources in phase 2"
	wait_for_log "deleting resources in phase 3"

	# validate that the app reference was removed
	if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json \| tr -d '\n')" != "[]" ]; then
	printf "App reference was not removed\n\n"
	exit 1
	fi

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-replicated-sdk:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0},
	{distribution: openshift, version: 4.13.0-okd}
	]
	env:
	KOTS_NAMESPACE: replicated-sdk
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Create Cluster
	id: create-cluster
	uses: replicatedhq/replicated-actions/create-cluster@v1
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	kubernetes-distribution: ${{ matrix.cluster.distribution }}
	kubernetes-version: ${{ matrix.cluster.version }}
	cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
	timeout-minutes: '120'
	ttl: 2h
	export-kubeconfig: true

	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/

	- run: chmod +x bin/kots

	- name: create namespace and dockerhub secret
	run: \|
	kubectl create ns "$KOTS_NAMESPACE"
	kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$KOTS_NAMESPACE"

	- name: run upgrade-to-replicated-sdk test
	id: upgrade-to-replicated-sdk
	env:
	APP_SLUG: upgrade-to-replicated-sdk
	run: \|
	set +e

	echo ${{ secrets.UPGRADE_TO_REPLICATED_SDK_LICENSE }} \| base64 -d > license.yaml
	LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
	echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"

	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$KOTS_NAMESPACE" \
	--shared-password password \
	--app-version-label v1.0.0 \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$KOTS_NAMESPACE" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit $EXIT_CODE
	fi

	# wait for the app to be ready
	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$KOTS_NAMESPACE"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit 1
	fi
	sleep 1
	done

	# upgrade the app to the new version
	./bin/kots upstream upgrade "$APP_SLUG" -n "$KOTS_NAMESPACE" --deploy
	sleep 5

	# wait for the app to be ready again
	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$KOTS_NAMESPACE"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit 1
	fi
	sleep 1
	done

	# get the version of the replicated-sdk that is running and set it as an output
	REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' \| grep -oE '[^:]+$')
	echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"

	- run: rm -rf ./replicated-sdk
	- name: Checkout replicated-sdk
	uses: actions/checkout@v4
	with:
	repository: replicatedhq/replicated-sdk
	path: replicated-sdk
	ref: ${{ steps.upgrade-to-replicated-sdk.outputs.replicated-sdk-version }}

	- name: Validate endpoints
	uses: ./replicated-sdk/.github/actions/validate-endpoints
	with:
	license-id: ${{ steps.upgrade-to-replicated-sdk.outputs.license-id }}
	license-fields: '[{"name":"expires_at","value": ""}]'
	integration-enabled: true
	namespace: "$KOTS_NAMESPACE"

	- name: remove upgrade-to-replicated-sdk app
	env:
	APP_SLUG: upgrade-to-replicated-sdk
	run: \|
	# remove the app
	./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy

	# validate that the app reference was removed
	if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json \| tr -d '\n')" != "[]" ]; then
	printf "App reference was not removed\n\n"
	exit 1
	fi

	- name: run replicated-sdk-subchart-native-helm-v1 test
	id: replicated-sdk-subchart-native-helm-v1
	env:
	APP_SLUG: replicated-sdk-subchart-native-helm-v1
	run: \|
	set +e

	echo ${{ secrets.REPLICATED_SDK_SUBCHART_NATIVE_HELM_V1 }} \| base64 -d > license.yaml
	LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
	echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"

	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$KOTS_NAMESPACE" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$KOTS_NAMESPACE" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit $EXIT_CODE
	fi

	# wait for the app to be ready
	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$KOTS_NAMESPACE"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit 1
	fi
	sleep 1
	done

	# get the version of the replicated-sdk that is running and set it as an output
	REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' \| grep -oE '[^:]+$')
	echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"

	- run: rm -rf ./replicated-sdk
	- name: Checkout replicated-sdk
	uses: actions/checkout@v4
	with:
	repository: replicatedhq/replicated-sdk
	path: replicated-sdk
	ref: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.replicated-sdk-version }}

	- name: Validate endpoints
	uses: ./replicated-sdk/.github/actions/validate-endpoints
	with:
	license-id: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.license-id }}
	license-fields: '[{"name":"expires_at","value": ""}]'
	integration-enabled: true
	namespace: "$KOTS_NAMESPACE"

	- name: remove replicated-sdk-subchart-native-helm-v1 app
	env:
	APP_SLUG: replicated-sdk-subchart-native-helm-v1
	run: \|
	# remove the app
	./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy

	# validate that the app reference was removed
	if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json \| tr -d '\n')" != "[]" ]; then
	printf "App reference was not removed\n\n"
	exit 1
	fi

	- name: run replicated-sdk-subchart-replicated-helm test
	id: replicated-sdk-subchart-replicated-helm
	env:
	APP_SLUG: replicated-sdk-subchart-replicated-helm
	run: \|
	set +e

	echo ${{ secrets.REPLICATED_SDK_SUBCHART_REPLICATED_HELM }} \| base64 -d > license.yaml
	LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
	echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"

	./bin/kots \
	install "$APP_SLUG/automated" \
	--license-file license.yaml \
	--no-port-forward \
	--namespace "$KOTS_NAMESPACE" \
	--shared-password password \
	--kotsadm-registry ttl.sh \
	--kotsadm-namespace automated-${{ github.run_id }} \
	--kotsadm-tag 24h

	EXIT_CODE=$?
	if [ $EXIT_CODE -ne 0 ]; then
	echo "------pods:"
	kubectl -n "$KOTS_NAMESPACE" get pods
	echo "------kotsadm logs"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit $EXIT_CODE
	fi

	# wait for the app to be ready
	COUNTER=1
	while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" \| awk 'NR>1{print $2}')" != "ready" ]; do
	((COUNTER += 1))
	if [ $COUNTER -gt 120 ]; then
	echo "Timed out waiting for app to be ready"
	./bin/kots get apps --namespace "$KOTS_NAMESPACE"
	echo "kotsadm logs:"
	kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
	exit 1
	fi
	sleep 1
	done

	# get the version of the replicated-sdk that is running and set it as an output
	REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' \| grep -oE '[^:]+$')
	echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"

	- run: rm -rf ./replicated-sdk
	- name: Checkout replicated-sdk
	uses: actions/checkout@v4
	with:
	repository: replicatedhq/replicated-sdk
	path: replicated-sdk
	ref: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.replicated-sdk-version }}

	- name: Validate endpoints
	uses: ./replicated-sdk/.github/actions/validate-endpoints
	with:
	license-id: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.license-id }}
	license-fields: '[{"name":"expires_at","value": ""}]'
	integration-enabled: true
	namespace: "$KOTS_NAMESPACE"
	deployed-via-kubectl: true

	- name: remove replicated-sdk-subchart-replicated-helm app
	env:
	APP_SLUG: replicated-sdk-subchart-replicated-helm
	run: \|
	# remove the app
	./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy

	# validate that the app reference was removed
	if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json \| tr -d '\n')" != "[]" ]; then
	printf "App reference was not removed\n\n"
	exit 1
	fi

	- name: Generate support bundle on failure
	if: failure()
	uses: ./.github/actions/generate-support-bundle
	with:
	kots-namespace: "$APP_SLUG"
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

	- name: Remove Cluster
	id: remove-cluster
	uses: replicatedhq/replicated-actions/remove-cluster@v1
	if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
	continue-on-error: true
	with:
	api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
	cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


	validate-support-bundle:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0},
	{distribution: openshift, version: 4.13.0-okd}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'Support Bundle'
	kots-namespace: 'support-bundle'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-gitops:
	runs-on: ubuntu-20.04
	needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
	strategy:
	fail-fast: false
	matrix:
	cluster: [
	{distribution: kind, version: v1.28.0}
	]
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: download e2e deps
	uses: actions/download-artifact@v3
	with:
	name: e2e
	path: e2e/bin/
	- run: docker load -i e2e/bin/e2e-deps.tar
	- run: chmod +x e2e/bin/*
	- name: download kots binary
	uses: actions/download-artifact@v3
	with:
	name: kots
	path: bin/
	- run: chmod +x bin/*
	- uses: ./.github/actions/kots-e2e
	with:
	test-focus: 'GitOps'
	kots-namespace: 'gitops'
	k8s-distribution: ${{ matrix.cluster.distribution }}
	k8s-version: ${{ matrix.cluster.version }}
	testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
	testim-branch: ${{ github.head_ref == 'main' && 'master' \|\| github.head_ref }}
	aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
	aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
	replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
	kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
	kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'


	validate-pr-tests:
	runs-on: ubuntu-20.04
	needs:
	# static and unit tests
	- lint-web
	- unit-test-web
	- vet-kots
	- ci-test-kots
	# testim tests
	- validate-existing-online-install-minimal
	- validate-smoke-test
	- validate-backup-and-restore
	- validate-no-required-config
	- validate-version-history-pagination
	- validate-change-license
	- validate-min-kots-version
	- validate-target-kots-version
	- validate-range-kots-version
	- validate-multi-app-backup-and-restore
	- validate-multi-app-install
	- validate-airgap-smoke-test
	- validate-config
	- validate-support-bundle
	- validate-gitops
	# non-testim tests
	- validate-minimal-rbac
	- validate-minimal-rbac-override
	- validate-multi-namespace
	- validate-kots-pull
	- validate-app-version-label
	- validate-helm-install-order
	- validate-no-redeploy-on-restart
	- validate-kubernetes-installer-preflight
	- validate-kots-upgrade
	- validate-remove-app
	- validate-registry-check
	- validate-kots-helm-release-secret-migration
	- validate-native-helm-v2
	- validate-deployment-orchestration
	- validate-replicated-sdk
	- validate-strict-preflight-checks
	# cli-only tests
	- validate-kots-push-images-anonymous
	steps:
	- run: echo "All PR tests passed"


	# this job will validate that the kurl add-on validation did not fail and that all pr-tests succeed
	# it is used for the github branch protection rule
	validate-success:
	runs-on: ubuntu-20.04
	needs:
	- validate-kurl-addon
	- validate-pr-tests
	if: always()
	steps:
	# https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
	# if the validate-kurl-addon job failed or was cancelled, this job will fail. it's ok if this job was skipped
	- name: fail if validate-kurl-addon job failed
	if: needs.validate-kurl-addon.result == 'failure' \|\| needs.validate-kurl-addon.result == 'cancelled'
	run: exit 1
	# if the validate-pr-tests job was not successful, this job will fail
	- name: fail if validate-pr-tests job was not successful
	if: needs.validate-pr-tests.result != 'success'
	run: exit 1
	# if the validate-pr-tests job was successful, this job will succeed
	- name: succeed if validate-pr-tests job succeeded
	if: needs.validate-pr-tests.result == 'success'
	run: echo "Validation succeeded"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: add support for embedded cluster updates #15072

Workflow file

feat: add support for embedded cluster updates #15072

Jobs

Run details

Workflow file for this run