From 0d1754ab8bff5eac5e458f6d6af234d703b9a835 Mon Sep 17 00:00:00 2001
From: Salah Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 16 Jan 2024 17:39:45 +0000
Subject: [PATCH] updates

---
 .../build-push-kotsadm-image/action.yml       |    4 +-
 .github/workflows/build-test.yaml             | 7752 ++++++++---------
 Makefile                                      |    2 +
 deploy/melange.yaml.tmpl                      |    4 +-
 web/babel.config.js                           |    8 +-
 web/dist/README.md                            |    3 -
 6 files changed, 3888 insertions(+), 3885 deletions(-)
 delete mode 100644 web/dist/README.md

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index ad8af17e60..e3db10f69d 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -66,14 +66,14 @@ runs:
     - uses: chainguard-dev/actions/melange-build@main
       with:
         config: deploy/melange.yaml
-        archs: x86_64,arm64
+        archs: arm64
         sign-with-temporary-key: true
         cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
 
     - uses: chainguard-images/actions/apko-publish@main
       with:
         config: deploy/apko.yaml
-        archs: x86_64,arm64
+        archs: arm64
         tag: ${{ inputs.image-name }}
         vcs-url: true
         generic-user: ${{ inputs.registry-username }}
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 78406fd85f..18e749bcdf 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -51,217 +51,217 @@ jobs:
         uses: ./.github/actions/version-tag
 
 
-  test-okteto-env:
-    runs-on: ubuntu-latest
-    needs: [ can-run-ci ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Test Okteto development environment
-        uses: replicatedhq/action-okteto-test@main
-        with:
-          token: ${{ secrets.OKTETO_TOKEN }}
-          branch: ${{ github.head_ref }}
-
-  deps-web:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v4
-        with:
-          node-version: '18.x'
-          cache: yarn
-          cache-dependency-path: web/yarn.lock
-
-      - name: Cache node_modules
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/web/node_modules
-          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
-
-      - name: Install web deps
-        run: make -C web deps
-
-
-  lint-web:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, deps-web ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v4
-        with:
-          node-version: '18.x'
-          cache: yarn
-          cache-dependency-path: web/yarn.lock
-
-      - name: Cache node_modules
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/web/node_modules
-          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
-
-      - name: Lint
-        run: make -C web lint
-
-
-  unit-test-web:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, deps-web ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v4
-        with:
-          node-version: '18.x'
-          cache: yarn
-          cache-dependency-path: web/yarn.lock
-
-      - name: Cache node_modules
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/web/node_modules
-          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
-
-      - name: Unit test
-        run: make -C web test-unit
-
-
-  build-web:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, deps-web, generate-tag ]
-    steps:
-      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
-      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      # this action checks out the remote branch and runs CI
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v4
-        with:
-          node-version: '18.x'
-          cache: yarn
-          cache-dependency-path: web/yarn.lock
-
-      - name: Cache node_modules
-        uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/web/node_modules
-          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
-
-      - name: Build web
-        env:
-          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
-        run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C web build-kotsadm
-
-      - name: Upload web artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: web
-          path: ./web/dist
-
-  deps-kots:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '^1.20.0'
-          cache: true
-
-      - run: go mod download
-
-  vet-kots:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, deps-kots ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '^1.20.0'
-          cache: true
-
-      - name: vet
-        run: make vet
-
-  ci-test-kots:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, deps-kots ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '^1.20.0'
-          cache: true
-
-      - name: test
-        run: make ci-test
-
-
-  build-kots:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, build-web, deps-kots, generate-tag ]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '^1.20.0'
-          cache: true
-
-      - name: Download web artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: web
-          path: ./web/dist
-
-      - name: Build kots
-        env:
-          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
-        run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make kots
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: kots
-          path: ./bin/kots
+  # test-okteto-env:
+  #   runs-on: ubuntu-latest
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: Test Okteto development environment
+  #       uses: replicatedhq/action-okteto-test@main
+  #       with:
+  #         token: ${{ secrets.OKTETO_TOKEN }}
+  #         branch: ${{ github.head_ref }}
+
+  # deps-web:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: Setup Node.js environment
+  #       uses: actions/setup-node@v4
+  #       with:
+  #         node-version: '18.x'
+  #         cache: yarn
+  #         cache-dependency-path: web/yarn.lock
+
+  #     - name: Cache node_modules
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ${{ github.workspace }}/web/node_modules
+  #         key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
+
+  #     - name: Install web deps
+  #       run: make -C web deps
+
+
+  # lint-web:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, deps-web ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: Setup Node.js environment
+  #       uses: actions/setup-node@v4
+  #       with:
+  #         node-version: '18.x'
+  #         cache: yarn
+  #         cache-dependency-path: web/yarn.lock
+
+  #     - name: Cache node_modules
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ${{ github.workspace }}/web/node_modules
+  #         key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
+
+  #     - name: Lint
+  #       run: make -C web lint
+
+
+  # unit-test-web:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, deps-web ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: Setup Node.js environment
+  #       uses: actions/setup-node@v4
+  #       with:
+  #         node-version: '18.x'
+  #         cache: yarn
+  #         cache-dependency-path: web/yarn.lock
+
+  #     - name: Cache node_modules
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ${{ github.workspace }}/web/node_modules
+  #         key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
+
+  #     - name: Unit test
+  #       run: make -C web test-unit
+
+
+  # build-web:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, deps-web, generate-tag ]
+  #   steps:
+  #     # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
+  #     # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+  #     # this action checks out the remote branch and runs CI
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: Setup Node.js environment
+  #       uses: actions/setup-node@v4
+  #       with:
+  #         node-version: '18.x'
+  #         cache: yarn
+  #         cache-dependency-path: web/yarn.lock
+
+  #     - name: Cache node_modules
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: ${{ github.workspace }}/web/node_modules
+  #         key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}
+
+  #     - name: Build web
+  #       env:
+  #         GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
+  #       run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C web build-kotsadm
+
+  #     - name: Upload web artifact
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: web
+  #         path: ./web/dist
+
+  # deps-kots:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '^1.20.0'
+  #         cache: true
+
+  #     - run: go mod download
+
+  # vet-kots:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, deps-kots ]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '^1.20.0'
+  #         cache: true
+
+  #     - name: vet
+  #       run: make vet
+
+  # ci-test-kots:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, deps-kots ]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '^1.20.0'
+  #         cache: true
+
+  #     - name: test
+  #       run: make ci-test
+
+
+  # build-kots:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, build-web, deps-kots, generate-tag ]
+
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '^1.20.0'
+  #         cache: true
+
+  #     - name: Download web artifact
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: web
+  #         path: ./web/dist
+
+  #     - name: Build kots
+  #       env:
+  #         GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
+  #       run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make kots
+
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: ./bin/kots
 
 
   build-kotsadm:
@@ -292,3671 +292,3671 @@ jobs:
           git-tag: ${{ needs.generate-tag.outputs.tag }}
 
 
-  build-kots-helm:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: main
-          repository: replicatedhq/kots-helm
-
-      - name: Build KOTS Helm chart
-        env:
-          GIT_COMMIT: ${{ github.sha }}
-        run: |
-          curl -O -L "https://raw.githubusercontent.com/replicatedhq/kots/${GIT_COMMIT}/.image.env"
-          mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}"
-
-          export CHART_VERSION=0.0.${{ github.run_id }}-automated
-          export KOTS_VERSION=24h
-          export KOTS_TAG=24h
-          export KOTSADM_REGISTRY=ttl.sh/automated-${{ github.run_id }}
-
-          envsubst < Chart.yaml.tmpl > Chart.yaml
-          envsubst < values.yaml.tmpl > values.yaml
-
-          CHART_NAME=$(helm package . | rev | cut -d/ -f1 | rev)
-          export CHART_NAME
-          helm push "$CHART_NAME" oci://ttl.sh/automated-${{ github.run_id }}
-
-
-  build-e2e:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-    steps:
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '^1.20.0'
-
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Cache Go modules
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-e2e-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-e2e-
-      - run: make -C e2e build deps
-      - run: docker save e2e-deps -o e2e/bin/e2e-deps.tar
-      - uses: actions/upload-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-
-
-  build-kurl-proxy:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, generate-tag ]
-    steps:
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '^1.20.0'
-          cache: true
-
-      - name: setup env
-        run: |
-          echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
-          echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
-        shell: bash
-
-      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
-      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      # this action creates a branch based on remote branch and runs the tests
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: ./.github/actions/build-push-kurl-proxy-image
-        with:
-          image-name: ttl.sh/automated-${{ github.run_id }}/kurl-proxy:24h
-          git-tag: ${{ needs.generate-tag.outputs.tag }}
-
-
-  build-migrations:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, generate-tag ]
-    steps:
-      - name: setup env
-        run: |
-          echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
-          echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
-        shell: bash
-
-      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
-      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      # this action creates a branch based on remote branch and runs the tests
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: ./.github/actions/build-push-kotsadm-migrations-image
-        with:
-          image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h
-          git-tag: ${{ needs.generate-tag.outputs.tag }}
-
-
-  push-minio:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-    steps:
-      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
-      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      # this action creates a branch based on remote branch and runs the tests
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: load environment variables from .image.env
-        uses: falti/dotenv-action@v1
-        id: dotenv
-        with:
-          path: .image.env
-
-      - name: push minio
-        run: |
-          docker pull kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }}
-          docker tag kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }}
-          docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }}
-
-
-  push-rqlite:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-    steps:
-      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
-      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      # this action creates a branch based on remote branch and runs the tests
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: load environment variables from .image.env
-        uses: falti/dotenv-action@v1
-        id: dotenv
-        with:
-          path: .image.env
-
-      - name: push rqlite
-        run: |
-          docker pull kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
-          docker tag kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
-          docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
-
-  push-dex:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
-    steps:
-      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
-      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      # this action creates a branch based on remote branch and runs the tests
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: load environment variables from .image.env
-        uses: falti/dotenv-action@v1
-        id: dotenv
-        with:
-          path: .image.env
-
-      - name: push dex
-        run: |
-          docker pull kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }}
-          docker tag kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }}
-          docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }}
-
-  # only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**"
-  kurl-addon-changes-filter:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, enable-tests ]
-    outputs:
-      ok-to-test: ${{ steps.filter.outputs.kurl-addon }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            kurl-addon:
-              - 'deploy/kurl/kotsadm/template/**'
-              - 'deploy/kurl/kotsadm/testgrid-os-spec.yaml'
-  validate-kurl-addon:
-    runs-on: ubuntu-20.04
-    if: ${{ needs.kurl-addon-changes-filter.outputs.ok-to-test == 'true' }}
-    needs: [ can-run-ci, enable-tests, generate-tag, kurl-addon-changes-filter, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-dex ]
-    steps:
-      - name: checkout
-        uses: actions/checkout@v4
-      - name: set outputs
-        id: vars
-        run: |
-          addon_version=${{ needs.generate-tag.outputs.tag }}
-          echo "addon_version=${addon_version#v}" >> "$GITHUB_OUTPUT"
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - name: prepare kots binary executable
-        run: |
-          chmod +x bin/*
-          tar -C bin/ -czvf bin/kots.tar.gz kots
-      - name: generate kurl add-on
-        id: addon-generate
-        uses: ./.github/actions/kurl-addon-kots-generate
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.KURL_ADDONS_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.KURL_ADDONS_AWS_SECRET_ACCESS_KEY }}
-          AWS_DEFAULT_REGION: us-east-1
-        with:
-          addon_version: ${{ steps.vars.outputs.addon_version }}
-          kotsadm_image_registry: ttl.sh
-          kotsadm_image_namespace: automated-${{ github.run_id }}
-          kotsadm_image_tag: 24h
-          kotsadm_binary_override: bin/kots.tar.gz
-      - name: test kurl add-on
-        id: addon-test
-        uses: ./.github/actions/kurl-addon-kots-test
-        with:
-          addon_version: ${{ steps.vars.outputs.addon_version }}
-          addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }}
-          testgrid_api_token: ${{ secrets.TESTGRID_PROD_API_TOKEN }}
-      - name: comment testgrid url
-        uses: mshick/add-pr-comment@v2
-        with:
-          message: ${{ steps.addon-test.outputs.testgrid_run_message }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allow-repeats: false
-
-
-  cmx-versions:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - name: get CMX versions to test
-        id: cmx-versions-to-test
-        uses: ./.github/actions/cmx-versions
-        with:
-          replicated-api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-    outputs:
-      versions-to-test: ${{ steps.cmx-versions-to-test.outputs.versions-to-test }}
-
-
-  validate-existing-online-install-minimal:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: |
-          docker load -i e2e/bin/e2e-deps.tar
-          chmod +x e2e/bin/*
-          chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          kots-namespace: 'qakotsregression'
-          test-focus: 'Regression'
-          k8s-distribution: k3s
-          k8s-version: v1.26
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-smoke-test:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
-    continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'Smoke Test'
-          kots-namespace: 'smoke-test'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-minimal-rbac:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0},
-          {distribution: openshift, version: 4.13.0-okd}
-        ]
-    env:
-      APP_SLUG: minimal-rbac
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h | tee output.txt
-
-          if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
-            echo "Expected to see an RBAC error for preflight checks, but did not" 
-            exit 1
-          fi
-
-          if ! grep -q 'The app was not deployed.' output.txt; then
-            printf "Expected to see message about app not being deployed, but did not\n"
-            exit 1
-          fi
-
-          if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then
-            printf "Did not expect to see a failure about number of nodes, but did\n"
-            exit 1
-          fi
-
-          if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then
-            echo "kotsadm role not found in namespace $APP_SLUG"
-            kubectl get role -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then
-            echo "kotsadm rolebinding not found in namespace $APP_SLUG"
-            kubectl get rolebinding -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if kubectl get clusterrole | grep -q kotsadm; then
-            echo "found kotsadm clusterrole in minimal RBAC install"
-            kubectl get clusterrole
-            exit 1
-          fi
-
-          if kubectl get clusterrolebinding | grep -q kotsadm; then
-            echo "found kotsadm clusterrolebinding in minimal RBAC install"
-            kubectl get clusterrolebinding
-            exit 1
-          fi
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-backup-and-restore:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
-    continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'Backup and Restore'
-          kots-namespace: 'backup-and-restore'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-no-required-config:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'No Required Config'
-          kots-namespace: 'no-required-config'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-strict-preflight-checks:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    env:
-      APP_SLUG: strict-preflight-checks
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.STRICT_PREFLIGHT_CHECKS_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --skip-preflights \
-            --kotsadm-tag 24h | tee output.txt
-
-          if ! grep -q 'The app was not deployed.' output.txt; then
-            printf "Expected to see message about app not being deployed, but did not\n"
-            exit 1
-          fi
-
-          if ! grep FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then
-            printf "Expected to see a failure about kubernetes version, but did not\n"
-            exit 1
-          fi
-
-          if ! kubectl logs deploy/kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then
-            echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs"
-            echo "kotsadm logs:"
-            kubectl logs deploy/kotsadm -n "$APP_SLUG"
-            exit 1
-          fi
-
-          # disable the strict preflight check and app should deploy successfully
-          ./bin/kots set config "$APP_SLUG" enable_failing_strict_analyzers="0" --namespace "$APP_SLUG" --deploy
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs deploy/kotsadm --tail=100 -n "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          printf "App is installed successfully and is ready\n\n"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-config:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'Config Validation'
-          kots-namespace: 'config-validation'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-version-history-pagination:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: k3s-local, version: v1.27.1-k3s1}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'Version History Pagination'
-          kots-namespace: 'version-history-pagination'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-change-license:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'Change License'
-          kots-namespace: 'change-license'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-minimal-rbac-override:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0},
-          {distribution: openshift, version: 4.13.0-okd}
-        ]
-    env:
-      APP_SLUG: minimal-rbac
-      APP_VERSION_LABEL: "0.0.1"
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: minimal rbac override on command line
-        run: |
-          set +e
-          echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --app-version-label "$APP_VERSION_LABEL" \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h \
-            --use-minimal-rbac | tee output.txt
-
-          if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
-            echo "Expected to see an RBAC error for preflight checks, but did not" 
-            exit 1
-          fi
-
-          if ! grep -q 'The app was not deployed.' output.txt; then
-            printf "Expected to see message about app not being deployed, but did not\n"
-            exit 1
-          fi
-
-          if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then
-            printf "Did not expect to see a failure about number of nodes, but did\n"
-            exit 1
-          fi
-
-          if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then
-            echo "kotsadm role not found in namespace $APP_SLUG"
-            kubectl get role -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then
-            echo "kotsadm rolebinding not found in namespace $APP_SLUG"
-            kubectl get rolebinding -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if kubectl get clusterrole | grep -q kotsadm; then
-            echo "found kotsadm clusterrole in minimal RBAC install"
-            kubectl get clusterrole
-            exit 1
-          fi
-
-          if kubectl get clusterrolebinding | grep -q kotsadm; then
-            echo "found kotsadm clusterrolebinding in minimal RBAC install"
-            kubectl get clusterrolebinding
-            exit 1
-          fi
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl delete ns "$APP_SLUG" --ignore-not-found
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: no minimal rbac override on command line
-        run: |
-          set +e
-          echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --app-version-label "$APP_VERSION_LABEL" \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h | tee output.txt
-
-          if grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
-            echo "Did not expect to see an RBAC error for preflight checks, but did" 
-            exit 1
-          fi
-
-          if ! grep -q 'The app was not deployed.' output.txt; then
-            printf "Expected to see message about app not being deployed, but did not\n"
-            exit 1
-          fi
-
-          if ! grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then
-            printf "Expected to see a failure about number of nodes, but did not\n"
-            exit 1
-          fi
-
-          if kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then
-            echo "kotsadm role found in cluster scoped install"
-            kubectl get role -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then
-            echo "kotsadm rolebinding found in cluster scoped install"
-            kubectl get rolebinding -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if ! kubectl get clusterrole | grep -q kotsadm; then
-            echo "No kotsadm clusterrole found in cluster scoped install"
-            kubectl get clusterrole
-            exit 1
-          fi
-
-          if ! kubectl get clusterrolebinding | grep -q kotsadm; then
-            echo "No kotsadm clusterrolebinding found in cluster scoped install"
-            kubectl get clusterrolebinding
-            exit 1
-          fi
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-multi-namespace:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
-    continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
-    env:
-      APP_SLUG: multi-namespace-yeti
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - uses: azure/setup-helm@v3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create Cluster 
-        id: create-cluster
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 180 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly
-
-          if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then
-            printf "postgresql helm release not found in postgres-test namespace\n\n"
-            helm ls -n postgres-test
-            exit 1
-          fi
-
-          if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then
-            printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG"
-            helm ls -n "$APP_SLUG"
-            exit 1
-          fi
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  # build-kots-helm:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: main
+  #         repository: replicatedhq/kots-helm
+
+  #     - name: Build KOTS Helm chart
+  #       env:
+  #         GIT_COMMIT: ${{ github.sha }}
+  #       run: |
+  #         curl -O -L "https://raw.githubusercontent.com/replicatedhq/kots/${GIT_COMMIT}/.image.env"
+  #         mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}"
+
+  #         export CHART_VERSION=0.0.${{ github.run_id }}-automated
+  #         export KOTS_VERSION=24h
+  #         export KOTS_TAG=24h
+  #         export KOTSADM_REGISTRY=ttl.sh/automated-${{ github.run_id }}
+
+  #         envsubst < Chart.yaml.tmpl > Chart.yaml
+  #         envsubst < values.yaml.tmpl > values.yaml
+
+  #         CHART_NAME=$(helm package . | rev | cut -d/ -f1 | rev)
+  #         export CHART_NAME
+  #         helm push "$CHART_NAME" oci://ttl.sh/automated-${{ github.run_id }}
+
+
+  # build-e2e:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '^1.20.0'
+
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: Cache Go modules
+  #       uses: actions/cache@v3
+  #       with:
+  #         path: |
+  #           ~/.cache/go-build
+  #           ~/go/pkg/mod
+  #         key: ${{ runner.os }}-go-e2e-${{ hashFiles('**/go.sum') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-go-e2e-
+  #     - run: make -C e2e build deps
+  #     - run: docker save e2e-deps -o e2e/bin/e2e-deps.tar
+  #     - uses: actions/upload-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+
+
+  # build-kurl-proxy:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, generate-tag ]
+  #   steps:
+  #     - uses: actions/setup-go@v5
+  #       with:
+  #         go-version: '^1.20.0'
+  #         cache: true
+
+  #     - name: setup env
+  #       run: |
+  #         echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
+  #         echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
+  #       shell: bash
+
+  #     # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
+  #     # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+  #     # this action creates a branch based on remote branch and runs the tests
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - uses: ./.github/actions/build-push-kurl-proxy-image
+  #       with:
+  #         image-name: ttl.sh/automated-${{ github.run_id }}/kurl-proxy:24h
+  #         git-tag: ${{ needs.generate-tag.outputs.tag }}
+
+
+  # build-migrations:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, generate-tag ]
+  #   steps:
+  #     - name: setup env
+  #       run: |
+  #         echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
+  #         echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
+  #       shell: bash
+
+  #     # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
+  #     # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+  #     # this action creates a branch based on remote branch and runs the tests
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - uses: ./.github/actions/build-push-kotsadm-migrations-image
+  #       with:
+  #         image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h
+  #         git-tag: ${{ needs.generate-tag.outputs.tag }}
+
+
+  # push-minio:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
+  #     # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+  #     # this action creates a branch based on remote branch and runs the tests
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: load environment variables from .image.env
+  #       uses: falti/dotenv-action@v1
+  #       id: dotenv
+  #       with:
+  #         path: .image.env
+
+  #     - name: push minio
+  #       run: |
+  #         docker pull kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }}
+  #         docker tag kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }}
+  #         docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }}
+
+
+  # push-rqlite:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
+  #     # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+  #     # this action creates a branch based on remote branch and runs the tests
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: load environment variables from .image.env
+  #       uses: falti/dotenv-action@v1
+  #       id: dotenv
+  #       with:
+  #         path: .image.env
+
+  #     - name: push rqlite
+  #       run: |
+  #         docker pull kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
+  #         docker tag kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
+  #         docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
+
+  # push-dex:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci ]
+  #   steps:
+  #     # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
+  #     # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+  #     # this action creates a branch based on remote branch and runs the tests
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: load environment variables from .image.env
+  #       uses: falti/dotenv-action@v1
+  #       id: dotenv
+  #       with:
+  #         path: .image.env
+
+  #     - name: push dex
+  #       run: |
+  #         docker pull kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }}
+  #         docker tag kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }}
+  #         docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }}
+
+  # # only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**"
+  # kurl-addon-changes-filter:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ can-run-ci, enable-tests ]
+  #   outputs:
+  #     ok-to-test: ${{ steps.filter.outputs.kurl-addon }}
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: dorny/paths-filter@v2
+  #       id: filter
+  #       with:
+  #         filters: |
+  #           kurl-addon:
+  #             - 'deploy/kurl/kotsadm/template/**'
+  #             - 'deploy/kurl/kotsadm/testgrid-os-spec.yaml'
+  # validate-kurl-addon:
+  #   runs-on: ubuntu-20.04
+  #   if: ${{ needs.kurl-addon-changes-filter.outputs.ok-to-test == 'true' }}
+  #   needs: [ can-run-ci, enable-tests, generate-tag, kurl-addon-changes-filter, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-dex ]
+  #   steps:
+  #     - name: checkout
+  #       uses: actions/checkout@v4
+  #     - name: set outputs
+  #       id: vars
+  #       run: |
+  #         addon_version=${{ needs.generate-tag.outputs.tag }}
+  #         echo "addon_version=${addon_version#v}" >> "$GITHUB_OUTPUT"
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - name: prepare kots binary executable
+  #       run: |
+  #         chmod +x bin/*
+  #         tar -C bin/ -czvf bin/kots.tar.gz kots
+  #     - name: generate kurl add-on
+  #       id: addon-generate
+  #       uses: ./.github/actions/kurl-addon-kots-generate
+  #       env:
+  #         AWS_ACCESS_KEY_ID: ${{ secrets.KURL_ADDONS_AWS_ACCESS_KEY_ID }}
+  #         AWS_SECRET_ACCESS_KEY: ${{ secrets.KURL_ADDONS_AWS_SECRET_ACCESS_KEY }}
+  #         AWS_DEFAULT_REGION: us-east-1
+  #       with:
+  #         addon_version: ${{ steps.vars.outputs.addon_version }}
+  #         kotsadm_image_registry: ttl.sh
+  #         kotsadm_image_namespace: automated-${{ github.run_id }}
+  #         kotsadm_image_tag: 24h
+  #         kotsadm_binary_override: bin/kots.tar.gz
+  #     - name: test kurl add-on
+  #       id: addon-test
+  #       uses: ./.github/actions/kurl-addon-kots-test
+  #       with:
+  #         addon_version: ${{ steps.vars.outputs.addon_version }}
+  #         addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }}
+  #         testgrid_api_token: ${{ secrets.TESTGRID_PROD_API_TOKEN }}
+  #     - name: comment testgrid url
+  #       uses: mshick/add-pr-comment@v2
+  #       with:
+  #         message: ${{ steps.addon-test.outputs.testgrid_run_message }}
+  #         repo-token: ${{ secrets.GITHUB_TOKEN }}
+  #         allow-repeats: false
+
+
+  # cmx-versions:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci ]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+  #     - name: get CMX versions to test
+  #       id: cmx-versions-to-test
+  #       uses: ./.github/actions/cmx-versions
+  #       with:
+  #         replicated-api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #   outputs:
+  #     versions-to-test: ${{ steps.cmx-versions-to-test.outputs.versions-to-test }}
+
+
+  # validate-existing-online-install-minimal:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: |
+  #         docker load -i e2e/bin/e2e-deps.tar
+  #         chmod +x e2e/bin/*
+  #         chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         kots-namespace: 'qakotsregression'
+  #         test-focus: 'Regression'
+  #         k8s-distribution: k3s
+  #         k8s-version: v1.26
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-smoke-test:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
+  #   continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'Smoke Test'
+  #         kots-namespace: 'smoke-test'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-minimal-rbac:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0},
+  #         {distribution: openshift, version: 4.13.0-okd}
+  #       ]
+  #   env:
+  #     APP_SLUG: minimal-rbac
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h | tee output.txt
+
+  #         if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
+  #           echo "Expected to see an RBAC error for preflight checks, but did not" 
+  #           exit 1
+  #         fi
+
+  #         if ! grep -q 'The app was not deployed.' output.txt; then
+  #           printf "Expected to see message about app not being deployed, but did not\n"
+  #           exit 1
+  #         fi
+
+  #         if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then
+  #           printf "Did not expect to see a failure about number of nodes, but did\n"
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then
+  #           echo "kotsadm role not found in namespace $APP_SLUG"
+  #           kubectl get role -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then
+  #           echo "kotsadm rolebinding not found in namespace $APP_SLUG"
+  #           kubectl get rolebinding -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if kubectl get clusterrole | grep -q kotsadm; then
+  #           echo "found kotsadm clusterrole in minimal RBAC install"
+  #           kubectl get clusterrole
+  #           exit 1
+  #         fi
+
+  #         if kubectl get clusterrolebinding | grep -q kotsadm; then
+  #           echo "found kotsadm clusterrolebinding in minimal RBAC install"
+  #           kubectl get clusterrolebinding
+  #           exit 1
+  #         fi
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-backup-and-restore:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
+  #   continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'Backup and Restore'
+  #         kots-namespace: 'backup-and-restore'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-no-required-config:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'No Required Config'
+  #         kots-namespace: 'no-required-config'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-strict-preflight-checks:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   env:
+  #     APP_SLUG: strict-preflight-checks
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.STRICT_PREFLIGHT_CHECKS_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --skip-preflights \
+  #           --kotsadm-tag 24h | tee output.txt
+
+  #         if ! grep -q 'The app was not deployed.' output.txt; then
+  #           printf "Expected to see message about app not being deployed, but did not\n"
+  #           exit 1
+  #         fi
+
+  #         if ! grep FAIL output.txt | grep -q 'The application requires a Kubernetes 2.0.0 or later.'; then
+  #           printf "Expected to see a failure about kubernetes version, but did not\n"
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl logs deploy/kotsadm -n "$APP_SLUG" | grep -q "preflights will not be skipped, strict preflights are set to true"; then
+  #           echo "Failed to find a log line about strict preflights not being skipped in kotsadm logs"
+  #           echo "kotsadm logs:"
+  #           kubectl logs deploy/kotsadm -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # disable the strict preflight check and app should deploy successfully
+  #         ./bin/kots set config "$APP_SLUG" enable_failing_strict_analyzers="0" --namespace "$APP_SLUG" --deploy
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs deploy/kotsadm --tail=100 -n "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         printf "App is installed successfully and is ready\n\n"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-config:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'Config Validation'
+  #         kots-namespace: 'config-validation'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-version-history-pagination:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: k3s-local, version: v1.27.1-k3s1}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'Version History Pagination'
+  #         kots-namespace: 'version-history-pagination'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-change-license:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'Change License'
+  #         kots-namespace: 'change-license'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-minimal-rbac-override:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0},
+  #         {distribution: openshift, version: 4.13.0-okd}
+  #       ]
+  #   env:
+  #     APP_SLUG: minimal-rbac
+  #     APP_VERSION_LABEL: "0.0.1"
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: minimal rbac override on command line
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --app-version-label "$APP_VERSION_LABEL" \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h \
+  #           --use-minimal-rbac | tee output.txt
+
+  #         if ! grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
+  #           echo "Expected to see an RBAC error for preflight checks, but did not" 
+  #           exit 1
+  #         fi
+
+  #         if ! grep -q 'The app was not deployed.' output.txt; then
+  #           printf "Expected to see message about app not being deployed, but did not\n"
+  #           exit 1
+  #         fi
+
+  #         if grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then
+  #           printf "Did not expect to see a failure about number of nodes, but did\n"
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then
+  #           echo "kotsadm role not found in namespace $APP_SLUG"
+  #           kubectl get role -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then
+  #           echo "kotsadm rolebinding not found in namespace $APP_SLUG"
+  #           kubectl get rolebinding -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if kubectl get clusterrole | grep -q kotsadm; then
+  #           echo "found kotsadm clusterrole in minimal RBAC install"
+  #           kubectl get clusterrole
+  #           exit 1
+  #         fi
+
+  #         if kubectl get clusterrolebinding | grep -q kotsadm; then
+  #           echo "found kotsadm clusterrolebinding in minimal RBAC install"
+  #           kubectl get clusterrolebinding
+  #           exit 1
+  #         fi
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl delete ns "$APP_SLUG" --ignore-not-found
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: no minimal rbac override on command line
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.MINIMAL_RBAC_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --app-version-label "$APP_VERSION_LABEL" \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h | tee output.txt
+
+  #         if grep -q "The Kubernetes RBAC policy that the Admin Console is running with does not have access to complete the Preflight Checks. It's recommended that you run these manually before proceeding." output.txt; then
+  #           echo "Did not expect to see an RBAC error for preflight checks, but did" 
+  #           exit 1
+  #         fi
+
+  #         if ! grep -q 'The app was not deployed.' output.txt; then
+  #           printf "Expected to see message about app not being deployed, but did not\n"
+  #           exit 1
+  #         fi
+
+  #         if ! grep FAIL output.txt | grep -q 'This application requires at least 100 nodes'; then
+  #           printf "Expected to see a failure about number of nodes, but did not\n"
+  #           exit 1
+  #         fi
+
+  #         if kubectl get role -n "$APP_SLUG" | grep -q kotsadm; then
+  #           echo "kotsadm role found in cluster scoped install"
+  #           kubectl get role -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if kubectl get rolebinding -n "$APP_SLUG" | grep -q kotsadm; then
+  #           echo "kotsadm rolebinding found in cluster scoped install"
+  #           kubectl get rolebinding -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl get clusterrole | grep -q kotsadm; then
+  #           echo "No kotsadm clusterrole found in cluster scoped install"
+  #           kubectl get clusterrole
+  #           exit 1
+  #         fi
+
+  #         if ! kubectl get clusterrolebinding | grep -q kotsadm; then
+  #           echo "No kotsadm clusterrolebinding found in cluster scoped install"
+  #           kubectl get clusterrolebinding
+  #           exit 1
+  #         fi
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-multi-namespace:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
+  #   continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
+  #   env:
+  #     APP_SLUG: multi-namespace-yeti
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - uses: azure/setup-helm@v3
+  #       with:
+  #         token: ${{ secrets.GITHUB_TOKEN }}
+
+  #     - name: Create Cluster 
+  #       id: create-cluster
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 180 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly
+
+  #         if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then
+  #           printf "postgresql helm release not found in postgres-test namespace\n\n"
+  #           helm ls -n postgres-test
+  #           exit 1
+  #         fi
+
+  #         if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then
+  #           printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG"
+  #           helm ls -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
   
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-kots-pull:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
-    continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
-    env:
-      APP_NAME: multi-namespace-yeti
-      APP_SLUG: multi-namespace
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_NAME"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_NAME"
-
-      - name: run kots pull
-        run: |
-          set +e
-          echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots pull "$APP_NAME/automated" \
-            --license-file license.yaml \
-            --shared-password password \
-            --namespace "$APP_NAME" \
-            --exclude-admin-console
-
-          kubectl create ns "$APP_NAME"
-          kubectl create ns nginx-test
-          kubectl create ns redis-test
-          kubectl create ns postgres-test
-
-          # HACK: without operator, additonal namespaces don't get image pull secrets
-          echo ${{ secrets.MULTI_NAMESPACE_REGISTRY_AUTH }} | base64 -d > replicated-registry-auth.json
-          kubectl -n nginx-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
-          kubectl -n redis-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
-          kubectl -n redis-test create secret generic multi-namespace-yeti-redis-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
-          kubectl -n postgres-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
-          kubectl -n default create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
-
-          kustomize build "$PWD/$APP_SLUG/overlays/midstream" | kubectl apply -f -
-          kustomize build "$PWD/$APP_SLUG/overlays/midstream/charts/redis" | kubectl apply -f -
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "Failed to apply spec"
-            echo "------pods:"
-            kubectl get pods -A
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          echo "Waiting for pods to start"
-
-          COUNTER=1
-          while [ "$(kubectl get pods --no-headers | grep -v Running | grep -cv Completed)" -gt 0 ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for pods to start"
-              kubectl get pods -A
-              exit 1
-            fi
-            sleep 1
-          done
-
-          echo "All pods started"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_NAME"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-app-version-label:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: app-version-label
-      APP_VERSION_LABEL: v1.0.0
-      LATEST_APP_VERSION_LABEL: v1.0.1
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: test kots install with version label
-        run: |
-          set +e
-          echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --app-version-label "$APP_VERSION_LABEL" \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$APP_VERSION_LABEL" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be installed with correct version label: $APP_VERSION_LABEL"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          printf "App is installed successfully with the correct version label: %s\n\n" "$APP_VERSION_LABEL"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-          # test setting DockerHub credentials
-
-          set +e
-
-          # TODO: deploy and check secrets are actually created and images are pulled
-          ./bin/kots docker ensure-secret --dockerhub-username replicatedtests --dockerhub-password ${{ secrets.DOCKERHUB_RATELIMIT_PASSWORD }} -n "$APP_SLUG"
-          ./bin/kots download -n "$APP_SLUG" --slug "$APP_SLUG"
-          if grep "${APP_SLUG}-kotsadm-dockerhub" -w "./${APP_SLUG}/overlays/midstream/secret.yaml"; then
-            echo "Found DockerHub secret in ${APP_SLUG} latest version"
-          else
-            echo "No DockerHub secret found in appication namespace"
-            exit 1
-          fi
-
-      - name: remove the app
-        run: |
-          set +e
-          ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force
-
-      - name: test kots install without version label
-        run: |
-          set +e
-          echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$LATEST_APP_VERSION_LABEL" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be installed with latest version label: $LATEST_APP_VERSION_LABEL"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          printf "App is installed successfully with the correct version label: %s\n\n" "$LATEST_APP_VERSION_LABEL"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-helm-install-order:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    env:
-      APP_SLUG: helm-install-order
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --skip-preflights \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          printf "App is installed successfully and is ready\n\n"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-no-redeploy-on-restart:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: no-redeploy-on-restart
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.NO_REDEPLOY_ON_RESTART_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --skip-preflights \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          # wait for application job to be created
-          COUNTER=1
-          while [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" == "0" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 60 ]; then
-              echo "Timed out waiting for job to be created"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # delete the application job and restart the admin console
-          kubectl delete jobs -n "$APP_SLUG" --all
-          kubectl delete pods -l app=kotsadm -n "$APP_SLUG"
-
-          # wait for old kotsadm pod to terminate
-          COUNTER=1
-          while [ "$(kubectl get pods -l app=kotsadm -n "$APP_SLUG" | awk 'NR>1' | wc -l)" != "1" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 60 ]; then
-              echo "More than 1 kotsadm pod found"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # wait for new kotsadm pod to become ready
-          kubectl wait --for=condition=ready pod -l app=kotsadm -n "$APP_SLUG" --timeout=60s
-
-          # delay in case the app takes a bit to be deployed
-          sleep 20
-
-          # validate that the application wasn't re-deployed and the job wasn't re-created
-          if [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" != "0" ]; then
-              echo "App should not be re-deployed after restart"
-              exit 1
-          fi
-
-          printf "Success. App was not re-deployed after restart\n\n"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-kubernetes-installer-preflight:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: kubernetes-installer-preflight
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-
-          echo ${{ secrets.KUBERNETES_INSTALLER_PREFLIGHT_LICENSE }} | base64 -d > license.yaml
-
-          # Fake kurl installation using the crd and installer spec from the application manifests:
-
-          # Pull application manifests
-          ./bin/kots pull "$APP_SLUG/automated" --license-file license.yaml --shared-password password
-
-          # Apply installer crd
-          kubectl apply -f "$APP_SLUG/upstream/installer-crd.yaml"
-
-          # Wait for crd to be created
-          kubectl wait --for condition=established --timeout=60s crd/installers.cluster.kurl.sh
-
-          # Seems that the above does not always guarantee the crd exists? So just in case...
-          sleep 10
-
-          # Apply installer
-          kubectl apply -f "$APP_SLUG/upstream/installer.yaml"
-
-          # Create kurl-config configmap in kube-system
-          kubectl create cm kurl-config -n kube-system --from-literal=installer_id=7cc8094
-
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          # validate that preflight checks ran
-          JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}"
-          if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" | grep -c pending_preflight)" != "1" ]; then
-              echo "Preflight checks did not run"
-              exit 1
-          fi
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # try get apps without namespace (using kubeconfig)
-          # validate that output is the same as above
-          mkdir -p /tmp/.kube
-          sudo cp "$KUBECONFIG" /tmp/.kube/config
-          sudo chmod -R 777 /tmp/.kube
-          export KUBECONFIG=/tmp/.kube/config
-          kubectl config set-context --current --namespace="$APP_SLUG"
-          if [ "$(./bin/kots get apps | awk 'NR>1{print $2}')" != "ready" ]; then
-              echo "kots get apps output is not the same as above"
-              exit 1
-          fi
-
-          printf "App is installed successfully and is ready\n\n"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-kots-push-images-anonymous:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: run kots admin-console push-images
-        run: |
-          set +e
-          ./bin/kots admin-console push-images ./hack/tests/small.airgap ttl.sh/automated-${{ github.run_id }}
-
-
-  validate-kots-admin-console-generate-manifests:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: minimal-rbac
-      BASE_KOTS_VERSION: v1.72.0
-      NAMESPACE: generate-manifests
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: run kots admin-console generate-manifests without k8s context
-        run: |
-          set +e
-          ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password
-
-      - name: validate that ./admin-console exists and is not empty
-        run: |
-          set +e
-          if [ ! -d ./admin-console ]; then
-            echo "admin-console directory does not exist"
-            exit 1
-          fi
-          if [ -z "$(ls -A ./admin-console)" ]; then
-            echo "admin-console directory is empty"
-            exit 1
-          fi
-
-      - name: remove admin-console directory
-        run: rm -rf ./admin-console
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download base kots version
-        run: |
-          curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
-            && tar zxvf kots_linux_amd64.tar.gz \
-            && mv kots "kots-$BASE_KOTS_VERSION"
-
-      - name: create namespace
-        run: |
-          set +e
-          kubectl create namespace "$NAMESPACE"
-
-      - name: run kots admin-console generate-manifests using base kots version with k8s context
-        run: |
-          set +e
-          "./kots-$BASE_KOTS_VERSION" admin-console generate-manifests -n "$NAMESPACE" --shared-password password
-
-      - name: apply the generated manifests
-        run: |
-          set +e
-          kubectl apply -f ./admin-console -n "$NAMESPACE"
-
-      - name: wait for the kotsadm-minio-0 pod to be created
-        run: |
-          set +e
-          COUNTER=1
-          while ! kubectl get pods -n "$NAMESPACE" | grep -q kotsadm-minio-0; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 30 ]; then
-              echo "timed out waiting for kotsadm-minio-0 pod to be created"
-              exit 1
-            fi
-            sleep 1
-          done
-
-      - name: wait for kotsadm-minio-0 pod to be ready
-        run: |
-          set +e
-          kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"
-
-      - name: wait for the kotsadm deployment to be ready
-        run: |
-          set +e
-          kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"
-
-      - name: run kots admin-console generate-manifests using new kots version with k8s context
-        run: |
-          set +e
-          ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-      - name: validate that ./admin-console/minio-statefulset.yaml has initContainers since a migration is needed
-        run: |
-          set +e
-          if ! grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
-            echo "admin-console/minio-statefulset.yaml does not have initContainers"
-            exit 1
-          fi
-
-      - name: apply the generated manifests
-        run: |
-          set +e
-          kubectl apply -f ./admin-console -n "$NAMESPACE"
-
-      - name: wait for kotsadm-minio-0 pod to be ready
-        run: |
-          set +e
-          sleep 10
-          kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"
-
-      - name: wait for the kotsadm deployment to be ready
-        run: |
-          set +e
-          kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"
-
-      - name: run kots admin-console generate-manifests using new kots version with k8s context
-        run: |
-          set +e
-          ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-      - name: validate that ./admin-console/minio-statefulset.yaml does not have initContainers since a migration is not needed
-        run: |
-          set +e
-          if grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
-            echo "admin-console/minio-statefulset.yaml has initContainers"
-            exit 1
-          fi
-
-      - name: apply the generated manifests
-        run: |
-          set +e
-          kubectl apply -f ./admin-console -n "$NAMESPACE"
-
-      - name: wait for kotsadm-minio-0 pod to be ready
-        run: |
-          set +e
-          sleep 10
-          kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"
-
-      - name: wait for the kotsadm deployment to be ready
-        run: |
-          set +e
-          kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"
-
-      - name: print pods and logs on failure
-        if: failure()
-        run: |
-          echo "------pods:"
-          kubectl -n "$NAMESPACE" get pods
-          echo "------kotsadm logs"
-          kubectl logs -l app=kotsadm --tail=100 --namespace "$NAMESPACE"
-          echo "------kotsadm-minio logs"
-          kubectl logs -l app=kotsadm-minio --tail=100 --namespace "$NAMESPACE"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: automated-${{ github.run_id }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-min-kots-version:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
-    env:
-      APP_SLUG: min-kots-version
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: |
-          docker load -i e2e/bin/e2e-deps.tar
-          chmod +x e2e/bin/*
-          chmod +x bin/*
-          cp ./bin/kots /usr/local/bin/kubectl-kots
-          sudo apt-get update -y && sudo apt-get install jq -y
-      - uses: ./.github/actions/kots-e2e
-        id: kots-e2e
-        with:
-          test-focus: 'Min KOTS Version'
-          kots-namespace: 'min-kots-version'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          kotsadm-image-registry: ttl.sh
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-          k8s-cluster-skip-teardown: true
-
-      - name: validate that kots install fails early
-        run: |
-          set +e
-
-          result=$(kubectl kots install "$APP_SLUG/automated" --no-port-forward --namespace "$APP_SLUG" --shared-password password 2>&1 >/dev/null)
-          echo "$result"
-
-          if [[ "$result" == *"requires"* ]] && [[ "$result" == *"10000.0.0"* ]]; then
-            exit 0
-          else
-            exit 1
-          fi
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}
-
-
-  validate-target-kots-version:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: |
-          docker load -i e2e/bin/e2e-deps.tar
-          chmod +x e2e/bin/*
-          chmod +x bin/*
-          cp ./bin/kots /usr/local/bin/kubectl-kots
-          sudo apt-get update -y && sudo apt-get install jq -y
-      - uses: ./.github/actions/kots-e2e
-        id: kots-e2e
-        with:
-          test-focus: 'Target KOTS Version'
-          kots-namespace: 'target-kots-version'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-          k8s-cluster-skip-teardown: true
-
-      - name: validate that kots install fails early
-        run: |
-          set +e
-
-          result=$(kubectl kots install target-kots-version/automated --no-port-forward --namespace target-kots-version --shared-password password 2>&1 >/dev/null)
-          echo "$result"
-
-          if [[ "$result" == *"requires"* ]] && [[ "$result" == *"1.0.0"* ]]; then
-            exit 0
-          else
-            exit 1
-          fi
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}
-
-
-  validate-range-kots-version:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: |
-          docker load -i e2e/bin/e2e-deps.tar
-          chmod +x e2e/bin/*
-          chmod +x bin/*
-          cp ./bin/kots /usr/local/bin/kubectl-kots
-          sudo apt-get update -y && sudo apt-get install jq -y
-      - uses: ./.github/actions/kots-e2e
-        id: kots-e2e
-        with:
-          test-focus: 'Range KOTS Version'
-          kots-namespace: 'range-kots-version'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-          k8s-cluster-skip-teardown: true
-
-      - name: validate that kots install fails early
-        run: |
-          set +e
-
-          result=$(kubectl kots install range-kots-version/automated --no-port-forward --namespace range-kots-version --shared-password password 2>&1 >/dev/null)
-          echo "$result"
-
-          if [[ "$result" == *"requires"* ]] && [[ "$result" == *"11000.0.0"* ]]; then
-            exit 0
-          else
-            exit 1
-          fi
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}
-
-
-  validate-kots-upgrade:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
-    continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
-    env:
-      APP_SLUG: postgres-to-rqlite
-      BASE_KOTS_VERSION: v1.57.0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
-          export-kubeconfig: true
-
-      - name: download base kots version
-        run: |
-          curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
-            && tar zxvf kots_linux_amd64.tar.gz \
-            && mv kots "kots-$BASE_KOTS_VERSION"
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml
-
-          # install using the base KOTS version
-
-          "./kots-$BASE_KOTS_VERSION" \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --port-forward=false \
-            --namespace "$APP_SLUG" \
-            --shared-password password
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # upgrade using the new KOTS version
-
-          ./bin/kots admin-console upgrade \
-            --namespace "$APP_SLUG" \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          # verify that the postgres to rqlite migration was successful
-
-          if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then
-            echo "Failed to find a successful migration log line"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          # verify that the minio migration happened
-
-          if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then
-            echo "Failed to find initContainers in the kotsadm-minio statefulset"
-            echo "kotsadm-minio statefulset:"
-            kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml
-            exit 1
-          fi
-
-          # make sure app is still installed and ready
-
-          if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then
-            echo "App is not ready after the upgrade"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          printf "App is still installed and is ready after the migration\n\n"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-kots-helm-release-secret-migration:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: helm-release-secret-migration
-      RELEASE_NAME: helm-release-chart
-      RELEASE_NAMESPACE: helm-release
-      BASE_KOTS_VERSION: v1.94.0
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download base kots version
-        run: |
-          curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
-            && tar zxvf kots_linux_amd64.tar.gz \
-            && mv kots "kots-$BASE_KOTS_VERSION"
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.HELM_RELEASE_SECRET_MIGRATION_LICENSE }} | base64 -d > license.yaml
-
-          # install using the base KOTS version
-          "./kots-$BASE_KOTS_VERSION" \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward=true \
-            --namespace "$APP_SLUG" \
-            --shared-password password
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # verify that the helm release secret is created in the kotsadm namespace
-          releaseSecretName=$(kubectl get secret -n "$APP_SLUG" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
-          if [ -z "$releaseSecretName" ]; then
-            echo "Failed to find the helm release secret in the $APP_SLUG namespace"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          # if there are more than one helm release secrets, fail
-          if [ "$(echo "$releaseSecretName" | wc -l)" -gt 1 ]; then
-            echo "Found more than one helm release secret in the $APP_SLUG namespace"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          # upgrade using the new KOTS version
-          ./bin/kots admin-console upgrade \
-            --namespace "$APP_SLUG" \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          # make a config change and redeploy the app
-          ./bin/kots set config "$APP_SLUG" create_new_sequence=true --deploy --namespace "$APP_SLUG"
-
-          # make sure app is still installed and ready
-          if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then
-            echo "App is not ready after the upgrade"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          COUNTER=1
-          while [ "$(helm ls -n "$RELEASE_NAMESPACE" | grep "$RELEASE_NAME" | awk 'NR>0{print $3}')" != "2" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for $RELEASE_NAME to be upgraded to revision 2"
-              helm ls -n "$RELEASE_NAMESPACE"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # verify that the helm release secret is created in the helm relase namespace
-          COUNT=1
-          releaseSecret=""
-          while [ -z "$releaseSecret" ]; do
-            ((COUNT += 1))
-            if [ $COUNT -gt 10 ]; then
-              echo "Timed out waiting for the helm release secret to be migrated"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-            releaseSecret=$(kubectl get secret "$releaseSecretName" -n "$RELEASE_NAMESPACE")
-          done
-
-          # verify that the release secret in app namepspace is deleted
-          oldReleaseSecret=$(kubectl get secret "$releaseSecretName" -n "$APP_SLUG")
-          if [ -n "$oldReleaseSecret" ]; then
-            echo "Found the helm release secret in the $APP_SLUG namespace"
-            echo "$oldReleaseSecret"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          # verify that there are two helm release secrets in the helm release namespace
-          releaseSecretNames=$(kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
-          releaseSecretNamesCount=$(echo "$releaseSecretNames" | wc -w)
-          if [ "$releaseSecretNamesCount" -ne 2 ]; then
-            echo "Found $releaseSecretNamesCount helm release secrets in the $RELEASE_NAMESPACE namespace(Want: 2)"
-            kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME"
-            echo "kotsadm logs:"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit 1
-          fi
-
-          printf "Helm release secret migration test passed\n\n"
-          ./bin/kots get apps --namespace "$APP_SLUG"
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-multi-app-backup-and-restore:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'multi-app-backup-and-restore'
-          kots-namespace: 'multi-app-backup-and-restore'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-multi-app-install:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'multi-app-install'
-          kots-namespace: 'multi-app-install'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-airgap-smoke-test:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'airgap-smoke-test'
-          kots-namespace: 'airgap-smoke-test'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          kots-airgap: true
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-remove-app:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0, instance-type: r1.medium}
-        ]
-    env:
-      APP_SLUG: remove-app
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          instance-type: ${{ matrix.cluster.instance-type }}
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-kots-pull:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
+  #   continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
+  #   env:
+  #     APP_NAME: multi-namespace-yeti
+  #     APP_SLUG: multi-namespace
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_NAME"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_NAME"
+
+  #     - name: run kots pull
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots pull "$APP_NAME/automated" \
+  #           --license-file license.yaml \
+  #           --shared-password password \
+  #           --namespace "$APP_NAME" \
+  #           --exclude-admin-console
+
+  #         kubectl create ns "$APP_NAME"
+  #         kubectl create ns nginx-test
+  #         kubectl create ns redis-test
+  #         kubectl create ns postgres-test
+
+  #         # HACK: without operator, additonal namespaces don't get image pull secrets
+  #         echo ${{ secrets.MULTI_NAMESPACE_REGISTRY_AUTH }} | base64 -d > replicated-registry-auth.json
+  #         kubectl -n nginx-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
+  #         kubectl -n redis-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
+  #         kubectl -n redis-test create secret generic multi-namespace-yeti-redis-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
+  #         kubectl -n postgres-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
+  #         kubectl -n default create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
+
+  #         kustomize build "$PWD/$APP_SLUG/overlays/midstream" | kubectl apply -f -
+  #         kustomize build "$PWD/$APP_SLUG/overlays/midstream/charts/redis" | kubectl apply -f -
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "Failed to apply spec"
+  #           echo "------pods:"
+  #           kubectl get pods -A
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         echo "Waiting for pods to start"
+
+  #         COUNTER=1
+  #         while [ "$(kubectl get pods --no-headers | grep -v Running | grep -cv Completed)" -gt 0 ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for pods to start"
+  #             kubectl get pods -A
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         echo "All pods started"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_NAME"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-app-version-label:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: app-version-label
+  #     APP_VERSION_LABEL: v1.0.0
+  #     LATEST_APP_VERSION_LABEL: v1.0.1
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: test kots install with version label
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --app-version-label "$APP_VERSION_LABEL" \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$APP_VERSION_LABEL" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be installed with correct version label: $APP_VERSION_LABEL"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         printf "App is installed successfully with the correct version label: %s\n\n" "$APP_VERSION_LABEL"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #         # test setting DockerHub credentials
+
+  #         set +e
+
+  #         # TODO: deploy and check secrets are actually created and images are pulled
+  #         ./bin/kots docker ensure-secret --dockerhub-username replicatedtests --dockerhub-password ${{ secrets.DOCKERHUB_RATELIMIT_PASSWORD }} -n "$APP_SLUG"
+  #         ./bin/kots download -n "$APP_SLUG" --slug "$APP_SLUG"
+  #         if grep "${APP_SLUG}-kotsadm-dockerhub" -w "./${APP_SLUG}/overlays/midstream/secret.yaml"; then
+  #           echo "Found DockerHub secret in ${APP_SLUG} latest version"
+  #         else
+  #           echo "No DockerHub secret found in appication namespace"
+  #           exit 1
+  #         fi
+
+  #     - name: remove the app
+  #       run: |
+  #         set +e
+  #         ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force
+
+  #     - name: test kots install without version label
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$LATEST_APP_VERSION_LABEL" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be installed with latest version label: $LATEST_APP_VERSION_LABEL"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         printf "App is installed successfully with the correct version label: %s\n\n" "$LATEST_APP_VERSION_LABEL"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-helm-install-order:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   env:
+  #     APP_SLUG: helm-install-order
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --skip-preflights \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         printf "App is installed successfully and is ready\n\n"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-no-redeploy-on-restart:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: no-redeploy-on-restart
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.NO_REDEPLOY_ON_RESTART_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --skip-preflights \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         # wait for application job to be created
+  #         COUNTER=1
+  #         while [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" == "0" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 60 ]; then
+  #             echo "Timed out waiting for job to be created"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # delete the application job and restart the admin console
+  #         kubectl delete jobs -n "$APP_SLUG" --all
+  #         kubectl delete pods -l app=kotsadm -n "$APP_SLUG"
+
+  #         # wait for old kotsadm pod to terminate
+  #         COUNTER=1
+  #         while [ "$(kubectl get pods -l app=kotsadm -n "$APP_SLUG" | awk 'NR>1' | wc -l)" != "1" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 60 ]; then
+  #             echo "More than 1 kotsadm pod found"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # wait for new kotsadm pod to become ready
+  #         kubectl wait --for=condition=ready pod -l app=kotsadm -n "$APP_SLUG" --timeout=60s
+
+  #         # delay in case the app takes a bit to be deployed
+  #         sleep 20
+
+  #         # validate that the application wasn't re-deployed and the job wasn't re-created
+  #         if [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" != "0" ]; then
+  #             echo "App should not be re-deployed after restart"
+  #             exit 1
+  #         fi
+
+  #         printf "Success. App was not re-deployed after restart\n\n"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-kubernetes-installer-preflight:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: kubernetes-installer-preflight
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+
+  #         echo ${{ secrets.KUBERNETES_INSTALLER_PREFLIGHT_LICENSE }} | base64 -d > license.yaml
+
+  #         # Fake kurl installation using the crd and installer spec from the application manifests:
+
+  #         # Pull application manifests
+  #         ./bin/kots pull "$APP_SLUG/automated" --license-file license.yaml --shared-password password
+
+  #         # Apply installer crd
+  #         kubectl apply -f "$APP_SLUG/upstream/installer-crd.yaml"
+
+  #         # Wait for crd to be created
+  #         kubectl wait --for condition=established --timeout=60s crd/installers.cluster.kurl.sh
+
+  #         # Seems that the above does not always guarantee the crd exists? So just in case...
+  #         sleep 10
+
+  #         # Apply installer
+  #         kubectl apply -f "$APP_SLUG/upstream/installer.yaml"
+
+  #         # Create kurl-config configmap in kube-system
+  #         kubectl create cm kurl-config -n kube-system --from-literal=installer_id=7cc8094
+
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         # validate that preflight checks ran
+  #         JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}"
+  #         if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" | grep -c pending_preflight)" != "1" ]; then
+  #             echo "Preflight checks did not run"
+  #             exit 1
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # try get apps without namespace (using kubeconfig)
+  #         # validate that output is the same as above
+  #         mkdir -p /tmp/.kube
+  #         sudo cp "$KUBECONFIG" /tmp/.kube/config
+  #         sudo chmod -R 777 /tmp/.kube
+  #         export KUBECONFIG=/tmp/.kube/config
+  #         kubectl config set-context --current --namespace="$APP_SLUG"
+  #         if [ "$(./bin/kots get apps | awk 'NR>1{print $2}')" != "ready" ]; then
+  #             echo "kots get apps output is not the same as above"
+  #             exit 1
+  #         fi
+
+  #         printf "App is installed successfully and is ready\n\n"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-kots-push-images-anonymous:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: run kots admin-console push-images
+  #       run: |
+  #         set +e
+  #         ./bin/kots admin-console push-images ./hack/tests/small.airgap ttl.sh/automated-${{ github.run_id }}
+
+
+  # validate-kots-admin-console-generate-manifests:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: minimal-rbac
+  #     BASE_KOTS_VERSION: v1.72.0
+  #     NAMESPACE: generate-manifests
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: run kots admin-console generate-manifests without k8s context
+  #       run: |
+  #         set +e
+  #         ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password
+
+  #     - name: validate that ./admin-console exists and is not empty
+  #       run: |
+  #         set +e
+  #         if [ ! -d ./admin-console ]; then
+  #           echo "admin-console directory does not exist"
+  #           exit 1
+  #         fi
+  #         if [ -z "$(ls -A ./admin-console)" ]; then
+  #           echo "admin-console directory is empty"
+  #           exit 1
+  #         fi
+
+  #     - name: remove admin-console directory
+  #       run: rm -rf ./admin-console
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download base kots version
+  #       run: |
+  #         curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
+  #           && tar zxvf kots_linux_amd64.tar.gz \
+  #           && mv kots "kots-$BASE_KOTS_VERSION"
+
+  #     - name: create namespace
+  #       run: |
+  #         set +e
+  #         kubectl create namespace "$NAMESPACE"
+
+  #     - name: run kots admin-console generate-manifests using base kots version with k8s context
+  #       run: |
+  #         set +e
+  #         "./kots-$BASE_KOTS_VERSION" admin-console generate-manifests -n "$NAMESPACE" --shared-password password
+
+  #     - name: apply the generated manifests
+  #       run: |
+  #         set +e
+  #         kubectl apply -f ./admin-console -n "$NAMESPACE"
+
+  #     - name: wait for the kotsadm-minio-0 pod to be created
+  #       run: |
+  #         set +e
+  #         COUNTER=1
+  #         while ! kubectl get pods -n "$NAMESPACE" | grep -q kotsadm-minio-0; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 30 ]; then
+  #             echo "timed out waiting for kotsadm-minio-0 pod to be created"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #     - name: wait for kotsadm-minio-0 pod to be ready
+  #       run: |
+  #         set +e
+  #         kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"
+
+  #     - name: wait for the kotsadm deployment to be ready
+  #       run: |
+  #         set +e
+  #         kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"
+
+  #     - name: run kots admin-console generate-manifests using new kots version with k8s context
+  #       run: |
+  #         set +e
+  #         ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #     - name: validate that ./admin-console/minio-statefulset.yaml has initContainers since a migration is needed
+  #       run: |
+  #         set +e
+  #         if ! grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
+  #           echo "admin-console/minio-statefulset.yaml does not have initContainers"
+  #           exit 1
+  #         fi
+
+  #     - name: apply the generated manifests
+  #       run: |
+  #         set +e
+  #         kubectl apply -f ./admin-console -n "$NAMESPACE"
+
+  #     - name: wait for kotsadm-minio-0 pod to be ready
+  #       run: |
+  #         set +e
+  #         sleep 10
+  #         kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"
+
+  #     - name: wait for the kotsadm deployment to be ready
+  #       run: |
+  #         set +e
+  #         kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"
+
+  #     - name: run kots admin-console generate-manifests using new kots version with k8s context
+  #       run: |
+  #         set +e
+  #         ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #     - name: validate that ./admin-console/minio-statefulset.yaml does not have initContainers since a migration is not needed
+  #       run: |
+  #         set +e
+  #         if grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
+  #           echo "admin-console/minio-statefulset.yaml has initContainers"
+  #           exit 1
+  #         fi
+
+  #     - name: apply the generated manifests
+  #       run: |
+  #         set +e
+  #         kubectl apply -f ./admin-console -n "$NAMESPACE"
+
+  #     - name: wait for kotsadm-minio-0 pod to be ready
+  #       run: |
+  #         set +e
+  #         sleep 10
+  #         kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"
+
+  #     - name: wait for the kotsadm deployment to be ready
+  #       run: |
+  #         set +e
+  #         kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"
+
+  #     - name: print pods and logs on failure
+  #       if: failure()
+  #       run: |
+  #         echo "------pods:"
+  #         kubectl -n "$NAMESPACE" get pods
+  #         echo "------kotsadm logs"
+  #         kubectl logs -l app=kotsadm --tail=100 --namespace "$NAMESPACE"
+  #         echo "------kotsadm-minio logs"
+  #         kubectl logs -l app=kotsadm-minio --tail=100 --namespace "$NAMESPACE"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: automated-${{ github.run_id }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-min-kots-version:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
+  #   env:
+  #     APP_SLUG: min-kots-version
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: |
+  #         docker load -i e2e/bin/e2e-deps.tar
+  #         chmod +x e2e/bin/*
+  #         chmod +x bin/*
+  #         cp ./bin/kots /usr/local/bin/kubectl-kots
+  #         sudo apt-get update -y && sudo apt-get install jq -y
+  #     - uses: ./.github/actions/kots-e2e
+  #       id: kots-e2e
+  #       with:
+  #         test-focus: 'Min KOTS Version'
+  #         kots-namespace: 'min-kots-version'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         kotsadm-image-registry: ttl.sh
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+  #         k8s-cluster-skip-teardown: true
+
+  #     - name: validate that kots install fails early
+  #       run: |
+  #         set +e
+
+  #         result=$(kubectl kots install "$APP_SLUG/automated" --no-port-forward --namespace "$APP_SLUG" --shared-password password 2>&1 >/dev/null)
+  #         echo "$result"
+
+  #         if [[ "$result" == *"requires"* ]] && [[ "$result" == *"10000.0.0"* ]]; then
+  #           exit 0
+  #         else
+  #           exit 1
+  #         fi
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}
+
+
+  # validate-target-kots-version:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: |
+  #         docker load -i e2e/bin/e2e-deps.tar
+  #         chmod +x e2e/bin/*
+  #         chmod +x bin/*
+  #         cp ./bin/kots /usr/local/bin/kubectl-kots
+  #         sudo apt-get update -y && sudo apt-get install jq -y
+  #     - uses: ./.github/actions/kots-e2e
+  #       id: kots-e2e
+  #       with:
+  #         test-focus: 'Target KOTS Version'
+  #         kots-namespace: 'target-kots-version'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+  #         k8s-cluster-skip-teardown: true
+
+  #     - name: validate that kots install fails early
+  #       run: |
+  #         set +e
+
+  #         result=$(kubectl kots install target-kots-version/automated --no-port-forward --namespace target-kots-version --shared-password password 2>&1 >/dev/null)
+  #         echo "$result"
+
+  #         if [[ "$result" == *"requires"* ]] && [[ "$result" == *"1.0.0"* ]]; then
+  #           exit 0
+  #         else
+  #           exit 1
+  #         fi
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}
+
+
+  # validate-range-kots-version:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-e2e, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: |
+  #         docker load -i e2e/bin/e2e-deps.tar
+  #         chmod +x e2e/bin/*
+  #         chmod +x bin/*
+  #         cp ./bin/kots /usr/local/bin/kubectl-kots
+  #         sudo apt-get update -y && sudo apt-get install jq -y
+  #     - uses: ./.github/actions/kots-e2e
+  #       id: kots-e2e
+  #       with:
+  #         test-focus: 'Range KOTS Version'
+  #         kots-namespace: 'range-kots-version'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+  #         k8s-cluster-skip-teardown: true
+
+  #     - name: validate that kots install fails early
+  #       run: |
+  #         set +e
+
+  #         result=$(kubectl kots install range-kots-version/automated --no-port-forward --namespace range-kots-version --shared-password password 2>&1 >/dev/null)
+  #         echo "$result"
+
+  #         if [[ "$result" == *"requires"* ]] && [[ "$result" == *"11000.0.0"* ]]; then
+  #           exit 0
+  #         else
+  #           exit 1
+  #         fi
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}
+
+
+  # validate-kots-upgrade:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: ${{ fromJson(needs.cmx-versions.outputs.versions-to-test) }}
+  #   continue-on-error: ${{ matrix.cluster.stage != 'stable' }}
+  #   env:
+  #     APP_SLUG: postgres-to-rqlite
+  #     BASE_KOTS_VERSION: v1.57.0
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         instance-type: ${{ matrix.cluster.distribution == 'gke' && 'n2-standard-4' || '' }}
+  #         export-kubeconfig: true
+
+  #     - name: download base kots version
+  #       run: |
+  #         curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
+  #           && tar zxvf kots_linux_amd64.tar.gz \
+  #           && mv kots "kots-$BASE_KOTS_VERSION"
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml
+
+  #         # install using the base KOTS version
+
+  #         "./kots-$BASE_KOTS_VERSION" \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --port-forward=false \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # upgrade using the new KOTS version
+
+  #         ./bin/kots admin-console upgrade \
+  #           --namespace "$APP_SLUG" \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         # verify that the postgres to rqlite migration was successful
+
+  #         if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then
+  #           echo "Failed to find a successful migration log line"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # verify that the minio migration happened
+
+  #         if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then
+  #           echo "Failed to find initContainers in the kotsadm-minio statefulset"
+  #           echo "kotsadm-minio statefulset:"
+  #           kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml
+  #           exit 1
+  #         fi
+
+  #         # make sure app is still installed and ready
+
+  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then
+  #           echo "App is not ready after the upgrade"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         printf "App is still installed and is ready after the migration\n\n"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-kots-helm-release-secret-migration:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: helm-release-secret-migration
+  #     RELEASE_NAME: helm-release-chart
+  #     RELEASE_NAMESPACE: helm-release
+  #     BASE_KOTS_VERSION: v1.94.0
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download base kots version
+  #       run: |
+  #         curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
+  #           && tar zxvf kots_linux_amd64.tar.gz \
+  #           && mv kots "kots-$BASE_KOTS_VERSION"
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.HELM_RELEASE_SECRET_MIGRATION_LICENSE }} | base64 -d > license.yaml
+
+  #         # install using the base KOTS version
+  #         "./kots-$BASE_KOTS_VERSION" \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward=true \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # verify that the helm release secret is created in the kotsadm namespace
+  #         releaseSecretName=$(kubectl get secret -n "$APP_SLUG" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
+  #         if [ -z "$releaseSecretName" ]; then
+  #           echo "Failed to find the helm release secret in the $APP_SLUG namespace"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # if there are more than one helm release secrets, fail
+  #         if [ "$(echo "$releaseSecretName" | wc -l)" -gt 1 ]; then
+  #           echo "Found more than one helm release secret in the $APP_SLUG namespace"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # upgrade using the new KOTS version
+  #         ./bin/kots admin-console upgrade \
+  #           --namespace "$APP_SLUG" \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         # make a config change and redeploy the app
+  #         ./bin/kots set config "$APP_SLUG" create_new_sequence=true --deploy --namespace "$APP_SLUG"
+
+  #         # make sure app is still installed and ready
+  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then
+  #           echo "App is not ready after the upgrade"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(helm ls -n "$RELEASE_NAMESPACE" | grep "$RELEASE_NAME" | awk 'NR>0{print $3}')" != "2" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for $RELEASE_NAME to be upgraded to revision 2"
+  #             helm ls -n "$RELEASE_NAMESPACE"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # verify that the helm release secret is created in the helm relase namespace
+  #         COUNT=1
+  #         releaseSecret=""
+  #         while [ -z "$releaseSecret" ]; do
+  #           ((COUNT += 1))
+  #           if [ $COUNT -gt 10 ]; then
+  #             echo "Timed out waiting for the helm release secret to be migrated"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #           releaseSecret=$(kubectl get secret "$releaseSecretName" -n "$RELEASE_NAMESPACE")
+  #         done
+
+  #         # verify that the release secret in app namepspace is deleted
+  #         oldReleaseSecret=$(kubectl get secret "$releaseSecretName" -n "$APP_SLUG")
+  #         if [ -n "$oldReleaseSecret" ]; then
+  #           echo "Found the helm release secret in the $APP_SLUG namespace"
+  #           echo "$oldReleaseSecret"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # verify that there are two helm release secrets in the helm release namespace
+  #         releaseSecretNames=$(kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
+  #         releaseSecretNamesCount=$(echo "$releaseSecretNames" | wc -w)
+  #         if [ "$releaseSecretNamesCount" -ne 2 ]; then
+  #           echo "Found $releaseSecretNamesCount helm release secrets in the $RELEASE_NAMESPACE namespace(Want: 2)"
+  #           kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME"
+  #           echo "kotsadm logs:"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         printf "Helm release secret migration test passed\n\n"
+  #         ./bin/kots get apps --namespace "$APP_SLUG"
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-multi-app-backup-and-restore:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'multi-app-backup-and-restore'
+  #         kots-namespace: 'multi-app-backup-and-restore'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-multi-app-install:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'multi-app-install'
+  #         kots-namespace: 'multi-app-install'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-airgap-smoke-test:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'airgap-smoke-test'
+  #         kots-namespace: 'airgap-smoke-test'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         kots-airgap: true
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-remove-app:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0, instance-type: r1.medium}
+  #       ]
+  #   env:
+  #     APP_SLUG: remove-app
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         instance-type: ${{ matrix.cluster.instance-type }}
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
           
-          echo ${{ secrets.REMOVE_APP_LICENSE }} | base64 -d > license.yaml
-          HELM_APP_NAMESPACES=(postgres-test redis-test "$APP_SLUG")
-          APP_NAMESPACES=(postgres-test redis-test nginx-test rabbitmq-system "$APP_SLUG")
-
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # test that --undeploy deletes application resources from the cluster
-
-          ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --undeploy
-
-          if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
-            printf "App reference was not removed\n\n"
-            exit 1
-          fi
-
-          if [ "$(kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
-            printf "Application resources are still found in the cluster\n\n"
-            kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG"
-            exit 1
-          fi
-
-          if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
-            printf "rabbitmqclusters.rabbitmq.com CRD was not removed\n\n"
-            kubectl get crd rabbitmqclusters.rabbitmq.com
-            exit 1
-          fi
-
-          for ns in "${HELM_APP_NAMESPACES[@]}"; do
-            if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
-              printf "Application Helm release secrets are still found in the namepspace %s\n\n" "$ns"
-              kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
-              exit 1
-            fi
-          done
-
-          # re-install and test that running the remove command without --undeploy does _not_ delete any resources from the cluster
-
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force
-
-          if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
-            printf "App reference was not removed\n\n"
-            exit 1
-          fi
-
-          for ns in "${APP_NAMESPACES[@]}"; do
-            if [ "$(kubectl get all,secrets,configmap,pvc -n "$ns" -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
-              printf "%s namespace does not contain any resources\n\n" "$ns"
-              kubectl get all,secrets,configmap,pvc -n "$ns"
-              exit 1
-            fi
-          done
-
-          if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
-            printf "rabbitmqclusters.rabbitmq.com CRD was not found\n\n"
-            kubectl get crd rabbitmqclusters.rabbitmq.com
-            exit 1
-          fi
-
-          if [ "$(kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
-            printf "rabbitmq-system namespace does not contain rabbitmqclusters\n\n"
-            kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system
-            exit 1
-          fi
-
-          for ns in "${HELM_APP_NAMESPACES[@]}"; do
-            if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
-              printf "Application Helm release secrets are not found in namepspace %s\n\n" "$ns"
-              kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
-              exit 1
-            fi
-          done
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-registry-check:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: run the test
-        run: |
-          set +e
-
-          ./bin/kots admin-console push-images \
-            not-an-app.airgap localhost:1234/not-a-namespace \
-            --registry-username not-a-username \
-            --registry-password not-a-password > output.txt 2>&1
-
-          if ! grep -q 'Failed to test access' output.txt; then
-            printf "Expected registry validation to fail before pushing images, but did not.\n\n"
-            cat output.txt
-            exit 1
-          fi
-
-          rm output.txt
-
-          ./bin/kots install not-an-app \
-            --airgap-bundle not-an-app.airgap \
-            --kotsadm-registry localhost:1234/not-a-namespace \
-            --registry-username not-a-username \
-            --registry-password not-a-password > output.txt 2>&1
-
-          if ! grep -q 'Failed to test access' output.txt; then
-            printf "Expected registry validation to fail before installation, but did not.\n\n"
-            cat output.txt
-            exit 1
-          fi
-
-          rm output.txt
-
-          ./bin/kots upstream upgrade not-an-app \
-            --airgap-bundle not-an-app.airgap \
-            --kotsadm-registry localhost:1234/not-a-namespace \
-            --registry-username not-a-username \
-            --registry-password not-a-password \
-            --namespace not-a-namespace > output.txt 2>&1
-
-          if ! grep -q 'Failed to test access' output.txt; then
-            printf "Expected registry validation to fail before upgrading, but did not.\n\n"
-            cat output.txt
-            exit 1
-          fi
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-native-helm-v2:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: native-helm-v2
-      INITIAL_VERSION: '0.1.1'
-      UPGRADE_VERSION: '0.2.1'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - uses: azure/setup-helm@v3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.NATIVE_HELM_V2_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --app-version-label "$INITIAL_VERSION" \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$INITIAL_VERSION" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # validate that helm charts installed using both native helm workflow were deployed via the helm CLI correctly
-
-          if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-chart-release; then
-            printf "my-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
-            helm ls -n "$APP_SLUG"
-            exit 1
-          fi
-
-          if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-other-chart-release; then
-            printf "my-other-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
-            helm ls -n "$APP_SLUG"
-            exit 1
-          fi
-
-          # validate that `helm get values` works for the v1beta2 chart
-
-          if ! helm get values my-chart-release -n "$APP_SLUG" | grep -q my-value; then
-            printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
-            helm get values my-chart-release -n "$APP_SLUG"
-            exit 1
-          fi
-
-          # upgrade the app version
-
-          ./bin/kots upstream upgrade "$APP_SLUG" -n "$APP_SLUG" --deploy
-
-          # wait for my-chart-release to be uninstalled and my-other-chart-release to be upgraded to revision 2
+  #         echo ${{ secrets.REMOVE_APP_LICENSE }} | base64 -d > license.yaml
+  #         HELM_APP_NAMESPACES=(postgres-test redis-test "$APP_SLUG")
+  #         APP_NAMESPACES=(postgres-test redis-test nginx-test rabbitmq-system "$APP_SLUG")
+
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # test that --undeploy deletes application resources from the cluster
+
+  #         ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --undeploy
+
+  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
+  #           printf "App reference was not removed\n\n"
+  #           exit 1
+  #         fi
+
+  #         if [ "$(kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
+  #           printf "Application resources are still found in the cluster\n\n"
+  #           kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
+  #           printf "rabbitmqclusters.rabbitmq.com CRD was not removed\n\n"
+  #           kubectl get crd rabbitmqclusters.rabbitmq.com
+  #           exit 1
+  #         fi
+
+  #         for ns in "${HELM_APP_NAMESPACES[@]}"; do
+  #           if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
+  #             printf "Application Helm release secrets are still found in the namepspace %s\n\n" "$ns"
+  #             kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
+  #             exit 1
+  #           fi
+  #         done
+
+  #         # re-install and test that running the remove command without --undeploy does _not_ delete any resources from the cluster
+
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force
+
+  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
+  #           printf "App reference was not removed\n\n"
+  #           exit 1
+  #         fi
+
+  #         for ns in "${APP_NAMESPACES[@]}"; do
+  #           if [ "$(kubectl get all,secrets,configmap,pvc -n "$ns" -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
+  #             printf "%s namespace does not contain any resources\n\n" "$ns"
+  #             kubectl get all,secrets,configmap,pvc -n "$ns"
+  #             exit 1
+  #           fi
+  #         done
+
+  #         if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
+  #           printf "rabbitmqclusters.rabbitmq.com CRD was not found\n\n"
+  #           kubectl get crd rabbitmqclusters.rabbitmq.com
+  #           exit 1
+  #         fi
+
+  #         if [ "$(kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
+  #           printf "rabbitmq-system namespace does not contain rabbitmqclusters\n\n"
+  #           kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system
+  #           exit 1
+  #         fi
+
+  #         for ns in "${HELM_APP_NAMESPACES[@]}"; do
+  #           if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
+  #             printf "Application Helm release secrets are not found in namepspace %s\n\n" "$ns"
+  #             kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
+  #             exit 1
+  #           fi
+  #         done
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-registry-check:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+
+  #         ./bin/kots admin-console push-images \
+  #           not-an-app.airgap localhost:1234/not-a-namespace \
+  #           --registry-username not-a-username \
+  #           --registry-password not-a-password > output.txt 2>&1
+
+  #         if ! grep -q 'Failed to test access' output.txt; then
+  #           printf "Expected registry validation to fail before pushing images, but did not.\n\n"
+  #           cat output.txt
+  #           exit 1
+  #         fi
+
+  #         rm output.txt
+
+  #         ./bin/kots install not-an-app \
+  #           --airgap-bundle not-an-app.airgap \
+  #           --kotsadm-registry localhost:1234/not-a-namespace \
+  #           --registry-username not-a-username \
+  #           --registry-password not-a-password > output.txt 2>&1
+
+  #         if ! grep -q 'Failed to test access' output.txt; then
+  #           printf "Expected registry validation to fail before installation, but did not.\n\n"
+  #           cat output.txt
+  #           exit 1
+  #         fi
+
+  #         rm output.txt
+
+  #         ./bin/kots upstream upgrade not-an-app \
+  #           --airgap-bundle not-an-app.airgap \
+  #           --kotsadm-registry localhost:1234/not-a-namespace \
+  #           --registry-username not-a-username \
+  #           --registry-password not-a-password \
+  #           --namespace not-a-namespace > output.txt 2>&1
+
+  #         if ! grep -q 'Failed to test access' output.txt; then
+  #           printf "Expected registry validation to fail before upgrading, but did not.\n\n"
+  #           cat output.txt
+  #           exit 1
+  #         fi
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-native-helm-v2:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: native-helm-v2
+  #     INITIAL_VERSION: '0.1.1'
+  #     UPGRADE_VERSION: '0.2.1'
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - uses: azure/setup-helm@v3
+  #       with:
+  #         token: ${{ secrets.GITHUB_TOKEN }}
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.NATIVE_HELM_V2_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --app-version-label "$INITIAL_VERSION" \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$INITIAL_VERSION" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # validate that helm charts installed using both native helm workflow were deployed via the helm CLI correctly
+
+  #         if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-chart-release; then
+  #           printf "my-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
+  #           helm ls -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-other-chart-release; then
+  #           printf "my-other-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
+  #           helm ls -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # validate that `helm get values` works for the v1beta2 chart
+
+  #         if ! helm get values my-chart-release -n "$APP_SLUG" | grep -q my-value; then
+  #           printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
+  #           helm get values my-chart-release -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # upgrade the app version
+
+  #         ./bin/kots upstream upgrade "$APP_SLUG" -n "$APP_SLUG" --deploy
+
+  #         # wait for my-chart-release to be uninstalled and my-other-chart-release to be upgraded to revision 2
           
-          COUNTER=1
-          while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-chart-release)" != "" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for my-chart-release to be uninstalled"
-              helm ls -n "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          COUNTER=1
-          while [ "$(helm ls -n "$APP_SLUG" | grep my-other-chart-release | awk 'NR>0{print $3}')" != "2" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for my-other-chart-release to be upgraded to revision 2"
-              helm ls -n "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # validate that `helm get values` works for the upgraded v1beta2 chart
-          if ! helm get values my-other-chart-release -n "$APP_SLUG" | grep -q my-value; then
-            printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
-            helm get values my-chart-release -n "$APP_SLUG"
-            exit 1
-          fi
-
-          # wait for the app to be ready again
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$UPGRADE_VERSION" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # validate that the conditional chart is installed
-          COUNTER=1
-          while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" == "" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for my-conditional-chart-release to be installed"
-              helm ls -n "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # toggle the config option to exclude the conditional chart
-          ./bin/kots set config "$APP_SLUG" install_conditional_chart=0 --deploy --namespace "$APP_SLUG"
-
-          # wait for my-conditional-chart-release to be uninstalled
-          COUNTER=1
-          while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" != "" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for my-conditional-chart-release to be uninstalled"
-              helm ls -n "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-deployment-orchestration:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ] 
-    env:
-      APP_SLUG: deployment-orchestration
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - uses: azure/setup-helm@v3
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$APP_SLUG"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
-
-      - name: run the test
-        run: |
-          set +e
-          echo ${{ secrets.DEPLOYMENT_ORCHESTRATION_LICENSE }} | base64 -d > license.yaml
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$APP_SLUG" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$APP_SLUG" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-            exit $EXIT_CODE
-          fi
-
-          function wait_for_log {
-            local counter=1
-            local timeout=30
-            local log_pattern="$1"
-
-            while ! kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "$log_pattern"; do
-              ((counter += 1))
-              if [ $counter -gt $timeout ]; then
-                echo "Timed out waiting for log $log_pattern"
-                exit 1
-              fi
-              sleep 1
-            done
-          }
-
-          wait_for_log "applying phase -9999"
-
-          wait_for_log "applying phase -3"
-          wait_for_log "waiting for resource apiextensions.k8s.io/v1/CustomResourceDefinition/myresources.example.com in namespace $APP_SLUG to be ready"
-
-          wait_for_log "applying phase -2"
-
-          # validate that phase -1 has not deployed yet since we're waiting on the CR status fields
-          if kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "applying phase -1"; then
-            printf "phase -1 was deployed before phase -2 completed"
-            kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG"
-            exit 1
-          fi
-
-          wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.extract=true"
-
-          # set .status.tasks.extract=true
-          kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"extract": true}}}' --type=merge
-
-          wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.transform=true"
-
-          # set .status.tasks.transform=true
-          kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"transform": true}}}' --type=merge
-
-          wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.load=true"
-
-          # set .status.tasks.load=true
-          kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"load": true}}}' --type=merge
-
-          # now validate that the remaining phases are deployed (-1, 0, and 1)
-          wait_for_log "applying phase -1"
-          wait_for_log "waiting for resource apps/v1/Deployment/nginx-1 in namespace $APP_SLUG to be ready"
-
-          wait_for_log "applying phase 0"
-          wait_for_log "waiting for resource apps/v1/Deployment/nginx-2 in namespace $APP_SLUG to be ready"
-
-          wait_for_log "applying phase 1"
-          wait_for_log "waiting for resource /v1/Service/nginx-2 in namespace $APP_SLUG to be ready"
-
-          # wait for the app to be ready
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$APP_SLUG"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # remove the app
-          ./bin/kots remove "$APP_SLUG" -n "$APP_SLUG" --undeploy
-
-          wait_for_log "deleting resources in phase -1"
-          wait_for_log "deleting resources in phase 0"
-          wait_for_log "deleting resources in phase 1"
-          wait_for_log "deleting resources in phase 2"
-          wait_for_log "deleting resources in phase 3"
-
-          # validate that the app reference was removed
-          if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
-            printf "App reference was not removed\n\n"
-            exit 1
-          fi
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-replicated-sdk:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0},
-          {distribution: openshift, version: 4.13.0-okd}
-        ]
-    env:
-      KOTS_NAMESPACE: replicated-sdk
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Create Cluster 
-        id: create-cluster 
-        uses: replicatedhq/replicated-actions/create-cluster@v1 
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          kubernetes-distribution: ${{ matrix.cluster.distribution }}
-          kubernetes-version: ${{ matrix.cluster.version }}
-          cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
-          timeout-minutes: '120'
-          ttl: 2h
-          export-kubeconfig: true
-
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-
-      - run: chmod +x bin/kots
-
-      - name: create namespace and dockerhub secret
-        run: |
-          kubectl create ns "$KOTS_NAMESPACE"
-          kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$KOTS_NAMESPACE"
-
-      - name: run upgrade-to-replicated-sdk test
-        id: upgrade-to-replicated-sdk
-        env:
-          APP_SLUG: upgrade-to-replicated-sdk
-        run: |
-          set +e
-
-          echo ${{ secrets.UPGRADE_TO_REPLICATED_SDK_LICENSE }} | base64 -d > license.yaml
-          LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
-          echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"
-
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$KOTS_NAMESPACE" \
-            --shared-password password \
-            --app-version-label v1.0.0 \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$KOTS_NAMESPACE" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-            exit $EXIT_CODE
-          fi
-
-          # wait for the app to be ready
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # upgrade the app to the new version
-          ./bin/kots upstream upgrade "$APP_SLUG" -n "$KOTS_NAMESPACE" --deploy
-          sleep 5
-
-          # wait for the app to be ready again
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # get the version of the replicated-sdk that is running and set it as an output
-          REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$')
-          echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"
-
-      - run: rm -rf ./replicated-sdk
-      - name: Checkout replicated-sdk
-        uses: actions/checkout@v4
-        with:
-          repository: replicatedhq/replicated-sdk
-          path: replicated-sdk
-          ref: ${{ steps.upgrade-to-replicated-sdk.outputs.replicated-sdk-version }}
-
-      - name: Validate endpoints
-        uses: ./replicated-sdk/.github/actions/validate-endpoints
-        with:
-          license-id: ${{ steps.upgrade-to-replicated-sdk.outputs.license-id }}
-          license-fields: '[{"name":"expires_at","value": ""}]'
-          integration-enabled: true
-          namespace: "$KOTS_NAMESPACE"
-
-      - name: remove upgrade-to-replicated-sdk app
-        env:
-          APP_SLUG: upgrade-to-replicated-sdk
-        run: |
-          # remove the app
-          ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy
-
-          # validate that the app reference was removed
-          if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then
-            printf "App reference was not removed\n\n"
-            exit 1
-          fi
-
-      - name: run replicated-sdk-subchart-native-helm-v1 test
-        id: replicated-sdk-subchart-native-helm-v1
-        env:
-          APP_SLUG: replicated-sdk-subchart-native-helm-v1
-        run: |
-          set +e
-
-          echo ${{ secrets.REPLICATED_SDK_SUBCHART_NATIVE_HELM_V1 }} | base64 -d > license.yaml
-          LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
-          echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"
-
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$KOTS_NAMESPACE" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$KOTS_NAMESPACE" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-            exit $EXIT_CODE
-          fi
-
-          # wait for the app to be ready
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # get the version of the replicated-sdk that is running and set it as an output
-          REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$')
-          echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"
-
-      - run: rm -rf ./replicated-sdk
-      - name: Checkout replicated-sdk
-        uses: actions/checkout@v4
-        with:
-          repository: replicatedhq/replicated-sdk
-          path: replicated-sdk
-          ref: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.replicated-sdk-version }}
-
-      - name: Validate endpoints
-        uses: ./replicated-sdk/.github/actions/validate-endpoints
-        with:
-          license-id: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.license-id }}
-          license-fields: '[{"name":"expires_at","value": ""}]'
-          integration-enabled: true
-          namespace: "$KOTS_NAMESPACE"
-
-      - name: remove replicated-sdk-subchart-native-helm-v1 app
-        env:
-          APP_SLUG: replicated-sdk-subchart-native-helm-v1
-        run: |
-          # remove the app
-          ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy
-
-          # validate that the app reference was removed
-          if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then
-            printf "App reference was not removed\n\n"
-            exit 1
-          fi
-
-      - name: run replicated-sdk-subchart-replicated-helm test
-        id: replicated-sdk-subchart-replicated-helm
-        env:
-          APP_SLUG: replicated-sdk-subchart-replicated-helm
-        run: |
-          set +e
-
-          echo ${{ secrets.REPLICATED_SDK_SUBCHART_REPLICATED_HELM }} | base64 -d > license.yaml
-          LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
-          echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"
-
-          ./bin/kots \
-            install "$APP_SLUG/automated" \
-            --license-file license.yaml \
-            --no-port-forward \
-            --namespace "$KOTS_NAMESPACE" \
-            --shared-password password \
-            --kotsadm-registry ttl.sh \
-            --kotsadm-namespace automated-${{ github.run_id }} \
-            --kotsadm-tag 24h
-
-          EXIT_CODE=$?
-          if [ $EXIT_CODE -ne 0 ]; then
-            echo "------pods:"
-            kubectl -n "$KOTS_NAMESPACE" get pods
-            echo "------kotsadm logs"
-            kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-            exit $EXIT_CODE
-          fi
-
-          # wait for the app to be ready
-          COUNTER=1
-          while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
-            ((COUNTER += 1))
-            if [ $COUNTER -gt 120 ]; then
-              echo "Timed out waiting for app to be ready"
-              ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
-              echo "kotsadm logs:"
-              kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
-              exit 1
-            fi
-            sleep 1
-          done
-
-          # get the version of the replicated-sdk that is running and set it as an output
-          REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$')
-          echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"
-
-      - run: rm -rf ./replicated-sdk
-      - name: Checkout replicated-sdk
-        uses: actions/checkout@v4
-        with:
-          repository: replicatedhq/replicated-sdk
-          path: replicated-sdk
-          ref: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.replicated-sdk-version }}
-
-      - name: Validate endpoints
-        uses: ./replicated-sdk/.github/actions/validate-endpoints
-        with:
-          license-id: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.license-id }}
-          license-fields: '[{"name":"expires_at","value": ""}]'
-          integration-enabled: true
-          namespace: "$KOTS_NAMESPACE"
-          deployed-via-kubectl: true
-
-      - name: remove replicated-sdk-subchart-replicated-helm app
-        env:
-          APP_SLUG: replicated-sdk-subchart-replicated-helm
-        run: |
-          # remove the app
-          ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy
-
-          # validate that the app reference was removed
-          if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then
-            printf "App reference was not removed\n\n"
-            exit 1
-          fi
-
-      - name: Generate support bundle on failure
-        if: failure()
-        uses: ./.github/actions/generate-support-bundle
-        with:
-          kots-namespace: "$APP_SLUG"
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-
-      - name: Remove Cluster
-        id: remove-cluster
-        uses: replicatedhq/replicated-actions/remove-cluster@v1
-        if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
-        continue-on-error: true
-        with:
-          api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
-          cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
-
-
-  validate-support-bundle:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0},
-          {distribution: openshift, version: 4.13.0-okd}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'Support Bundle'
-          kots-namespace: 'support-bundle'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-gitops:
-    runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
-    strategy:
-      fail-fast: false
-      matrix:
-        cluster: [
-          {distribution: kind, version: v1.28.0}
-        ]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: download e2e deps
-        uses: actions/download-artifact@v4
-        with:
-          name: e2e
-          path: e2e/bin/
-      - run: docker load -i e2e/bin/e2e-deps.tar
-      - run: chmod +x e2e/bin/*
-      - name: download kots binary
-        uses: actions/download-artifact@v4
-        with:
-          name: kots
-          path: bin/
-      - run: chmod +x bin/*
-      - uses: ./.github/actions/kots-e2e
-        with:
-          test-focus: 'GitOps'
-          kots-namespace: 'gitops'
-          k8s-distribution: ${{ matrix.cluster.distribution }}
-          k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
-          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
-          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
-          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
-          kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
-          kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
-
-
-  validate-pr-tests:
-    runs-on: ubuntu-20.04
-    needs:
-      # static and unit tests
-      - lint-web
-      - unit-test-web
-      - vet-kots
-      - ci-test-kots
-      # testim tests
-      - validate-existing-online-install-minimal
-      - validate-smoke-test
-      - validate-backup-and-restore
-      - validate-no-required-config
-      - validate-version-history-pagination
-      - validate-change-license
-      - validate-min-kots-version
-      - validate-target-kots-version
-      - validate-range-kots-version
-      - validate-multi-app-backup-and-restore
-      - validate-multi-app-install
-      - validate-airgap-smoke-test
-      - validate-config
-      - validate-support-bundle
-      - validate-gitops
-      # non-testim tests
-      - validate-minimal-rbac
-      - validate-minimal-rbac-override
-      - validate-multi-namespace
-      - validate-kots-pull
-      - validate-app-version-label
-      - validate-helm-install-order
-      - validate-no-redeploy-on-restart
-      - validate-kubernetes-installer-preflight
-      - validate-kots-upgrade
-      - validate-remove-app
-      - validate-registry-check
-      - validate-kots-helm-release-secret-migration
-      - validate-native-helm-v2
-      - validate-deployment-orchestration
-      - validate-replicated-sdk
-      - validate-strict-preflight-checks
-      # cli-only tests
-      - validate-kots-push-images-anonymous
-    steps:
-      - run: echo "All PR tests passed"
-
-
-  # this job will validate that the kurl add-on validation did not fail and that all pr-tests succeed
-  # it is used for the github branch protection rule
-  validate-success:
-    runs-on: ubuntu-20.04
-    needs:
-      - validate-kurl-addon
-      - validate-pr-tests
-    if: always()
-    steps:
-      # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
-      # if the validate-kurl-addon job failed or was cancelled, this job will fail. it's ok if this job was skipped
-      - name: fail if validate-kurl-addon job failed
-        if: needs.validate-kurl-addon.result == 'failure' || needs.validate-kurl-addon.result == 'cancelled'
-        run: exit 1
-      # if the validate-pr-tests job was not successful, this job will fail
-      - name: fail if validate-pr-tests job was not successful
-        if: needs.validate-pr-tests.result != 'success'
-        run: exit 1
-      # if the validate-pr-tests job was successful, this job will succeed
-      - name: succeed if validate-pr-tests job succeeded
-        if: needs.validate-pr-tests.result == 'success'
-        run: echo "Validation succeeded"
+  #         COUNTER=1
+  #         while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-chart-release)" != "" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for my-chart-release to be uninstalled"
+  #             helm ls -n "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         COUNTER=1
+  #         while [ "$(helm ls -n "$APP_SLUG" | grep my-other-chart-release | awk 'NR>0{print $3}')" != "2" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for my-other-chart-release to be upgraded to revision 2"
+  #             helm ls -n "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # validate that `helm get values` works for the upgraded v1beta2 chart
+  #         if ! helm get values my-other-chart-release -n "$APP_SLUG" | grep -q my-value; then
+  #           printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
+  #           helm get values my-chart-release -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         # wait for the app to be ready again
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$UPGRADE_VERSION" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # validate that the conditional chart is installed
+  #         COUNTER=1
+  #         while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" == "" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for my-conditional-chart-release to be installed"
+  #             helm ls -n "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # toggle the config option to exclude the conditional chart
+  #         ./bin/kots set config "$APP_SLUG" install_conditional_chart=0 --deploy --namespace "$APP_SLUG"
+
+  #         # wait for my-conditional-chart-release to be uninstalled
+  #         COUNTER=1
+  #         while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-conditional-chart-release)" != "" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for my-conditional-chart-release to be uninstalled"
+  #             helm ls -n "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-deployment-orchestration:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ] 
+  #   env:
+  #     APP_SLUG: deployment-orchestration
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - uses: azure/setup-helm@v3
+  #       with:
+  #         token: ${{ secrets.GITHUB_TOKEN }}
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$APP_SLUG"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$APP_SLUG"
+
+  #     - name: run the test
+  #       run: |
+  #         set +e
+  #         echo ${{ secrets.DEPLOYMENT_ORCHESTRATION_LICENSE }} | base64 -d > license.yaml
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$APP_SLUG" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$APP_SLUG" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         function wait_for_log {
+  #           local counter=1
+  #           local timeout=30
+  #           local log_pattern="$1"
+
+  #           while ! kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "$log_pattern"; do
+  #             ((counter += 1))
+  #             if [ $counter -gt $timeout ]; then
+  #               echo "Timed out waiting for log $log_pattern"
+  #               exit 1
+  #             fi
+  #             sleep 1
+  #           done
+  #         }
+
+  #         wait_for_log "applying phase -9999"
+
+  #         wait_for_log "applying phase -3"
+  #         wait_for_log "waiting for resource apiextensions.k8s.io/v1/CustomResourceDefinition/myresources.example.com in namespace $APP_SLUG to be ready"
+
+  #         wait_for_log "applying phase -2"
+
+  #         # validate that phase -1 has not deployed yet since we're waiting on the CR status fields
+  #         if kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "applying phase -1"; then
+  #           printf "phase -1 was deployed before phase -2 completed"
+  #           kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG"
+  #           exit 1
+  #         fi
+
+  #         wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.extract=true"
+
+  #         # set .status.tasks.extract=true
+  #         kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"extract": true}}}' --type=merge
+
+  #         wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.transform=true"
+
+  #         # set .status.tasks.transform=true
+  #         kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"transform": true}}}' --type=merge
+
+  #         wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.load=true"
+
+  #         # set .status.tasks.load=true
+  #         kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"load": true}}}' --type=merge
+
+  #         # now validate that the remaining phases are deployed (-1, 0, and 1)
+  #         wait_for_log "applying phase -1"
+  #         wait_for_log "waiting for resource apps/v1/Deployment/nginx-1 in namespace $APP_SLUG to be ready"
+
+  #         wait_for_log "applying phase 0"
+  #         wait_for_log "waiting for resource apps/v1/Deployment/nginx-2 in namespace $APP_SLUG to be ready"
+
+  #         wait_for_log "applying phase 1"
+  #         wait_for_log "waiting for resource /v1/Service/nginx-2 in namespace $APP_SLUG to be ready"
+
+  #         # wait for the app to be ready
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$APP_SLUG"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # remove the app
+  #         ./bin/kots remove "$APP_SLUG" -n "$APP_SLUG" --undeploy
+
+  #         wait_for_log "deleting resources in phase -1"
+  #         wait_for_log "deleting resources in phase 0"
+  #         wait_for_log "deleting resources in phase 1"
+  #         wait_for_log "deleting resources in phase 2"
+  #         wait_for_log "deleting resources in phase 3"
+
+  #         # validate that the app reference was removed
+  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
+  #           printf "App reference was not removed\n\n"
+  #           exit 1
+  #         fi
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-replicated-sdk:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0},
+  #         {distribution: openshift, version: 4.13.0-okd}
+  #       ]
+  #   env:
+  #     KOTS_NAMESPACE: replicated-sdk
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Create Cluster 
+  #       id: create-cluster 
+  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
+  #         kubernetes-version: ${{ matrix.cluster.version }}
+  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
+  #         timeout-minutes: '120'
+  #         ttl: 2h
+  #         export-kubeconfig: true
+
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+
+  #     - run: chmod +x bin/kots
+
+  #     - name: create namespace and dockerhub secret
+  #       run: |
+  #         kubectl create ns "$KOTS_NAMESPACE"
+  #         kubectl create secret docker-registry kotsadm-dockerhub --docker-server index.docker.io --docker-username "${{ secrets.E2E_DOCKERHUB_USERNAME }}" --docker-password "${{ secrets.E2E_DOCKERHUB_PASSWORD }}" --namespace "$KOTS_NAMESPACE"
+
+  #     - name: run upgrade-to-replicated-sdk test
+  #       id: upgrade-to-replicated-sdk
+  #       env:
+  #         APP_SLUG: upgrade-to-replicated-sdk
+  #       run: |
+  #         set +e
+
+  #         echo ${{ secrets.UPGRADE_TO_REPLICATED_SDK_LICENSE }} | base64 -d > license.yaml
+  #         LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
+  #         echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"
+
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$KOTS_NAMESPACE" \
+  #           --shared-password password \
+  #           --app-version-label v1.0.0 \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$KOTS_NAMESPACE" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         # wait for the app to be ready
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # upgrade the app to the new version
+  #         ./bin/kots upstream upgrade "$APP_SLUG" -n "$KOTS_NAMESPACE" --deploy
+  #         sleep 5
+
+  #         # wait for the app to be ready again
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # get the version of the replicated-sdk that is running and set it as an output
+  #         REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$')
+  #         echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"
+
+  #     - run: rm -rf ./replicated-sdk
+  #     - name: Checkout replicated-sdk
+  #       uses: actions/checkout@v4
+  #       with:
+  #         repository: replicatedhq/replicated-sdk
+  #         path: replicated-sdk
+  #         ref: ${{ steps.upgrade-to-replicated-sdk.outputs.replicated-sdk-version }}
+
+  #     - name: Validate endpoints
+  #       uses: ./replicated-sdk/.github/actions/validate-endpoints
+  #       with:
+  #         license-id: ${{ steps.upgrade-to-replicated-sdk.outputs.license-id }}
+  #         license-fields: '[{"name":"expires_at","value": ""}]'
+  #         integration-enabled: true
+  #         namespace: "$KOTS_NAMESPACE"
+
+  #     - name: remove upgrade-to-replicated-sdk app
+  #       env:
+  #         APP_SLUG: upgrade-to-replicated-sdk
+  #       run: |
+  #         # remove the app
+  #         ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy
+
+  #         # validate that the app reference was removed
+  #         if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then
+  #           printf "App reference was not removed\n\n"
+  #           exit 1
+  #         fi
+
+  #     - name: run replicated-sdk-subchart-native-helm-v1 test
+  #       id: replicated-sdk-subchart-native-helm-v1
+  #       env:
+  #         APP_SLUG: replicated-sdk-subchart-native-helm-v1
+  #       run: |
+  #         set +e
+
+  #         echo ${{ secrets.REPLICATED_SDK_SUBCHART_NATIVE_HELM_V1 }} | base64 -d > license.yaml
+  #         LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
+  #         echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"
+
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$KOTS_NAMESPACE" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$KOTS_NAMESPACE" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         # wait for the app to be ready
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # get the version of the replicated-sdk that is running and set it as an output
+  #         REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$')
+  #         echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"
+
+  #     - run: rm -rf ./replicated-sdk
+  #     - name: Checkout replicated-sdk
+  #       uses: actions/checkout@v4
+  #       with:
+  #         repository: replicatedhq/replicated-sdk
+  #         path: replicated-sdk
+  #         ref: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.replicated-sdk-version }}
+
+  #     - name: Validate endpoints
+  #       uses: ./replicated-sdk/.github/actions/validate-endpoints
+  #       with:
+  #         license-id: ${{ steps.replicated-sdk-subchart-native-helm-v1.outputs.license-id }}
+  #         license-fields: '[{"name":"expires_at","value": ""}]'
+  #         integration-enabled: true
+  #         namespace: "$KOTS_NAMESPACE"
+
+  #     - name: remove replicated-sdk-subchart-native-helm-v1 app
+  #       env:
+  #         APP_SLUG: replicated-sdk-subchart-native-helm-v1
+  #       run: |
+  #         # remove the app
+  #         ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy
+
+  #         # validate that the app reference was removed
+  #         if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then
+  #           printf "App reference was not removed\n\n"
+  #           exit 1
+  #         fi
+
+  #     - name: run replicated-sdk-subchart-replicated-helm test
+  #       id: replicated-sdk-subchart-replicated-helm
+  #       env:
+  #         APP_SLUG: replicated-sdk-subchart-replicated-helm
+  #       run: |
+  #         set +e
+
+  #         echo ${{ secrets.REPLICATED_SDK_SUBCHART_REPLICATED_HELM }} | base64 -d > license.yaml
+  #         LICENSE_ID="$(grep -oP 'licenseID:\s*\K\w+' license.yaml)"
+  #         echo "license-id=$LICENSE_ID" >> "$GITHUB_OUTPUT"
+
+  #         ./bin/kots \
+  #           install "$APP_SLUG/automated" \
+  #           --license-file license.yaml \
+  #           --no-port-forward \
+  #           --namespace "$KOTS_NAMESPACE" \
+  #           --shared-password password \
+  #           --kotsadm-registry ttl.sh \
+  #           --kotsadm-namespace automated-${{ github.run_id }} \
+  #           --kotsadm-tag 24h
+
+  #         EXIT_CODE=$?
+  #         if [ $EXIT_CODE -ne 0 ]; then
+  #           echo "------pods:"
+  #           kubectl -n "$KOTS_NAMESPACE" get pods
+  #           echo "------kotsadm logs"
+  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #           exit $EXIT_CODE
+  #         fi
+
+  #         # wait for the app to be ready
+  #         COUNTER=1
+  #         while [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" | awk 'NR>1{print $2}')" != "ready" ]; do
+  #           ((COUNTER += 1))
+  #           if [ $COUNTER -gt 120 ]; then
+  #             echo "Timed out waiting for app to be ready"
+  #             ./bin/kots get apps --namespace "$KOTS_NAMESPACE"
+  #             echo "kotsadm logs:"
+  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$KOTS_NAMESPACE"
+  #             exit 1
+  #           fi
+  #           sleep 1
+  #         done
+
+  #         # get the version of the replicated-sdk that is running and set it as an output
+  #         REPLICATED_SDK_VERSION=$(kubectl get deploy replicated -n "$KOTS_NAMESPACE" -o jsonpath='{.spec.template.spec.containers[0].image}' | grep -oE '[^:]+$')
+  #         echo "replicated-sdk-version=$REPLICATED_SDK_VERSION" >> "$GITHUB_OUTPUT"
+
+  #     - run: rm -rf ./replicated-sdk
+  #     - name: Checkout replicated-sdk
+  #       uses: actions/checkout@v4
+  #       with:
+  #         repository: replicatedhq/replicated-sdk
+  #         path: replicated-sdk
+  #         ref: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.replicated-sdk-version }}
+
+  #     - name: Validate endpoints
+  #       uses: ./replicated-sdk/.github/actions/validate-endpoints
+  #       with:
+  #         license-id: ${{ steps.replicated-sdk-subchart-replicated-helm.outputs.license-id }}
+  #         license-fields: '[{"name":"expires_at","value": ""}]'
+  #         integration-enabled: true
+  #         namespace: "$KOTS_NAMESPACE"
+  #         deployed-via-kubectl: true
+
+  #     - name: remove replicated-sdk-subchart-replicated-helm app
+  #       env:
+  #         APP_SLUG: replicated-sdk-subchart-replicated-helm
+  #       run: |
+  #         # remove the app
+  #         ./bin/kots remove "$APP_SLUG" -n "$KOTS_NAMESPACE" --undeploy
+
+  #         # validate that the app reference was removed
+  #         if [ "$(./bin/kots get apps --namespace "$KOTS_NAMESPACE" --output=json | tr -d '\n')" != "[]" ]; then
+  #           printf "App reference was not removed\n\n"
+  #           exit 1
+  #         fi
+
+  #     - name: Generate support bundle on failure
+  #       if: failure()
+  #       uses: ./.github/actions/generate-support-bundle
+  #       with:
+  #         kots-namespace: "$APP_SLUG"
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+
+  #     - name: Remove Cluster
+  #       id: remove-cluster
+  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
+  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
+  #       continue-on-error: true
+  #       with:
+  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
+  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}
+
+
+  # validate-support-bundle:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0},
+  #         {distribution: openshift, version: 4.13.0-okd}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'Support Bundle'
+  #         kots-namespace: 'support-bundle'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-gitops:
+  #   runs-on: ubuntu-20.04
+  #   needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       cluster: [
+  #         {distribution: kind, version: v1.28.0}
+  #       ]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+  #     - name: download e2e deps
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: e2e
+  #         path: e2e/bin/
+  #     - run: docker load -i e2e/bin/e2e-deps.tar
+  #     - run: chmod +x e2e/bin/*
+  #     - name: download kots binary
+  #       uses: actions/download-artifact@v4
+  #       with:
+  #         name: kots
+  #         path: bin/
+  #     - run: chmod +x bin/*
+  #     - uses: ./.github/actions/kots-e2e
+  #       with:
+  #         test-focus: 'GitOps'
+  #         kots-namespace: 'gitops'
+  #         k8s-distribution: ${{ matrix.cluster.distribution }}
+  #         k8s-version: ${{ matrix.cluster.version }}
+  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
+  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
+  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
+  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
+  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
+  #         kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
+  #         kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
+
+
+  # validate-pr-tests:
+  #   runs-on: ubuntu-20.04
+  #   needs:
+  #     # static and unit tests
+  #     - lint-web
+  #     - unit-test-web
+  #     - vet-kots
+  #     - ci-test-kots
+  #     # testim tests
+  #     - validate-existing-online-install-minimal
+  #     - validate-smoke-test
+  #     - validate-backup-and-restore
+  #     - validate-no-required-config
+  #     - validate-version-history-pagination
+  #     - validate-change-license
+  #     - validate-min-kots-version
+  #     - validate-target-kots-version
+  #     - validate-range-kots-version
+  #     - validate-multi-app-backup-and-restore
+  #     - validate-multi-app-install
+  #     - validate-airgap-smoke-test
+  #     - validate-config
+  #     - validate-support-bundle
+  #     - validate-gitops
+  #     # non-testim tests
+  #     - validate-minimal-rbac
+  #     - validate-minimal-rbac-override
+  #     - validate-multi-namespace
+  #     - validate-kots-pull
+  #     - validate-app-version-label
+  #     - validate-helm-install-order
+  #     - validate-no-redeploy-on-restart
+  #     - validate-kubernetes-installer-preflight
+  #     - validate-kots-upgrade
+  #     - validate-remove-app
+  #     - validate-registry-check
+  #     - validate-kots-helm-release-secret-migration
+  #     - validate-native-helm-v2
+  #     - validate-deployment-orchestration
+  #     - validate-replicated-sdk
+  #     - validate-strict-preflight-checks
+  #     # cli-only tests
+  #     - validate-kots-push-images-anonymous
+  #   steps:
+  #     - run: echo "All PR tests passed"
+
+
+  # # this job will validate that the kurl add-on validation did not fail and that all pr-tests succeed
+  # # it is used for the github branch protection rule
+  # validate-success:
+  #   runs-on: ubuntu-20.04
+  #   needs:
+  #     - validate-kurl-addon
+  #     - validate-pr-tests
+  #   if: always()
+  #   steps:
+  #     # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
+  #     # if the validate-kurl-addon job failed or was cancelled, this job will fail. it's ok if this job was skipped
+  #     - name: fail if validate-kurl-addon job failed
+  #       if: needs.validate-kurl-addon.result == 'failure' || needs.validate-kurl-addon.result == 'cancelled'
+  #       run: exit 1
+  #     # if the validate-pr-tests job was not successful, this job will fail
+  #     - name: fail if validate-pr-tests job was not successful
+  #       if: needs.validate-pr-tests.result != 'success'
+  #       run: exit 1
+  #     # if the validate-pr-tests job was successful, this job will succeed
+  #     - name: succeed if validate-pr-tests job succeeded
+  #       if: needs.validate-pr-tests.result == 'success'
+  #       run: echo "Validation succeeded"
diff --git a/Makefile b/Makefile
index 7fa63f8f05..e3466cb5e3 100644
--- a/Makefile
+++ b/Makefile
@@ -51,6 +51,8 @@ kots: capture-start-time kots-real report-metric
 
 .PHONY: kots-real
 kots-real:
+	mkdir -p web/dist
+	touch web/dist/THIS_IS_KOTS # we need this for go:embed in the api, but it's not actually used in the cli
 	go build ${LDFLAGS} -o bin/kots $(BUILDFLAGS) github.com/replicatedhq/kots/cmd/kots
 
 .PHONY: fmt
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index 8f38f6553a..acf7d64146 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -17,8 +17,8 @@ environment:
       - ca-certificates-bundle
       - busybox
       - git
-      - go
-      - nodejs
+      - go-1.21
+      - nodejs-19
       - yarn
   environment:
     GOMODCACHE: '/var/cache/melange'
diff --git a/web/babel.config.js b/web/babel.config.js
index d1033e4c6c..4f36486ad7 100644
--- a/web/babel.config.js
+++ b/web/babel.config.js
@@ -5,7 +5,11 @@ module.exports = {
     "@babel/preset-typescript",
   ],
   plugins: [
-    ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-proposal-optional-chaining", { loose: true }],
+    "@babel/plugin-proposal-class-properties",
+    "@babel/plugin-proposal-optional-chaining",
   ],
+  "assumptions": {
+    "privateFieldsAsProperties": true,
+    "setPublicClassFields": true
+  }
 };
diff --git a/web/dist/README.md b/web/dist/README.md
deleted file mode 100644
index 7f1a1f8cb0..0000000000
--- a/web/dist/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-The dist directory is used in the go build to embed the compiled web resources into the kotsadm binary.  Because 
-web isn't always built (testing, okteto, etc), this README.md will allow compiling of the go binary without first 
-building web. 
\ No newline at end of file