build-test.yaml

name: build-test

on:
  pull_request:
  # pull_request_target:
  #   # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
  #   # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
  #   types: [ opened, synchronize, reopened, labeled ]

concurrency:
  group: ${{ github.head_ref }}
  cancel-in-progress: true

jobs:
  can-run-ci:
    runs-on: ubuntu-20.04
    # if the event is pull_request and:
    #   - it is not a fork OR it is from the dependabot
    #   - Then must have the label '@actions/safe-to-test'
    #
    # The 'pull_request_target' workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
    # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
    if: >
      ( github.event_name == 'pull_request' &&
        ( github.event.pull_request.head.repo.full_name == github.repository ||
          contains(github.event.pull_request.labels.*.name, '@actions/safe-to-test' )
        )
      )
    steps:
      - name: ok
        run: echo "yes"

  # Use this to disable tests when iteratig on a specific test to save time
  enable-tests:
    runs-on: ubuntu-20.04
    steps:
      - name: ok
        # change 0 to a positive interger to prevent all tests from running
        run: exit 0


  generate-tag:
    runs-on: ubuntu-20.04
    outputs:
      tag: ${{ steps.get_tag.outputs.GIT_TAG }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Get tags
        id: get_tag
        uses: ./.github/actions/version-tag


  test-okteto-env:
    runs-on: ubuntu-latest
    needs: [ can-run-ci ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Test Okteto development environment
        uses: replicatedhq/action-okteto-test@main
        with:
          token: ${{ secrets.OKTETO_TOKEN }}
          branch: ${{ github.head_ref }}

  deps-web:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Setup Node.js environment
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
          cache: yarn
          cache-dependency-path: web/yarn.lock

      - name: Cache node_modules
        uses: actions/cache@v3
        with:
          path: ${{ github.workspace }}/web/node_modules
          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

      - name: Install web deps
        run: make -C web deps


  lint-web:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, deps-web ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Setup Node.js environment
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
          cache: yarn
          cache-dependency-path: web/yarn.lock

      - name: Cache node_modules
        uses: actions/cache@v3
        with:
          path: ${{ github.workspace }}/web/node_modules
          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

      - name: Lint
        run: make -C web lint


  unit-test-web:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, deps-web ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Setup Node.js environment
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
          cache: yarn
          cache-dependency-path: web/yarn.lock

      - name: Cache node_modules
        uses: actions/cache@v3
        with:
          path: ${{ github.workspace }}/web/node_modules
          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

      - name: Unit test
        run: make -C web test-unit


  build-web:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, deps-web, generate-tag ]
    steps:
      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
      # this action checks out the remote branch and runs CI
      - name: Checkout
        uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Setup Node.js environment
        uses: actions/setup-node@v3
        with:
          node-version: '18.x'
          cache: yarn
          cache-dependency-path: web/yarn.lock

      - name: Cache node_modules
        uses: actions/cache@v3
        with:
          path: ${{ github.workspace }}/web/node_modules
          key: ${{ runner.os }}-${{ runner.arch }}-yarn-node_modules-${{ hashFiles('web/yarn.lock') }}

      - name: Build web
        env:
          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
        run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C web build-kotsadm

      - name: Upload web artifact
        uses: actions/upload-artifact@v3
        with:
          name: web
          path: ./web/dist

  deps-kots:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'
          cache: true

      - run: go mod download

  vet-kots:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, deps-kots ]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'
          cache: true

      - name: vet
        run: make vet

  ci-test-kots:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, deps-kots ]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'
          cache: true

      - name: test
        run: make ci-test

  build-kotsadm:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, build-web, deps-kots, generate-tag ]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'
          cache: true

      - name: Download web artifact
        uses: actions/download-artifact@v3
        with:
          name: web
          path: ./web/dist

      - name: Build
        env:
          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
        run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make build

      - uses: actions/upload-artifact@v3
        with:
          name: kotsadm
          path: ./bin/kotsadm

  build-kots:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, build-web, deps-kots, generate-tag ]

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'
          cache: true

      - name: Download web artifact
        uses: actions/download-artifact@v3
        with:
          name: web
          path: ./web/dist

      - name: Build kots
        env:
          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
        run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make kots

      - uses: actions/upload-artifact@v3
        with:
          name: kots
          path: ./bin/kots


  build-push-kotsadm-image:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, build-kotsadm, build-kots ]
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Download kots artifact
        uses: actions/download-artifact@v3
        with:
          name: kots
          path: bin/

      - name: Download kotsadm artifact
        uses: actions/download-artifact@v3
        with:
          name: kotsadm
          path: bin/

      - run: chmod +x bin/kots bin/kotsadm

      - name: build and push kotsadm for e2e
        uses: docker/build-push-action@v3
        with:
          tags: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
          context: ./
          file: ./deploy/Dockerfile
          push: true


  build-kots-helm:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]

    steps:
      - uses: actions/checkout@v3
        with:
          ref: main
          repository: replicatedhq/kots-helm

      - name: Build KOTS Helm chart
        env:
          GIT_COMMIT: ${{ github.sha }}
        run: |
          curl -O -L "https://raw.githubusercontent.com/replicatedhq/kots/${GIT_COMMIT}/.image.env"
          mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}"

          export CHART_VERSION=0.0.${{ github.run_id }}-automated
          export KOTS_VERSION=24h
          export KOTS_TAG=24h
          export KOTSADM_REGISTRY=ttl.sh/automated-${{ github.run_id }}

          envsubst < Chart.yaml.tmpl > Chart.yaml
          envsubst < values.yaml.tmpl > values.yaml

          CHART_NAME=$(helm package . | rev | cut -d/ -f1 | rev)
          export CHART_NAME
          helm push "$CHART_NAME" oci://ttl.sh/automated-${{ github.run_id }}


  build-e2e:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'

      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Cache Go modules
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-e2e-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-e2e-
      - run: make -C e2e build deps
      - run: docker save e2e-deps -o e2e/bin/e2e-deps.tar
      - uses: actions/upload-artifact@v3
        with:
          name: e2e
          path: e2e/bin/


  build-kurl-proxy:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/setup-go@v3
        with:
          go-version: '^1.20.0'

      - name: setup env
        run: |
          echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
          echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
        shell: bash

      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
      # this action creates a branch based on remote branch and runs the tests
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: Cache Go modules
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-kurlproxy-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-kurlproxy-

      - run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C kurl_proxy test build

      - name: build and push kurl_proxy for e2e
        uses: docker/build-push-action@v3
        with:
          tags: ttl.sh/automated-${{ github.run_id }}/kurl-proxy:24h
          context: ./kurl_proxy
          file: ./kurl_proxy/deploy/Dockerfile
          push: true


  build-migrations:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - name: setup env
        run: |
          echo "GOPATH=$(go env GOPATH)" >> "$GITHUB_ENV"
          echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
        shell: bash

      # This workflow trigger may lead to malicious PR authors being able to obtain repository write permissions or stealing repository secrets.
      # Please read https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
      # this action creates a branch based on remote branch and runs the tests
      - uses: actions/checkout@v3
        with:
          ref: ${{github.event.pull_request.head.ref}}
          repository: ${{github.event.pull_request.head.repo.full_name}}

      - name: load environment variables from .image.env
        uses: falti/dotenv-action@v1
        id: dotenv
        with:
          path: .image.env

      - name: build and push migrations for e2e
        uses: docker/build-push-action@v3
        with:
          tags: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h
          context: ./migrations
          file: ./migrations/deploy/Dockerfile
          push: true
          build-args: SCHEMAHERO_TAG=${{ steps.dotenv.outputs.schemahero_tag }}


  push-minio:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/checkout@v3

      - name: load environment variables from .image.env
        uses: falti/dotenv-action@v1
        id: dotenv
        with:
          path: .image.env

      - name: push minio for e2e
        run: |
          docker pull minio/minio:${{ steps.dotenv.outputs.minio_tag }}
          docker tag minio/minio:${{ steps.dotenv.outputs.minio_tag }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }}
          docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }}


  push-mc:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/checkout@v3

      - name: load environment variables from .image.env
        uses: falti/dotenv-action@v1
        id: dotenv
        with:
          path: .image.env

      - name: push minio client for e2e
        run: |
          docker pull minio/mc:${{ steps.dotenv.outputs.mc_tag }}
          docker tag minio/mc:${{ steps.dotenv.outputs.mc_tag }} ttl.sh/automated-${{ github.run_id }}/mc:${{ steps.dotenv.outputs.mc_tag }}
          docker push ttl.sh/automated-${{ github.run_id }}/mc:${{ steps.dotenv.outputs.mc_tag }}


  push-rqlite:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/checkout@v3

      - name: load environment variables from .image.env
        uses: falti/dotenv-action@v1
        id: dotenv
        with:
          path: .image.env

      - name: push rqlite for CI
        run: |
          docker pull rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
          docker tag rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}
          docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}


  push-dex:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci ]
    steps:
      - uses: actions/checkout@v3

      - name: load environment variables from .image.env
        uses: falti/dotenv-action@v1
        id: dotenv
        with:
          path: .image.env

      - name: push dex for CI
        run: |
          docker build --pull -f deploy/dex.Dockerfile -t ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} --build-arg TAG=${{ steps.dotenv.outputs.DEX_TAG }} .
          docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }}


  # only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**"
  kurl-addon-changes-filter:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, enable-tests ]
    outputs:
      ok-to-test: ${{ steps.filter.outputs.kurl-addon }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            kurl-addon:
              - 'deploy/kurl/kotsadm/template/**'
              - 'deploy/kurl/kotsadm/testgrid-os-spec.yaml'
  validate-kurl-addon:
    runs-on: ubuntu-20.04
    if: ${{ needs.kurl-addon-changes-filter.outputs.ok-to-test == 'true' }}
    needs: [ can-run-ci, enable-tests, generate-tag, kurl-addon-changes-filter, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-dex ]
    steps:
      - name: checkout
        uses: actions/checkout@v3
      - name: set outputs
        id: vars
        run: |
          addon_version=${{ needs.generate-tag.outputs.tag }}
          echo "addon_version=${addon_version#v}" >> "$GITHUB_OUTPUT"
      - name: download kots binary
        uses: actions/download-artifact@v3
        with:
          name: kots
          path: bin/
      - name: prepare kots binary executable
        run: |
          chmod +x bin/*
          tar -C bin/ -czvf bin/kots.tar.gz kots
      - name: generate kurl add-on
        id: addon-generate
        uses: ./.github/actions/kurl-addon-kots-generate
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.KURL_ADDONS_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.KURL_ADDONS_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: us-east-1
        with:
          addon_version: ${{ steps.vars.outputs.addon_version }}
          kotsadm_image_registry: ttl.sh
          kotsadm_image_namespace: automated-${{ github.run_id }}
          kotsadm_image_tag: 24h
          kotsadm_binary_override: bin/kots.tar.gz
      - name: test kurl add-on
        id: addon-test
        uses: ./.github/actions/kurl-addon-kots-test
        with:
          addon_version: ${{ steps.vars.outputs.addon_version }}
          addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }}
          testgrid_api_token: ${{ secrets.TESTGRID_PROD_API_TOKEN }}
      - name: comment testgrid url
        uses: mshick/add-pr-comment@v2
        with:
          message: ${{ steps.addon-test.outputs.testgrid_run_message }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          allow-repeats: false


  get-compatibility-matrix-versions:
    runs-on: ubuntu-20.04
    needs: [ can-run-ci, enable-tests ]
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: get CMX versions to test
        id: cmx-versions-to-test
        uses: ./.github/actions/cmx-versions
        with:
          replicated-api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
    outputs:
      versions-to-test: ${{ steps.cmx-versions-to-test.outputs.versions-to-test }}


  # validate-existing-online-install-minimal:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: |
  #         docker load -i e2e/bin/e2e-deps.tar
  #         chmod +x e2e/bin/*
  #         chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         kots-namespace: 'qakotsregression'
  #         test-focus: 'Regression'
  #         k8s-distribution: k3s
  #         k8s-version: v1.26
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-smoke-test:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: eks, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Smoke Test'
  #         kots-namespace: 'smoke-test'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  validate-smoke-test-compatibility-matrix:
    runs-on: ubuntu-20.04
    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite, get-compatibility-matrix-versions ]
    strategy:
      fail-fast: false
      matrix:
        cluster: ${{ fromJson(needs.get-compatibility-matrix-versions.outputs.versions-to-test) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: download e2e deps
        uses: actions/download-artifact@v3
        with:
          name: e2e
          path: e2e/bin/
      - run: docker load -i e2e/bin/e2e-deps.tar
      - run: chmod +x e2e/bin/*
      - name: download kots binary
        uses: actions/download-artifact@v3
        with:
          name: kots
          path: bin/
      - run: chmod +x bin/*
      - uses: ./.github/actions/kots-e2e
        with:
          test-focus: 'Smoke Test'
          kots-namespace: 'smoke-test'
          k8s-distribution: ${{ matrix.cluster.distribution }}
          k8s-version: ${{ matrix.cluster.version }}
          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
          aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
          aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
          replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-minimal-rbac:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: eks, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Minimal RBAC'
  #         kots-namespace: 'minimal-rbac'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-backup-and-restore:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: eks, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Backup and Restore'
  #         kots-namespace: 'backup-and-restore'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-no-required-config:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'No Required Config'
  #         kots-namespace: 'no-required-config'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-strict-preflight-checks:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Strict Preflight Checks'
  #         kots-namespace: 'strict-preflight-checks'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-config:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Config Validation'
  #         kots-namespace: 'config-validation'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-version-history-pagination:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s-local, version: v1.27.1-k3s1}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Version History Pagination'
  #         kots-namespace: 'version-history-pagination'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'


  # validate-change-license:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Change License'
  #         kots-namespace: 'change-license'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-minimal-rbac-override:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: minimal-rbac
  #     APP_VERSION_LABEL: "0.0.1"
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: minimal rbac override on command line
  #       run: |
  #         kubectl delete ns "$APP_SLUG" --ignore-not-found
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --app-version-label "$APP_VERSION_LABEL" \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h \
  #           --use-minimal-rbac
  #         if kubectl get roles -n "$APP_SLUG" | grep -q kotsadm; then
  #           echo "Found kotsadm role in ${APP_SLUG}"
  #         else
  #           echo "No kotsadm role found in appication namespace"
  #           exit 1
  #         fi
  #         if kubectl get clusterroles | grep -q kotsadm; then
  #           echo "Found kotsadm cluster roles in minimal RBAC install"
  #           exit
  #         fi
  #     - name: no minimal rbac override on command line
  #       run: |
  #         kubectl delete ns "$APP_SLUG" --ignore-not-found
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --app-version-label "$APP_VERSION_LABEL" \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h
  #         if kubectl get roles -n "$APP_SLUG" | grep -q kotsadm; then
  #           echo "Found kotsadm role in cluster scoped install"
  #           exit 1
  #         fi
  #         if kubectl get clusterroles | grep -q kotsadm; then
  #           echo "Found kotsadm cluster role in cluster scoped install"
  #         else
  #           echo "No kotsadm cluster role in cluster scoped install"
  #           exit 1
  #         fi
  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-multi-namespace:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: multi-namespace-yeti
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - uses: azure/setup-helm@v3
  #       with:
  #         token: ${{ secrets.GITHUB_TOKEN }}

  #     - name: Create Cluster 
  #       id: create-cluster
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # validate that helm charts installed using the native helm workflow were deployed via the helm CLI correctly

  #         if ! helm ls -n postgres-test | awk 'NR>1{print $1}' | grep -q postgresql; then
  #           printf "postgresql helm release not found in postgres-test namespace\n\n"
  #           helm ls -n postgres-test
  #           exit 1
  #         fi

  #         if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q private-chart; then
  #           printf "private-chart helm release not found in %s namespace\n\n" "$APP_SLUG"
  #           helm ls -n "$APP_SLUG"
  #           exit 1
  #         fi

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  
  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-gke-install:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-kots, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   env:
  #     APP_SLUG: helm-install-order
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create cluster
  #       uses: ./.github/actions/kots-gke-create
  #       with:
  #         service-key: ${{ secrets.GOOGLE_CLOUD_CREDENTIALS }}

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: reserve capacity
  #       run: |
  #         # https://cloud.google.com/kubernetes-engine/docs/how-to/capacity-provisioning
  #         kubectl apply -f ./.github/actions/kots-gke-create/reserve-capacity.yaml
  #         sleep 10
  #         kubectl wait pod -l app=capacity-res-job --for=condition=Ready --timeout=600s --all

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --wait-duration 5m \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 600 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Delete the GKE cluster
  #       if: always()
  #       uses: ./.github/actions/kots-gke-delete
  #       with:
  #         service-key: ${{ secrets.GOOGLE_CLOUD_CREDENTIALS }}


  # validate-kots-pull:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: kind, version: v1.27.0}
  #       ]
  #   env:
  #     APP_NAME: multi-namespace-yeti
  #     APP_SLUG: multi-namespace
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run kots pull
  #       run: |
  #         set +e
  #         echo ${{ secrets.MULTI_NAMESPACE_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots pull "$APP_NAME/automated" \
  #           --license-file license.yaml \
  #           --shared-password password \
  #           --namespace "$APP_NAME" \
  #           --exclude-admin-console

  #         kubectl create ns "$APP_NAME"
  #         kubectl create ns nginx-test
  #         kubectl create ns redis-test
  #         kubectl create ns postgres-test

  #         # HACK: without operator, additonal namespaces don't get image pull secrets
  #         echo ${{ secrets.MULTI_NAMESPACE_REGISTRY_AUTH }} | base64 -d > replicated-registry-auth.json
  #         kubectl -n nginx-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
  #         kubectl -n redis-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
  #         kubectl -n redis-test create secret generic multi-namespace-yeti-redis-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
  #         kubectl -n postgres-test create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json
  #         kubectl -n default create secret generic multi-namespace-yeti-registry --type=kubernetes.io/dockerconfigjson --from-file=.dockerconfigjson=./replicated-registry-auth.json

  #         kustomize build "$PWD/$APP_SLUG/overlays/midstream" | kubectl apply -f -
  #         kustomize build "$PWD/$APP_SLUG/overlays/midstream/charts/redis" | kubectl apply -f -

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "Failed to apply spec"
  #           echo "------pods:"
  #           kubectl get pods -A
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         echo "Waiting for pods to start"

  #         COUNTER=1
  #         while [ "$(kubectl get pods --no-headers | grep -v Running | grep -cv Completed)" -gt 0 ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for pods to start"
  #             kubectl get pods -A
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         echo "All pods started"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_NAME"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-app-version-label:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: app-version-label
  #     APP_VERSION_LABEL: v1.0.0
  #     LATEST_APP_VERSION_LABEL: v1.0.1
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: test kots install with version label
  #       run: |
  #         set +e
  #         echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --app-version-label "$APP_VERSION_LABEL" \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$APP_VERSION_LABEL" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be installed with correct version label: $APP_VERSION_LABEL"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         printf "App is installed successfully with the correct version label: %s\n\n" "$APP_VERSION_LABEL"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #         # test setting DockerHub credentials

  #         set +e

  #         # TODO: deploy and check secrets are actually created and images are pulled
  #         ./bin/kots docker ensure-secret --dockerhub-username replicatedtests --dockerhub-password ${{ secrets.DOCKERHUB_RATELIMIT_PASSWORD }} -n "$APP_SLUG"
  #         ./bin/kots download -n "$APP_SLUG" --slug "$APP_SLUG"
  #         if grep "${APP_SLUG}-kotsadm-dockerhub" -w "./${APP_SLUG}/overlays/midstream/secret.yaml"; then
  #           echo "Found DockerHub secret in ${APP_SLUG} latest version"
  #         else
  #           echo "No DockerHub secret found in appication namespace"
  #           exit 1
  #         fi

  #     - name: remove the app
  #       run: |
  #         set +e
  #         ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force

  #     - name: test kots install without version label
  #       run: |
  #         set +e
  #         echo ${{ secrets.APP_VERSION_LABEL_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$LATEST_APP_VERSION_LABEL" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be installed with latest version label: $LATEST_APP_VERSION_LABEL"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         printf "App is installed successfully with the correct version label: %s\n\n" "$LATEST_APP_VERSION_LABEL"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-helm-install-order:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: helm-install-order
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.HELM_INSTALL_ORDER_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --skip-preflights \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         printf "App is installed successfully and is ready\n\n"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-yamlescape:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: yamlescape
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.YAMLESCAPE_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --skip-preflights \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         printf "App is installed successfully and is ready\n\n"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-no-redeploy-on-restart:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: no-redeploy-on-restart
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.NO_REDEPLOY_ON_RESTART_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --skip-preflights \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         # wait for application job to be created
  #         COUNTER=1
  #         while [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" == "0" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 60 ]; then
  #             echo "Timed out waiting for job to be created"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # delete the application job and restart the admin console
  #         kubectl delete jobs -n "$APP_SLUG" --all
  #         kubectl delete pods -l app=kotsadm -n "$APP_SLUG"

  #         # wait for old kotsadm pod to terminate
  #         COUNTER=1
  #         while [ "$(kubectl get pods -l app=kotsadm -n "$APP_SLUG" | awk 'NR>1' | wc -l)" != "1" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 60 ]; then
  #             echo "More than 1 kotsadm pod found"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # wait for new kotsadm pod to become ready
  #         kubectl wait --for=condition=ready pod -l app=kotsadm -n "$APP_SLUG" --timeout=60s

  #         # delay in case the app takes a bit to be deployed
  #         sleep 20

  #         # validate that the application wasn't re-deployed and the job wasn't re-created
  #         if [ "$(kubectl get jobs -l app=example,component=job -n "$APP_SLUG" --ignore-not-found | awk 'NR>1' | wc -l)" != "0" ]; then
  #             echo "App should not be re-deployed after restart"
  #             exit 1
  #         fi

  #         printf "Success. App was not re-deployed after restart\n\n"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-kubernetes-installer-preflight:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: kubernetes-installer-preflight
  #   steps:
  #     - uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e

  #         echo ${{ secrets.KUBERNETES_INSTALLER_PREFLIGHT_LICENSE }} | base64 -d > license.yaml

  #         # Fake kurl installation using the crd and installer spec from the application manifests:

  #         # Pull application manifests
  #         ./bin/kots pull "$APP_SLUG/automated" --license-file license.yaml --shared-password password

  #         # Apply installer crd
  #         kubectl apply -f "$APP_SLUG/upstream/installer-crd.yaml"

  #         # Wait for crd to be created
  #         kubectl wait --for condition=established --timeout=60s crd/installers.cluster.kurl.sh

  #         # Seems that the above does not always guarantee the crd exists? So just in case...
  #         sleep 10

  #         # Apply installer
  #         kubectl apply -f "$APP_SLUG/upstream/installer.yaml"

  #         # Create kurl-config configmap in kube-system
  #         kubectl create cm kurl-config -n kube-system --from-literal=installer_id=7cc8094

  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         # validate that preflight checks ran
  #         JSON_PATH="jsonpath={.data['automated-install-slug-$APP_SLUG']}"
  #         if [ "$(kubectl get cm kotsadm-tasks -n "$APP_SLUG" -o "$JSON_PATH" | grep -c pending_preflight)" != "1" ]; then
  #             echo "Preflight checks did not run"
  #             exit 1
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # try get apps without namespace (using kubeconfig)
  #         # validate that output is the same as above
  #         mkdir -p /tmp/.kube
  #         sudo cp "$KUBECONFIG" /tmp/.kube/config
  #         sudo chmod -R 777 /tmp/.kube
  #         export KUBECONFIG=/tmp/.kube/config
  #         kubectl config set-context --current --namespace="$APP_SLUG"
  #         if [ "$(./bin/kots get apps | awk 'NR>1{print $2}')" != "ready" ]; then
  #             echo "kots get apps output is not the same as above"
  #             exit 1
  #         fi

  #         printf "App is installed successfully and is ready\n\n"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-tag-and-digest:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'Tag and Digest'
  #         kots-namespace: 'tag-and-digest'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-kots-push-images-anonymous:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run kots admin-console push-images
  #       run: |
  #         set +e
  #         ./bin/kots admin-console push-images ./hack/tests/small.airgap ttl.sh/automated-${{ github.run_id }}


  # validate-kots-admin-console-generate-manifests:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: minimal-rbac
  #     BASE_KOTS_VERSION: v1.72.0
  #     NAMESPACE: generate-manifests
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run kots admin-console generate-manifests without k8s context
  #       run: |
  #         set +e
  #         ./bin/kots admin-console generate-manifests -n "$NAMESPACE" --shared-password password

  #     - name: validate that ./admin-console exists and is not empty
  #       run: |
  #         set +e
  #         if [ ! -d ./admin-console ]; then
  #           echo "admin-console directory does not exist"
  #           exit 1
  #         fi
  #         if [ -z "$(ls -A ./admin-console)" ]; then
  #           echo "admin-console directory is empty"
  #           exit 1
  #         fi

  #     - name: remove admin-console directory
  #       run: rm -rf ./admin-console

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download base kots version
  #       run: |
  #         curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
  #           && tar zxvf kots_linux_amd64.tar.gz \
  #           && mv kots "kots-$BASE_KOTS_VERSION"

  #     - name: create namespace
  #       run: |
  #         set +e
  #         kubectl create namespace "$NAMESPACE"

  #     - name: run kots admin-console generate-manifests using base kots version with k8s context
  #       run: |
  #         set +e
  #         "./kots-$BASE_KOTS_VERSION" admin-console generate-manifests -n "$NAMESPACE" --shared-password password

  #     - name: apply the generated manifests
  #       run: |
  #         set +e
  #         kubectl apply -f ./admin-console -n "$NAMESPACE"

  #     - name: wait for the kotsadm-minio-0 pod to be created
  #       run: |
  #         set +e
  #         COUNTER=1
  #         while ! kubectl get pods -n "$NAMESPACE" | grep -q kotsadm-minio-0; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 30 ]; then
  #             echo "timed out waiting for kotsadm-minio-0 pod to be created"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #     - name: wait for kotsadm-minio-0 pod to be ready
  #       run: |
  #         set +e
  #         kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"

  #     - name: wait for the kotsadm deployment to be ready
  #       run: |
  #         set +e
  #         kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"

  #     - name: run kots admin-console generate-manifests using new kots version with k8s context
  #       run: |
  #         set +e
  #         ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #     - name: validate that ./admin-console/minio-statefulset.yaml has initContainers since a migration is needed
  #       run: |
  #         set +e
  #         if ! grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
  #           echo "admin-console/minio-statefulset.yaml does not have initContainers"
  #           exit 1
  #         fi

  #     - name: apply the generated manifests
  #       run: |
  #         set +e
  #         kubectl apply -f ./admin-console -n "$NAMESPACE"

  #     - name: wait for kotsadm-minio-0 pod to be ready
  #       run: |
  #         set +e
  #         sleep 10
  #         kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"

  #     - name: wait for the kotsadm deployment to be ready
  #       run: |
  #         set +e
  #         kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"

  #     - name: run kots admin-console generate-manifests using new kots version with k8s context
  #       run: |
  #         set +e
  #         ./bin/kots admin-console generate-manifests -n "$NAMESPACE" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #     - name: validate that ./admin-console/minio-statefulset.yaml does not have initContainers since a migration is not needed
  #       run: |
  #         set +e
  #         if grep -qE 'initContainers' ./admin-console/minio-statefulset.yaml; then
  #           echo "admin-console/minio-statefulset.yaml has initContainers"
  #           exit 1
  #         fi

  #     - name: apply the generated manifests
  #       run: |
  #         set +e
  #         kubectl apply -f ./admin-console -n "$NAMESPACE"

  #     - name: wait for kotsadm-minio-0 pod to be ready
  #       run: |
  #         set +e
  #         sleep 10
  #         kubectl wait --for=condition=ready --timeout=180s pod/kotsadm-minio-0 -n "$NAMESPACE"

  #     - name: wait for the kotsadm deployment to be ready
  #       run: |
  #         set +e
  #         kubectl wait --for=condition=available --timeout=300s deployment/kotsadm -n "$NAMESPACE"

  #     - name: print pods and logs on failure
  #       if: failure()
  #       run: |
  #         echo "------pods:"
  #         kubectl -n "$NAMESPACE" get pods
  #         echo "------kotsadm logs"
  #         kubectl logs -l app=kotsadm --tail=100 --namespace "$NAMESPACE"
  #         echo "------kotsadm-minio logs"
  #         kubectl logs -l app=kotsadm-minio --tail=100 --namespace "$NAMESPACE"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: automated-${{ github.run_id }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-min-kots-version:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite, generate-tag ]
  #   env:
  #     APP_SLUG: min-kots-version
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: |
  #         docker load -i e2e/bin/e2e-deps.tar
  #         chmod +x e2e/bin/*
  #         chmod +x bin/*
  #         cp ./bin/kots /usr/local/bin/kubectl-kots
  #         sudo apt-get update -y && sudo apt-get install jq -y
  #     - uses: ./.github/actions/kots-e2e
  #       id: kots-e2e
  #       with:
  #         test-focus: 'Min KOTS Version'
  #         kots-namespace: 'min-kots-version'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         kotsadm-image-registry: ttl.sh
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
  #         k8s-cluster-skip-teardown: true

  #     - name: validate that kots install fails early
  #       run: |
  #         set +e

  #         result=$(kubectl kots install "$APP_SLUG/automated" --no-port-forward --namespace "$APP_SLUG" --shared-password password 2>&1 >/dev/null)
  #         echo "$result"

  #         if [[ "$result" == *"requires"* ]] && [[ "$result" == *"10000.0.0"* ]]; then
  #           exit 0
  #         else
  #           exit 1
  #         fi

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}


  # validate-target-kots-version:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite, generate-tag ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: |
  #         docker load -i e2e/bin/e2e-deps.tar
  #         chmod +x e2e/bin/*
  #         chmod +x bin/*
  #         cp ./bin/kots /usr/local/bin/kubectl-kots
  #         sudo apt-get update -y && sudo apt-get install jq -y
  #     - uses: ./.github/actions/kots-e2e
  #       id: kots-e2e
  #       with:
  #         test-focus: 'Target KOTS Version'
  #         kots-namespace: 'target-kots-version'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
  #         k8s-cluster-skip-teardown: true

  #     - name: validate that kots install fails early
  #       run: |
  #         set +e

  #         result=$(kubectl kots install target-kots-version/automated --no-port-forward --namespace target-kots-version --shared-password password 2>&1 >/dev/null)
  #         echo "$result"

  #         if [[ "$result" == *"requires"* ]] && [[ "$result" == *"1.0.0"* ]]; then
  #           exit 0
  #         else
  #           exit 1
  #         fi

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}


  # validate-range-kots-version:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite, generate-tag ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: |
  #         docker load -i e2e/bin/e2e-deps.tar
  #         chmod +x e2e/bin/*
  #         chmod +x bin/*
  #         cp ./bin/kots /usr/local/bin/kubectl-kots
  #         sudo apt-get update -y && sudo apt-get install jq -y
  #     - uses: ./.github/actions/kots-e2e
  #       id: kots-e2e
  #       with:
  #         test-focus: 'Range KOTS Version'
  #         kots-namespace: 'range-kots-version'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
  #         k8s-cluster-skip-teardown: true

  #     - name: validate that kots install fails early
  #       run: |
  #         set +e

  #         result=$(kubectl kots install range-kots-version/automated --no-port-forward --namespace range-kots-version --shared-password password 2>&1 >/dev/null)
  #         echo "$result"

  #         if [[ "$result" == *"requires"* ]] && [[ "$result" == *"11000.0.0"* ]]; then
  #           exit 0
  #         else
  #           exit 1
  #         fi

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.kots-e2e.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.kots-e2e.outputs.cluster-id }}


  # validate-kots-upgrade:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: k3s, version: v1.24},
  #         {distribution: k3s, version: v1.25},
  #         {distribution: k3s, version: v1.26},
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: postgres-to-rqlite
  #     BASE_KOTS_VERSION: v1.57.0
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download base kots version
  #       run: |
  #         curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
  #           && tar zxvf kots_linux_amd64.tar.gz \
  #           && mv kots "kots-$BASE_KOTS_VERSION"

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.POSTGRES_TO_RQLITE_LICENSE }} | base64 -d > license.yaml

  #         # install using the base KOTS version

  #         "./kots-$BASE_KOTS_VERSION" \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --port-forward=false \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # upgrade using the new KOTS version

  #         ./bin/kots admin-console upgrade \
  #           --namespace "$APP_SLUG" \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         # verify that the postgres to rqlite migration was successful

  #         if ! kubectl logs -l app=kotsadm --namespace "$APP_SLUG" | grep -q "Migrated from Postgres to rqlite successfully"; then
  #           echo "Failed to find a successful migration log line"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --all-containers --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         # verify that the minio migration happened

  #         if [ -z "$(kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o jsonpath='{.spec.template.spec.initContainers}')" ]; then
  #           echo "Failed to find initContainers in the kotsadm-minio statefulset"
  #           echo "kotsadm-minio statefulset:"
  #           kubectl get statefulset kotsadm-minio -n "$APP_SLUG" -o yaml
  #           exit 1
  #         fi

  #         # make sure app is still installed and ready

  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then
  #           echo "App is not ready after the upgrade"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         printf "App is still installed and is ready after the migration\n\n"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-kots-helm-release-secret-migration:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: helm-release-secret-migration
  #     RELEASE_NAME: helm-release-chart
  #     RELEASE_NAMESPACE: helm-release
  #     BASE_KOTS_VERSION: v1.94.0
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download base kots version
  #       run: |
  #         curl -LO "https://github.com/replicatedhq/kots/releases/download/$BASE_KOTS_VERSION/kots_linux_amd64.tar.gz" \
  #           && tar zxvf kots_linux_amd64.tar.gz \
  #           && mv kots "kots-$BASE_KOTS_VERSION"

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.HELM_RELEASE_SECRET_MIGRATION_LICENSE }} | base64 -d > license.yaml

  #         # install using the base KOTS version
  #         "./kots-$BASE_KOTS_VERSION" \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward=true \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$("./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             "./kots-$BASE_KOTS_VERSION" get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # verify that the helm release secret is created in the kotsadm namespace
  #         releaseSecretName=$(kubectl get secret -n "$APP_SLUG" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
  #         if [ -z "$releaseSecretName" ]; then
  #           echo "Failed to find the helm release secret in the $APP_SLUG namespace"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         # if there are more than one helm release secrets, fail
  #         if [ "$(echo "$releaseSecretName" | wc -l)" -gt 1 ]; then
  #           echo "Found more than one helm release secret in the $APP_SLUG namespace"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         # upgrade using the new KOTS version
  #         ./bin/kots admin-console upgrade \
  #           --namespace "$APP_SLUG" \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         # make a config change and redeploy the app
  #         ./bin/kots set config "$APP_SLUG" create_new_sequence=true --deploy --namespace "$APP_SLUG"

  #         # make sure app is still installed and ready
  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; then
  #           echo "App is not ready after the upgrade"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         COUNTER=1
  #         while [ "$(helm ls -n "$RELEASE_NAMESPACE" | grep "$RELEASE_NAME" | awk 'NR>0{print $3}')" != "2" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for $RELEASE_NAME to be upgraded to revision 2"
  #             helm ls -n "$RELEASE_NAMESPACE"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # verify that the helm release secret is created in the helm relase namespace
  #         COUNT=1
  #         releaseSecret=""
  #         while [ -z "$releaseSecret" ]; do
  #           ((COUNT += 1))
  #           if [ $COUNT -gt 10 ]; then
  #             echo "Timed out waiting for the helm release secret to be migrated"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #           releaseSecret=$(kubectl get secret "$releaseSecretName" -n "$RELEASE_NAMESPACE")
  #         done

  #         # verify that the release secret in app namepspace is deleted
  #         oldReleaseSecret=$(kubectl get secret "$releaseSecretName" -n "$APP_SLUG")
  #         if [ -n "$oldReleaseSecret" ]; then
  #           echo "Found the helm release secret in the $APP_SLUG namespace"
  #           echo "$oldReleaseSecret"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         # verify that there are two helm release secrets in the helm release namespace
  #         releaseSecretNames=$(kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME" -o jsonpath='{.items[*].metadata.name}')
  #         releaseSecretNamesCount=$(echo "$releaseSecretNames" | wc -w)
  #         if [ "$releaseSecretNamesCount" -ne 2 ]; then
  #           echo "Found $releaseSecretNamesCount helm release secrets in the $RELEASE_NAMESPACE namespace(Want: 2)"
  #           kubectl get secret -n "$RELEASE_NAMESPACE" -l owner=helm,name="$RELEASE_NAME"
  #           echo "kotsadm logs:"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit 1
  #         fi

  #         printf "Helm release secret migration test passed\n\n"
  #         ./bin/kots get apps --namespace "$APP_SLUG"

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-multi-app-backup-and-restore:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'multi-app-backup-and-restore'
  #         kots-namespace: 'multi-app-backup-and-restore'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'

  # validate-multi-app-install:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'multi-app-install'
  #         kots-namespace: 'multi-app-install'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-airgap-smoke-test:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3
  #     - name: download e2e deps
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: e2e
  #         path: e2e/bin/
  #     - run: docker load -i e2e/bin/e2e-deps.tar
  #     - run: chmod +x e2e/bin/*
  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/
  #     - run: chmod +x bin/*
  #     - uses: ./.github/actions/kots-e2e
  #       with:
  #         test-focus: 'airgap-smoke-test'
  #         kots-namespace: 'airgap-smoke-test'
  #         k8s-distribution: ${{ matrix.cluster.distribution }}
  #         k8s-version: ${{ matrix.cluster.version }}
  #         testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
  #         testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
  #         kots-airgap: true
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
  #         replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'


  # validate-remove-app:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27}
  #       ]
  #   env:
  #     APP_SLUG: remove-app
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
          
  #         echo ${{ secrets.REMOVE_APP_LICENSE }} | base64 -d > license.yaml
  #         HELM_APP_NAMESPACES=(postgres-test redis-test "$APP_SLUG")
  #         APP_NAMESPACES=(postgres-test redis-test nginx-test rabbitmq-system "$APP_SLUG")

  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # test that --undeploy deletes application resources from the cluster

  #         ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --undeploy

  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
  #           printf "App reference was not removed\n\n"
  #           exit 1
  #         fi

  #         if [ "$(kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
  #           printf "Application resources are still found in the cluster\n\n"
  #           kubectl get all,secrets,configmap,pvc -A -l "kots.io/app-slug=$APP_SLUG"
  #           exit 1
  #         fi

  #         if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
  #           printf "rabbitmqclusters.rabbitmq.com CRD was not removed\n\n"
  #           kubectl get crd rabbitmqclusters.rabbitmq.com
  #           exit 1
  #         fi

  #         for ns in "${HELM_APP_NAMESPACES[@]}"; do
  #           if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" != "0" ]; then
  #             printf "Application Helm release secrets are still found in the namepspace %s\n\n" "$ns"
  #             kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
  #             exit 1
  #           fi
  #         done

  #         # re-install and test that running the remove command without --undeploy does _not_ delete any resources from the cluster

  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         ./bin/kots remove "$APP_SLUG" --namespace "$APP_SLUG" --force

  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
  #           printf "App reference was not removed\n\n"
  #           exit 1
  #         fi

  #         for ns in "${APP_NAMESPACES[@]}"; do
  #           if [ "$(kubectl get all,secrets,configmap,pvc -n "$ns" -l "kots.io/app-slug=$APP_SLUG" --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
  #             printf "%s namespace does not contain any resources\n\n" "$ns"
  #             kubectl get all,secrets,configmap,pvc -n "$ns"
  #             exit 1
  #           fi
  #         done

  #         if [ "$(kubectl get crd rabbitmqclusters.rabbitmq.com --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
  #           printf "rabbitmqclusters.rabbitmq.com CRD was not found\n\n"
  #           kubectl get crd rabbitmqclusters.rabbitmq.com
  #           exit 1
  #         fi

  #         if [ "$(kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
  #           printf "rabbitmq-system namespace does not contain rabbitmqclusters\n\n"
  #           kubectl get rabbitmqclusters.rabbitmq.com -n rabbitmq-system
  #           exit 1
  #         fi

  #         for ns in "${HELM_APP_NAMESPACES[@]}"; do
  #           if [ "$(kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1 --ignore-not-found | wc -l | tr -d '\n')" == "0" ]; then
  #             printf "Application Helm release secrets are not found in namepspace %s\n\n" "$ns"
  #             kubectl get secret -n "$ns" --field-selector type=helm.sh/release.v1
  #             exit 1
  #           fi
  #         done

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-registry-check:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e

  #         ./bin/kots admin-console push-images \
  #           not-an-app.airgap localhost:1234/not-a-namespace \
  #           --registry-username not-a-username \
  #           --registry-password not-a-password > output.txt 2>&1

  #         if ! grep -q 'Failed to test access' output.txt; then
  #           printf "Expected registry validation to fail before pushing images, but did not.\n\n"
  #           cat output.txt
  #           exit 1
  #         fi

  #         rm output.txt

  #         ./bin/kots install not-an-app \
  #           --airgap-bundle not-an-app.airgap \
  #           --kotsadm-registry localhost:1234/not-a-namespace \
  #           --registry-username not-a-username \
  #           --registry-password not-a-password > output.txt 2>&1

  #         if ! grep -q 'Failed to test access' output.txt; then
  #           printf "Expected registry validation to fail before installation, but did not.\n\n"
  #           cat output.txt
  #           exit 1
  #         fi

  #         rm output.txt

  #         ./bin/kots upstream upgrade not-an-app \
  #           --airgap-bundle not-an-app.airgap \
  #           --kotsadm-registry localhost:1234/not-a-namespace \
  #           --registry-username not-a-username \
  #           --registry-password not-a-password \
  #           --namespace not-a-namespace > output.txt 2>&1

  #         if ! grep -q 'Failed to test access' output.txt; then
  #           printf "Expected registry validation to fail before upgrading, but did not.\n\n"
  #           cat output.txt
  #           exit 1
  #         fi

  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-native-helm-v2:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: native-helm-v2
  #     INITIAL_VERSION: '0.1.1'
  #     UPGRADE_VERSION: '0.2.1'
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - uses: azure/setup-helm@v3
  #       with:
  #         token: ${{ secrets.GITHUB_TOKEN }}

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.NATIVE_HELM_V2_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --app-version-label "$INITIAL_VERSION" \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$INITIAL_VERSION" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # validate that helm charts installed using both native helm workflow were deployed via the helm CLI correctly

  #         if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-chart-release; then
  #           printf "my-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
  #           helm ls -n "$APP_SLUG"
  #           exit 1
  #         fi

  #         if ! helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep -q my-other-chart-release; then
  #           printf "my-other-chart-release helm release not found in %s namespace\n\n" "$APP_SLUG"
  #           helm ls -n "$APP_SLUG"
  #           exit 1
  #         fi

  #         # validate that `helm get values` works for the v1beta2 chart

  #         if ! helm get values my-chart-release -n "$APP_SLUG" | grep -q my-value; then
  #           printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
  #           helm get values my-chart-release -n "$APP_SLUG"
  #           exit 1
  #         fi

  #         # upgrade the app version

  #         ./bin/kots upstream upgrade "$APP_SLUG" -n "$APP_SLUG" --deploy

  #         # wait for my-chart-release to be uninstalled and my-other-chart-release to be upgraded to revision 2
          
  #         COUNTER=1
  #         while [ "$(helm ls -n "$APP_SLUG" | awk 'NR>1{print $1}' | grep my-chart-release)" != "" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for my-chart-release to be uninstalled"
  #             helm ls -n "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         COUNTER=1
  #         while [ "$(helm ls -n "$APP_SLUG" | grep my-other-chart-release | awk 'NR>0{print $3}')" != "2" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for my-other-chart-release to be upgraded to revision 2"
  #             helm ls -n "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # validate that `helm get values` works for the upgraded v1beta2 chart
  #         if ! helm get values my-other-chart-release -n "$APP_SLUG" | grep -q my-value; then
  #           printf "my-value not found in helm values for my-chart-release in %s namespace\n\n" "$APP_SLUG"
  #           helm get values my-chart-release -n "$APP_SLUG"
  #           exit 1
  #         fi

  #         # wait for the app to be ready again

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $3}')" != "$UPGRADE_VERSION" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done


  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-deployment-orchestration:
  #   runs-on: ubuntu-20.04
  #   needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-mc, push-rqlite ]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       cluster: [
  #         {distribution: kind, version: v1.27.0}
  #       ] 
  #   env:
  #     APP_SLUG: deployment-orchestration
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v3

  #     - uses: azure/setup-helm@v3
  #       with:
  #         token: ${{ secrets.GITHUB_TOKEN }}

  #     - name: Create Cluster 
  #       id: create-cluster 
  #       uses: replicatedhq/replicated-actions/create-cluster@v1 
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         kubernetes-distribution: ${{ matrix.cluster.distribution }}
  #         kubernetes-version: ${{ matrix.cluster.version }}
  #         cluster-name: automated-kots-${{ github.run_id }}-${{ matrix.cluster.distribution }}-${{ matrix.cluster.version }}
  #         timeout-minutes: '120'
  #         ttl: 2h
  #         export-kubeconfig: true

  #     - name: download kots binary
  #       uses: actions/download-artifact@v3
  #       with:
  #         name: kots
  #         path: bin/

  #     - run: chmod +x bin/kots

  #     - name: run the test
  #       run: |
  #         set +e
  #         echo ${{ secrets.DEPLOYMENT_ORCHESTRATION_LICENSE }} | base64 -d > license.yaml
  #         ./bin/kots \
  #           install "$APP_SLUG/automated" \
  #           --license-file license.yaml \
  #           --no-port-forward \
  #           --namespace "$APP_SLUG" \
  #           --shared-password password \
  #           --kotsadm-registry ttl.sh \
  #           --kotsadm-namespace automated-${{ github.run_id }} \
  #           --kotsadm-tag 24h

  #         EXIT_CODE=$?
  #         if [ $EXIT_CODE -ne 0 ]; then
  #           echo "------pods:"
  #           kubectl -n "$APP_SLUG" get pods
  #           echo "------kotsadm logs"
  #           kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #           exit $EXIT_CODE
  #         fi

  #         function wait_for_log {
  #           local counter=1
  #           local timeout=30
  #           local log_pattern="$1"

  #           while ! kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "$log_pattern"; do
  #             ((counter += 1))
  #             if [ $counter -gt $timeout ]; then
  #               echo "Timed out waiting for log $log_pattern"
  #               exit 1
  #             fi
  #             sleep 1
  #           done
  #         }

  #         wait_for_log "applying phase -9999"

  #         wait_for_log "applying phase -3"
  #         wait_for_log "waiting for resource apiextensions.k8s.io/v1/CustomResourceDefinition/myresources.example.com in namespace $APP_SLUG to be ready"

  #         wait_for_log "applying phase -2"

  #         # validate that phase -1 has not deployed yet since we're waiting on the CR status fields
  #         if kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG" | grep -i "applying phase -1"; then
  #           printf "phase -1 was deployed before phase -2 completed"
  #           kubectl logs deploy/kotsadm -c kotsadm -n "$APP_SLUG"
  #           exit 1
  #         fi

  #         wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.extract=true"

  #         # set .status.tasks.extract=true
  #         kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"extract": true}}}' --type=merge

  #         wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.transform=true"

  #         # set .status.tasks.transform=true
  #         kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"transform": true}}}' --type=merge

  #         wait_for_log "waiting for resource example.com/v1beta1/MyResource/my-resource in namespace $APP_SLUG to have property .status.tasks.load=true"

  #         # set .status.tasks.load=true
  #         kubectl patch myresources.example.com my-resource -n "$APP_SLUG" -p '{"status": {"tasks": {"load": true}}}' --type=merge

  #         # now validate that the remaining phases are deployed (-1, 0, and 1)
  #         wait_for_log "applying phase -1"
  #         wait_for_log "waiting for resource apps/v1/Deployment/nginx-1 in namespace $APP_SLUG to be ready"

  #         wait_for_log "applying phase 0"
  #         wait_for_log "waiting for resource apps/v1/Deployment/nginx-2 in namespace $APP_SLUG to be ready"

  #         wait_for_log "applying phase 1"
  #         wait_for_log "waiting for resource /v1/Service/nginx-2 in namespace $APP_SLUG to be ready"

  #         # wait for the app to be ready
  #         COUNTER=1
  #         while [ "$(./bin/kots get apps --namespace "$APP_SLUG" | awk 'NR>1{print $2}')" != "ready" ]; do
  #           ((COUNTER += 1))
  #           if [ $COUNTER -gt 120 ]; then
  #             echo "Timed out waiting for app to be ready"
  #             ./bin/kots get apps --namespace "$APP_SLUG"
  #             echo "kotsadm logs:"
  #             kubectl logs -l app=kotsadm --tail=100 --namespace "$APP_SLUG"
  #             exit 1
  #           fi
  #           sleep 1
  #         done

  #         # remove the app
  #         ./bin/kots remove "$APP_SLUG" -n "$APP_SLUG" --undeploy

  #         wait_for_log "deleting resources in phase -1"
  #         wait_for_log "deleting resources in phase 0"
  #         wait_for_log "deleting resources in phase 1"
  #         wait_for_log "deleting resources in phase 2"
  #         wait_for_log "deleting resources in phase 3"

  #         # validate that the app reference was removed
  #         if [ "$(./bin/kots get apps --namespace "$APP_SLUG" --output=json | tr -d '\n')" != "[]" ]; then
  #           printf "App reference was not removed\n\n"
  #           exit 1
  #         fi


  #     - name: Generate support bundle on failure
  #       if: failure()
  #       uses: ./.github/actions/generate-support-bundle
  #       with:
  #         kots-namespace: "$APP_SLUG"
  #         aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
  #         aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'

  #     - name: Remove Cluster
  #       id: remove-cluster
  #       uses: replicatedhq/replicated-actions/remove-cluster@v1
  #       if: ${{ always() && steps.create-cluster.outputs.cluster-id != '' }}
  #       continue-on-error: true
  #       with:
  #         api-token: ${{ secrets.C11Y_MATRIX_TOKEN }}
  #         cluster-id: ${{ steps.create-cluster.outputs.cluster-id }}


  # validate-pr-tests:
  #   runs-on: ubuntu-20.04
  #   needs:
  #     # static and unit tests
  #     - lint-web
  #     - unit-test-web
  #     - vet-kots
  #     - ci-test-kots
  #     # testim tests
  #     - validate-existing-online-install-minimal
  #     - validate-smoke-test
  #     - validate-minimal-rbac
  #     - validate-backup-and-restore
  #     - validate-no-required-config
  #     - validate-version-history-pagination
  #     - validate-change-license
  #     - validate-tag-and-digest
  #     - validate-min-kots-version
  #     - validate-target-kots-version
  #     - validate-range-kots-version
  #     - validate-multi-app-backup-and-restore
  #     - validate-multi-app-install
  #     - validate-airgap-smoke-test
  #     - validate-strict-preflight-checks
  #     - validate-config
  #     # non-testim tests
  #     - validate-minimal-rbac-override
  #     - validate-multi-namespace
  #     - validate-kots-pull
  #     - validate-app-version-label
  #     - validate-helm-install-order
  #     - validate-yamlescape
  #     - validate-no-redeploy-on-restart
  #     - validate-kubernetes-installer-preflight
  #     - validate-kots-upgrade
  #     - validate-remove-app
  #     - validate-registry-check
  #     - validate-gke-install
  #     - validate-kots-helm-release-secret-migration
  #     - validate-native-helm-v2
  #     - validate-deployment-orchestration
  #     # cli-only tests
  #     - validate-kots-push-images-anonymous
  #   steps:
  #     - run: echo "All PR tests passed"


  # # this job will validate that the kurl add-on validation did not fail and that all pr-tests succeed
  # # it is used for the github branch protection rule
  # validate-success:
  #   runs-on: ubuntu-20.04
  #   needs:
  #     - validate-kurl-addon
  #     - validate-pr-tests
  #   if: always()
  #   steps:
  #     # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
  #     # if the validate-kurl-addon job failed or was cancelled, this job will fail. it's ok if this job was skipped
  #     - name: fail if validate-kurl-addon job failed
  #       if: needs.validate-kurl-addon.result == 'failure' || needs.validate-kurl-addon.result == 'cancelled'
  #       run: exit 1
  #     # if the validate-pr-tests job was not successful, this job will fail
  #     - name: fail if validate-pr-tests job was not successful
  #       if: needs.validate-pr-tests.result != 'success'
  #       run: exit 1
  #     # if the validate-pr-tests job was successful, this job will succeed
  #     - name: succeed if validate-pr-tests job succeeded
  #       if: needs.validate-pr-tests.result == 'success'
  #       run: echo "Validation succeeded"