upgrade.yml

name: "Upgrade"
on:
  push:
    branches: [2.9, 3.1, 3.2, 3.3]
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
      - '**.go'
      - 'go.mod'
      - 'snap/**'
      - '.github/workflows/upgrade.yml'
      - '.github/setup-lxd/**'
    branches-ignore:
      - 'main'
  workflow_dispatch:

permissions:
  contents: read

jobs:

  Upgrade:
    name: Upgrade
    runs-on: [self-hosted, linux, x64, aws, large]
    if: github.event.pull_request.draft == false
    strategy:
      fail-fast: false
      matrix:
        cloud: ["localhost", "microk8s"]
    env:
      CHARM_localhost: apache2
      CHARM_microk8s: prometheus-k8s
      DOCKER_REGISTRY: 10.152.183.69
      RUN_TEST: RUN
      UPGRADE_FLAGS_localhost: --build-agent
      UPGRADE_FLAGS_microk8s: --agent-stream=develop
      MODEL_TYPE_localhost: iaas
      MODEL_TYPE_microk8s: caas

    steps:
      - name: Install Dependencies
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          sudo snap install juju --channel=3.1/stable
          mkdir -p ~/.local/share
          echo "/snap/bin" >> $GITHUB_PATH

      - name: Checkout
        if: env.RUN_TEST == 'RUN'
        uses: actions/checkout@v3

      - name: Remove LXD
        if: env.RUN_TEST == 'RUN'
        run: |
          set -euxo pipefail
          sudo snap remove lxd --purge

      - name: Setup LXD
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'localhost'
        uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd

      - name: Set some variables
        if: env.RUN_TEST == 'RUN'
        run: |
          set -euxo pipefail
          
          echo "base-juju-version=$(juju version | cut -d '-' -f 1)" >> $GITHUB_OUTPUT
          upstreamJujuVersion=$(grep -r "const version =" version/version.go | sed -r 's/^const version = \"(.*)\"$/\1/')
          echo "upstream-juju-version=${upstreamJujuVersion}" >> $GITHUB_OUTPUT
          currentStableChannel="$(echo $upstreamJujuVersion | cut -d'.' -f1,2)/stable"
          currentStableVersion=$(snap info juju | yq ".channels[\"$currentStableChannel\"]" | cut -d' ' -f1)
          echo "current-stable-juju-version=$currentStableVersion" >> $GITHUB_OUTPUT
          echo "juju-db-version=4.4" >> $GITHUB_OUTPUT
        id: vars

      - name: Set up Go
        if: env.RUN_TEST == 'RUN'
        uses: actions/setup-go@v3
        with:
          go-version-file: 'go.mod'
          cache: true

      - name: setup env
        shell: bash
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH

      - name: Setup Docker Mirror
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s'
        shell: bash
        run: |
          (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["https://docker-cache.us-west-2.aws.jujuqa.com:443"]' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
          docker system info

      - name: Setup k8s
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s'
        uses: balchua/microk8s-actions@e99a1ffcd3bb2682d941104cf6c1a215c657903f
        with:
          channel: "1.28-strict/stable"
          addons: '["dns", "hostpath-storage"]'
          launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml"

      - name: Setup local caas registry
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s'
        run: |
          set -euxo pipefail
          
          # Become a CA
          mkdir ~/certs
          sudo cp /var/snap/microk8s/current/certs/ca.crt ~/certs/
          sudo cp /var/snap/microk8s/current/certs/ca.key ~/certs/
          sudo chmod a+wr ~/certs/ca.crt
          sudo chmod a+wr ~/certs/ca.key
          
          # Recognise CA
          sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates
          sudo update-ca-certificates
                    
          # Generate certs
          openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry"
          
          openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \
            -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext
          
          # Deploy registry
          cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg snap_microk8s "microk8s kubectl create -f -"
          
          # Wait for registry
          sg snap_microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true
          sg snap_microk8s "microk8s kubectl describe pod -n container-registry"
          curl https://${DOCKER_REGISTRY}/v2/

      - name: Mirror docker images required for juju bootstrap
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s'
        env:
          BASE_JUJU_TAG: ${{ steps.vars.outputs.base-juju-version }}
          JUJU_DB_TAG: ${{ steps.vars.outputs.juju-db-version }}
          CHARM_BASE: ubuntu-20.04
        run: |
          set -euxo pipefail
          
          # Shim in recognition for our CA to jujud-operator
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM jujusolutions/jujud-operator:${BASE_JUJU_TAG}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG}
          
          docker pull jujusolutions/juju-db:${JUJU_DB_TAG}
          docker tag jujusolutions/juju-db:${JUJU_DB_TAG} ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG}
          
          docker pull jujusolutions/charm-base:${CHARM_BASE}
          docker tag jujusolutions/charm-base:${CHARM_BASE} ${DOCKER_REGISTRY}/test-repo/charm-base:${CHARM_BASE}
          docker push ${DOCKER_REGISTRY}/test-repo/charm-base:${CHARM_BASE}

      - name: Bootstrap Juju - localhost
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'localhost'
        shell: bash
        run: |
          set -euxo pipefail
          
          juju bootstrap localhost c \
            --constraints "arch=$(go env GOARCH)"
          juju version
          
          juju add-model m
          juju set-model-constraints arch=$(go env GOARCH)
          juju status

      - name: Bootstrap Juju - microk8s
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s'

        # TODO: Enabling developer-mode is a bit of a hack to get this working for now.
        # Ideally, we would mock our own simplestream, similar to Jenkins, to select
        # and filter with as standard, instead of skipping over them with this flag
        run: |
          set -euxo pipefail
          
          sg snap_microk8s <<EOF
            juju bootstrap microk8s c \
              --constraints "arch=$(go env GOARCH)" \
              --config caas-image-repo="${DOCKER_REGISTRY}/test-repo" \
              --config features="[developer-mode]"
          EOF
          juju version

          juju add-model m
          juju set-model-constraints arch=$(go env GOARCH)
          juju status

      - name: Deploy some applications
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          
          # On k8s, we have to grant the app access to the cluster.
          DEPLOY_FLAGS=''
          if [[ ${{ matrix.cloud }} == 'microk8s' ]]; then
            DEPLOY_FLAGS='--trust'
          fi
          
          juju deploy ${CHARM_${{ matrix.cloud }}} $DEPLOY_FLAGS
          
          juju wait-for application ${CHARM_${{ matrix.cloud }}}
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30

      - name: Update Juju
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          sudo snap remove juju --purge
          make go-install

      - name: Build jujud image
        if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s'
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
          CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }}
        run: |
          set -euxo pipefail
          
          make operator-image
          
          # Shim in recognition for our CA to jujud-operator
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM jujusolutions/jujud-operator:${UPSTREAM_JUJU_TAG}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG}
          
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM jujusolutions/jujud-operator:${CURRENT_STABLE_JUJU_TAG}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG}

      - name: Preflight
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          juju status
          juju version

      - name: Test upgrade controller
        if: env.RUN_TEST == 'RUN'
        shell: bash
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
          CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }}
        run: |
          set -euxo pipefail
          
          OUTPUT=$(juju upgrade-controller --debug ${UPGRADE_FLAGS_${{ matrix.cloud }}})
          if [[ $OUTPUT == 'no upgrades available' ]]; then
            exit 1
          fi
          .github/verify-agent-version.sh ${MODEL_TYPE_${{ matrix.cloud }}} ${UPSTREAM_JUJU_TAG}
          
          PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
              echo "Panic found:"
              juju debug-log --replay --no-tail -m controller
              exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30

      - name: Test upgrade model
        if: env.RUN_TEST == 'RUN'
        shell: bash
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
        run: |
          set -euxo pipefail
          
          while true; do
            juju upgrade-model 2>&1 | tee output.log || true
            RES=$(cat output.log | grep "upgrade in progress" || echo "NOT-UPGRADING")
            if [ "$RES" = "NOT-UPGRADING" ]; then
              break
            fi
          done
          
          attempt=0
          while true; do
            UPDATED=$((juju show-model m --format=json || echo "") | jq -r '.m."agent-version"')
            if [[ $UPDATED == $UPSTREAM_JUJU_TAG* ]]; then
              break
            fi
            sleep 10
            attempt=$((attempt+1))
            if [ "$attempt" -eq 48 ]; then
              echo "Upgrade model timed out"
              exit 1
            fi
          done
          
          PANIC=$(juju debug-log --replay --no-tail | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
            echo "Panic found:"
            juju debug-log --replay --no-tail
            exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30

      - name: Wrap up
        if: env.RUN_TEST == 'RUN'
        run: |
          set -euxo pipefail
          
          juju version
          juju status
          
          sg snap_microk8s "microk8s kubectl get all -A" || true
          lxc ls || true