upgrade.yml

name: "Upgrade"
on:
  push:
    branches: [2.9, 3.1, 3.2, 3.3, main]
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
      - '**.go'
      - 'go.mod'
      - 'snap/**'
      - '.github/workflows/upgrade.yml'
  workflow_dispatch:

permissions:
  contents: read

jobs:

  Upgrade:
    name: Upgrade
    runs-on: [self-hosted, linux, arm64, aws, large]
    if: github.event.pull_request.draft == false
    strategy:
      fail-fast: false
      matrix:
        snap_version: ["2.8/stable"]
        model_type: ["localhost", "microk8s"]
    env:
      CHARM_localhost: apache2
      CHARM_microk8s: prometheus-k8s
      DOCKER_REGISTRY: 10.152.183.69
      RUN_TEST: RUN

    steps:
      - name: Install Dependencies
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          sudo snap install snapcraft --classic
          sudo snap install juju --classic --channel=${{ matrix.snap_version }}
          echo "/snap/bin" >> $GITHUB_PATH

      - name: Checkout
        if: env.RUN_TEST == 'RUN'
        uses: actions/checkout@v3

      - name: Setup LXD
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost'
        uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd
        with:
          channel: 4.0/candidate

      - name: Set some variables
        if: env.RUN_TEST == 'RUN'
        run: |
          set -euxo pipefail
          
          echo "base-juju-version=$(juju version | cut -d '-' -f 1)" >> $GITHUB_OUTPUT
          upstreamJujuVersion=$(grep -r "const version =" version/version.go | sed -r 's/^const version = \"(.*)\"$/\1/')
          echo "upstream-juju-version=${upstreamJujuVersion}" >> $GITHUB_OUTPUT
          currentStableChannel="$(echo $upstreamJujuVersion | cut -d'.' -f1,2)/stable"
          currentStableVersion=$(snap info juju | yq ".channels[\"$currentStableChannel\"]" | cut -d' ' -f1)
          echo "current-stable-juju-version=$currentStableVersion" >> $GITHUB_OUTPUT
          echo "juju-db-version=4.0" >> $GITHUB_OUTPUT
        id: vars

      - name: Set up Go
        if: env.RUN_TEST == 'RUN'
        uses: actions/setup-go@v3
        with:
          go-version-file: 'go.mod'
          cache: true

      - name: setup env
        shell: bash
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH

      - name: Setup Docker Mirror
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        shell: bash
        run: |
          (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["https://docker-cache.us-west-2.aws.jujuqa.com:443"]' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
          docker system info

      - name: Setup k8s
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        uses: balchua/microk8s-actions@e99a1ffcd3bb2682d941104cf6c1a215c657903f
        with:
          channel: "1.28/stable"
          addons: '["dns", "storage"]'
          launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml"

      - name: Setup local caas registry
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        run: |
          set -euxo pipefail
          
          # Become a CA
          mkdir ~/certs
          sudo cp /var/snap/microk8s/current/certs/ca.crt ~/certs/
          sudo cp /var/snap/microk8s/current/certs/ca.key ~/certs/
          sudo chmod a+wr ~/certs/ca.crt
          sudo chmod a+wr ~/certs/ca.key
          
          # Recognise CA
          sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates
          sudo update-ca-certificates
                    
          # Generate certs
          openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry"
          
          openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \
            -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext
          
          # Deploy registry
          cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg microk8s "microk8s kubectl create -f -"
          
          # Wait for registry
          sg microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true
          sg microk8s "microk8s kubectl describe pod -n container-registry"
          curl https://${DOCKER_REGISTRY}/v2/

      - name: Mirror docker images required for juju bootstrap
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        env:
          BASE_JUJU_TAG: ${{ steps.vars.outputs.base-juju-version }}
          JUJU_DB_TAG: ${{ steps.vars.outputs.juju-db-version }}
        run: |
          set -euxo pipefail
          
          # Shim in recognition for our CA to jujud-operator
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM jujusolutions/jujud-operator:${BASE_JUJU_TAG}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG}
          
          docker pull jujusolutions/juju-db:${JUJU_DB_TAG}
          docker tag jujusolutions/juju-db:${JUJU_DB_TAG} ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG}

      - name: Bootstrap Juju - localhost
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost'
        shell: bash
        run: |
          set -euxo pipefail
          
          # Juju 2.8 doesn't support IPv6
          lxc network set lxdbr0 ipv6.address none
          juju bootstrap localhost c
          juju add-model m
          
          juju status
          juju version

      - name: Bootstrap Juju - microk8s
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        env:
          JUJU_DB_TAG: ${{ steps.vars.outputs.juju-db-version }}

        # TODO: Enabling developer-mode is a bit of a hack to get this working for now.
        # Ideally, we would mock our own simplestream, similar to Jenkins, to select
        # and filter with as standard, instead of skipping over them with this flag
        run: |
          set -euxo pipefail
          
          sg microk8s <<EOF
            microk8s.kubectl apply -f "$GITHUB_WORKSPACE/.github/microk8s-juju-2.8-fix.yaml"
            juju bootstrap microk8s c \
              --config caas-image-repo="${DOCKER_REGISTRY}/test-repo" \
              --config features="[developer-mode]"
          EOF
          juju add-model m
          
          juju status
          juju version

      # The `wait-for` plugin is used after deploying an application
      # This was added in Juju 2.9, so it's not installed by the 2.8 snap
      # However we just need to install the `wait-for` binary ourselves,
      # and then Juju 2.8 can use it (amazing!)
      - name: Add `wait-for` plugin
        shell: bash
        run: |
          go install github.com/juju/juju/cmd/plugins/juju-wait-for

      - name: Deploy some applications
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          
          # For prometheus-k8s, we need to use revision 1 - this is the only
          # revision with metadata v1, hence can be deployed on 2.8.
          # 2.8 which doesn't have the --revision flag, so append the rev no.
          DEPLOY_NAME=${CHARM_${{ matrix.model_type }}}
          if [[ $DEPLOY_NAME == 'prometheus-k8s' ]]; then
            DEPLOY_NAME='prometheus-k8s-1'
          fi
          
          juju deploy $DEPLOY_NAME
          
          juju wait-for application ${CHARM_${{ matrix.model_type }}}
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30

      - name: Install local juju
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          which juju
          sudo snap remove juju --purge
          make install
          which juju

      - name: Build jujud image
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
          CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }}
        run: |
          set -euxo pipefail
          
          make operator-image
          
          # Shim in recognition for our CA to jujud-operator
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM jujusolutions/jujud-operator:${UPSTREAM_JUJU_TAG}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG}
          
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM jujusolutions/jujud-operator:${CURRENT_STABLE_JUJU_TAG}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG}

      - name: Preflight
        if: env.RUN_TEST == 'RUN'
        shell: bash
        run: |
          set -euxo pipefail
          juju status
          juju version

      - name: Test upgrade controller - localhost
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost'
        shell: bash
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
          CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }}
        run: |
          set -euxo pipefail
          
          # Upgrade to the latest stable.
          juju upgrade-controller --debug
          $GITHUB_WORKSPACE/.github/verify-agent-version.sh iaas $CURRENT_STABLE_JUJU_TAG
          
          juju upgrade-controller --build-agent --debug
          $GITHUB_WORKSPACE/.github/verify-agent-version.sh iaas "${UPSTREAM_JUJU_TAG}.1"
          
          PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
              echo "Panic found:"
              juju debug-log --replay --no-tail -m controller
              exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30

      - name: Test upgrade controller - microk8s
        if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s'
        shell: bash
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
          CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }}
        run: |
          set -euxo pipefail
          
          # Upgrade to the latest stable.
          juju upgrade-controller --debug
          $GITHUB_WORKSPACE/.github/verify-agent-version.sh caas $CURRENT_STABLE_JUJU_TAG
          
          # Upgrade to local built version.
          juju upgrade-controller --agent-stream=develop --debug
          $GITHUB_WORKSPACE/.github/verify-agent-version.sh caas $UPSTREAM_JUJU_TAG
          
          PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
              echo "Panic found:"
              juju debug-log --replay --no-tail -m controller
              exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30

      - name: Test upgrade model
        if: env.RUN_TEST == 'RUN'
        shell: bash
        env:
          UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }}
        run: |
          set -euxo pipefail
          
          while true; do
            juju upgrade-model 2>&1 | tee output.log || true
            RES=$(cat output.log | grep "upgrade in progress" || echo "NOT-UPGRADING")
            if [ "$RES" = "NOT-UPGRADING" ]; then
              break
            fi
          done
          
          attempt=0
          while true; do
            UPDATED=$((juju show-model m --format=json || echo "") | jq -r '.m."agent-version"')
            if [[ $UPDATED == $UPSTREAM_JUJU_TAG* ]]; then
              break
            fi
            sleep 10
            attempt=$((attempt+1))
            if [ "$attempt" -eq 48 ]; then
              echo "Upgrade model timed out"
              exit 1
            fi
          done
          
          PANIC=$(juju debug-log --replay --no-tail | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
            echo "Panic found:"
            juju debug-log --replay --no-tail
            exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30

      - name: Wrap up
        if: env.RUN_TEST == 'RUN'
        run: |
          set -euxo pipefail
          
          juju version
          juju status
          
          sg microk8s "microk8s kubectl get all -A" || true
          lxc ls || true