upgrade.yml

name: "Upgrade"
on:
  push:
    branches: [3.*]
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
      - '**.go'
      - 'go.mod'
      - 'snap/**'
      - '.github/workflows/upgrade.yml'
      - '.github/setup-lxd/**'
      - 'scripts/dqlite/**'
      - 'Makefile'
      - 'make_functions.sh'
    branches-ignore:
      - 'main'
  workflow_dispatch:

permissions:
  contents: read

jobs:
  setup:
    name: Upgrade pre-check
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.check.outputs.version }}
      channel: ${{ steps.check.outputs.channel }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Check
        id: check
        run: |
          set -euxo pipefail
          version=$(grep -r "const version =" version/version.go | sed -r 's/^const version = \"(.*)\"$/\1/')
          majmin=$(echo $version | awk -F'[.-]' '{print $1 "." $2}')
          snap info juju > snap-info.txt
          channel=$(awk -F':' -v ver=$majmin 'NR == 1, /channels/ {next} {gsub(/ /, "", $1); gsub(/ /, "", $2); if (!match($1,ver)) next; if ($2 == "--") next; print $1; exit}' snap-info.txt)
          echo "version=$version" >> $GITHUB_OUTPUT 
          echo "channel=$channel" >> $GITHUB_OUTPUT 

  upgrade:
    name: Upgrade
    runs-on: [self-hosted, linux, x64, aws, xlarge]
    timeout-minutes: 30
    needs: setup
    if: github.event.pull_request.draft == false && needs.setup.outputs.channel != ''
    strategy:
      fail-fast: false
      matrix:
        cloud: ["localhost", "microk8s"]
    env:
      CHARM_localhost: apache2
      CHARM_microk8s: prometheus-k8s
      DOCKER_REGISTRY: 10.152.183.69
      UPGRADE_FLAGS_localhost: --build-agent
      UPGRADE_FLAGS_microk8s: --agent-stream=develop
      MODEL_TYPE_localhost: iaas
      MODEL_TYPE_microk8s: caas

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup LXD
        if: matrix.cloud == 'localhost'
        uses: canonical/setup-lxd@4e959f8e0d9c5feb27d44c5e4d9a330a782edee0
    
      - name: Wait for LXD
        if: matrix.cloud == 'localhost'
        run: |
          while ! ip link show lxdbr0; do
            echo "Waiting for lxdbr0..."
            sleep 10
          done

      - name: Set some variables
        run: |
          set -euxo pipefail
          
          sourceJujuVersion=$(snap info juju | yq ".channels[\"${{ needs.setup.outputs.channel }}\"]" | cut -d' ' -f1)
          echo "source-juju-version=${sourceJujuVersion}" >> $GITHUB_OUTPUT
        id: vars

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: 'go.mod'
          cache: true

      - name: Setup Environment
        shell: bash
        run: |
          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH

      - name: Install Juju
        shell: bash
        env:
          JUJU_SNAP_CHANNEL: ${{ needs.setup.outputs.channel }}
        run: |
          set -euxo pipefail
          sudo snap install juju --channel=${JUJU_SNAP_CHANNEL}
          mkdir -p ~/.local/share
          echo "/snap/bin" >> $GITHUB_PATH

      - name: Setup Docker Mirror
        if: matrix.cloud == 'microk8s'
        shell: bash
        run: |
          (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["https://docker-cache.us-west-2.aws.jujuqa.com:443"]' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
          docker system info

      - name: Setup k8s
        if: matrix.cloud == 'microk8s'
        uses: balchua/microk8s-actions@13f73436011eb4925c22526f64fb3ecdd81289a9
        with:
          channel: "1.30-strict/stable"
          addons: '["dns", "hostpath-storage"]'
          launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml"

      - name: Setup local caas registry
        if: matrix.cloud == 'microk8s'
        run: |
          set -euxo pipefail
          
          # Become a CA
          mkdir ~/certs
          sudo cp /var/snap/microk8s/current/certs/ca.crt ~/certs/
          sudo cp /var/snap/microk8s/current/certs/ca.key ~/certs/
          sudo chmod a+wr ~/certs/ca.crt
          sudo chmod a+wr ~/certs/ca.key
          
          # Recognise CA
          sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates
          sudo update-ca-certificates
                    
          # Generate certs
          openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry"
          
          openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \
            -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext
          
          # Deploy registry
          cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg snap_microk8s "microk8s kubectl create -f -"
          
          # Wait for registry
          sg snap_microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true
          sg snap_microk8s "microk8s kubectl describe pod -n container-registry"
          curl https://${DOCKER_REGISTRY}/v2/

      - name: Mirror docker images required for juju bootstrap
        if: matrix.cloud == 'microk8s'
        env:
          SOURCE_JUJU_VERSION: ${{ steps.vars.outputs.source-juju-version }}
        run: |
          set -euxo pipefail
          
          # Shim in recognition for our CA to jujud-operator
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM docker.io/jujusolutions/jujud-operator:${SOURCE_JUJU_VERSION}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${SOURCE_JUJU_VERSION}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${SOURCE_JUJU_VERSION}
          
          DOCKER_USERNAME=${DOCKER_REGISTRY}/test-repo make seed-repository

      - name: Bootstrap Juju - localhost
        if: matrix.cloud == 'localhost'
        shell: bash
        run: |
          set -euxo pipefail
          
          juju bootstrap localhost c \
            --constraints "arch=$(go env GOARCH)"
          juju version
          
          juju add-model m
          juju set-model-constraints arch=$(go env GOARCH)
          juju status

      - name: Bootstrap Juju - microk8s
        if: matrix.cloud == 'microk8s'

        # TODO: Enabling developer-mode is a bit of a hack to get this working for now.
        # Ideally, we would mock our own simplestream, similar to Jenkins, to select
        # and filter with as standard, instead of skipping over them with this flag
        run: |
          set -euxo pipefail
          
          sg snap_microk8s <<EOF
            juju bootstrap microk8s c \
              --constraints "arch=$(go env GOARCH)" \
              --config caas-image-repo="${DOCKER_REGISTRY}/test-repo" \
              --config features="[developer-mode]"
          EOF
          juju version

          juju add-model m
          juju set-model-constraints arch=$(go env GOARCH)
          juju status

      - name: Deploy some applications
        shell: bash
        run: |
          set -euxo pipefail
          
          # On k8s, we have to grant the app access to the cluster.
          DEPLOY_FLAGS=''
          if [[ ${{ matrix.cloud }} == 'microk8s' ]]; then
            DEPLOY_FLAGS='--trust'
          fi
          
          juju deploy ${CHARM_${{ matrix.cloud }}} $DEPLOY_FLAGS
          
          juju wait-for application ${CHARM_${{ matrix.cloud }}}
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30

      - name: Update Juju
        shell: bash
        run: |
          sudo snap remove juju --purge
          make go-install

      - name: Build jujud image
        if: matrix.cloud == 'microk8s'
        env:
          TARGET_JUJU_VERSION: ${{ needs.setup.outputs.version }}
        run: |
          set -euxo pipefail
          
          make operator-image
          
          # Shim in recognition for our CA to jujud-operator
          BUILD_TEMP=$(mktemp -d)
          cp ~/certs/ca.crt $BUILD_TEMP/
          cat >$BUILD_TEMP/Dockerfile <<EOL
            FROM docker.io/jujusolutions/jujud-operator:${TARGET_JUJU_VERSION}
          
            COPY ca.crt /usr/local/share/ca-certificates/ca.crt
          
            RUN update-ca-certificates
          EOL
          docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${TARGET_JUJU_VERSION}
          docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${TARGET_JUJU_VERSION}

      - name: Preflight
        shell: bash
        run: |
          set -euxo pipefail
          juju status
          juju version

      - name: Test upgrade controller
        shell: bash
        env:
          TARGET_JUJU_VERSION: ${{ needs.setup.outputs.version }}
        run: |
          set -euxo pipefail
          
          OUTPUT=$(juju upgrade-controller --debug ${UPGRADE_FLAGS_${{ matrix.cloud }}})
          if [[ $OUTPUT == 'no upgrades available' ]]; then
            exit 1
          fi
          .github/verify-agent-version.sh ${MODEL_TYPE_${{ matrix.cloud }}} ${TARGET_JUJU_VERSION}
          
          PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
              echo "Panic found:"
              juju debug-log --replay --no-tail -m controller
              exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30

      - name: Test upgrade model
        shell: bash
        env:
          TARGET_JUJU_VERSION: ${{ needs.setup.outputs.version }}
        run: |
          set -euxo pipefail
          
          while true; do
            juju upgrade-model 2>&1 | tee output.log || true
            RES=$(cat output.log | grep "upgrade in progress" || echo "NOT-UPGRADING")
            if [ "$RES" = "NOT-UPGRADING" ]; then
              break
            fi
          done
          
          attempt=0
          while true; do
            UPDATED=$((juju show-model m --format=json || echo "") | jq -r '.m."agent-version"')
            if [[ $UPDATED == $TARGET_JUJU_VERSION* ]]; then
              break
            fi
            sleep 10
            attempt=$((attempt+1))
            if [ "$attempt" -eq 48 ]; then
              echo "Upgrade model timed out"
              exit 1
            fi
          done
          
          PANIC=$(juju debug-log --replay --no-tail | grep "panic" || true)
          if [ "$PANIC" != "" ]; then
            echo "Panic found:"
            juju debug-log --replay --no-tail
            exit 1
          fi
          
          $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30

      - name: Wrap up
        run: |
          set -euxo pipefail
          
          juju version
          juju status
          
          sg snap_microk8s "microk8s kubectl get all -A" || true
          lxc ls || true