Merge pull request #18237 from hpidcock/merge-2.9-3.1-20241015 #845
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Upgrade" | |
| on: | |
| push: | |
| branches: [2.9, 3.1, 3.2, 3.3] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths: | |
| - '**.go' | |
| - 'go.mod' | |
| - 'snap/**' | |
| - '.github/workflows/upgrade.yml' | |
| - '.github/setup-lxd/**' | |
| branches-ignore: | |
| - 'main' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| Upgrade: | |
| name: Upgrade | |
| runs-on: [self-hosted, linux, x64, aws, large] | |
| if: github.event.pull_request.draft == false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cloud: ["localhost", "microk8s"] | |
| env: | |
| CHARM_localhost: apache2 | |
| CHARM_microk8s: prometheus-k8s | |
| DOCKER_REGISTRY: 10.152.183.69 | |
| RUN_TEST: RUN | |
| UPGRADE_FLAGS_localhost: --build-agent | |
| UPGRADE_FLAGS_microk8s: --agent-stream=develop | |
| MODEL_TYPE_localhost: iaas | |
| MODEL_TYPE_microk8s: caas | |
| steps: | |
| - name: Install Dependencies | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| sudo snap install juju --channel=3.1/stable | |
| mkdir -p ~/.local/share | |
| echo "/snap/bin" >> $GITHUB_PATH | |
| - name: Checkout | |
| if: env.RUN_TEST == 'RUN' | |
| uses: actions/checkout@v3 | |
| - name: Remove LXD | |
| if: env.RUN_TEST == 'RUN' | |
| run: | | |
| set -euxo pipefail | |
| sudo snap remove lxd --purge | |
| - name: Setup LXD | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'localhost' | |
| uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd | |
| - name: Set some variables | |
| if: env.RUN_TEST == 'RUN' | |
| run: | | |
| set -euxo pipefail | |
| echo "base-juju-version=$(juju version | cut -d '-' -f 1)" >> $GITHUB_OUTPUT | |
| upstreamJujuVersion=$(grep -r "const version =" version/version.go | sed -r 's/^const version = \"(.*)\"$/\1/') | |
| echo "upstream-juju-version=${upstreamJujuVersion}" >> $GITHUB_OUTPUT | |
| currentStableChannel="$(echo $upstreamJujuVersion | cut -d'.' -f1,2)/stable" | |
| currentStableVersion=$(snap info juju | yq ".channels[\"$currentStableChannel\"]" | cut -d' ' -f1) | |
| echo "current-stable-juju-version=$currentStableVersion" >> $GITHUB_OUTPUT | |
| echo "juju-db-version=4.4" >> $GITHUB_OUTPUT | |
| id: vars | |
| - name: Set up Go | |
| if: env.RUN_TEST == 'RUN' | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version-file: 'go.mod' | |
| cache: true | |
| - name: setup env | |
| shell: bash | |
| run: | | |
| echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Setup Docker Mirror | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s' | |
| shell: bash | |
| run: | | |
| (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["https://docker-cache.us-west-2.aws.jujuqa.com:443"]' | sudo tee /etc/docker/daemon.json | |
| sudo systemctl restart docker | |
| docker system info | |
| - name: Setup k8s | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s' | |
| uses: balchua/microk8s-actions@e99a1ffcd3bb2682d941104cf6c1a215c657903f | |
| with: | |
| channel: "1.28-strict/stable" | |
| addons: '["dns", "hostpath-storage"]' | |
| launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml" | |
| - name: Setup local caas registry | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s' | |
| run: | | |
| set -euxo pipefail | |
| # Become a CA | |
| mkdir ~/certs | |
| sudo cp /var/snap/microk8s/current/certs/ca.crt ~/certs/ | |
| sudo cp /var/snap/microk8s/current/certs/ca.key ~/certs/ | |
| sudo chmod a+wr ~/certs/ca.crt | |
| sudo chmod a+wr ~/certs/ca.key | |
| # Recognise CA | |
| sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates | |
| sudo update-ca-certificates | |
| # Generate certs | |
| openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry" | |
| openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \ | |
| -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext | |
| # Deploy registry | |
| cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg snap_microk8s "microk8s kubectl create -f -" | |
| # Wait for registry | |
| sg snap_microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true | |
| sg snap_microk8s "microk8s kubectl describe pod -n container-registry" | |
| curl https://${DOCKER_REGISTRY}/v2/ | |
| - name: Mirror docker images required for juju bootstrap | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s' | |
| env: | |
| BASE_JUJU_TAG: ${{ steps.vars.outputs.base-juju-version }} | |
| JUJU_DB_TAG: ${{ steps.vars.outputs.juju-db-version }} | |
| CHARM_BASE: ubuntu-20.04 | |
| run: | | |
| set -euxo pipefail | |
| # Shim in recognition for our CA to jujud-operator | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM jujusolutions/jujud-operator:${BASE_JUJU_TAG} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG} | |
| docker pull jujusolutions/juju-db:${JUJU_DB_TAG} | |
| docker tag jujusolutions/juju-db:${JUJU_DB_TAG} ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG} | |
| docker pull jujusolutions/charm-base:${CHARM_BASE} | |
| docker tag jujusolutions/charm-base:${CHARM_BASE} ${DOCKER_REGISTRY}/test-repo/charm-base:${CHARM_BASE} | |
| docker push ${DOCKER_REGISTRY}/test-repo/charm-base:${CHARM_BASE} | |
| - name: Bootstrap Juju - localhost | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'localhost' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| juju bootstrap localhost c \ | |
| --constraints "arch=$(go env GOARCH)" | |
| juju version | |
| juju add-model m | |
| juju set-model-constraints arch=$(go env GOARCH) | |
| juju status | |
| - name: Bootstrap Juju - microk8s | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s' | |
| # TODO: Enabling developer-mode is a bit of a hack to get this working for now. | |
| # Ideally, we would mock our own simplestream, similar to Jenkins, to select | |
| # and filter with as standard, instead of skipping over them with this flag | |
| run: | | |
| set -euxo pipefail | |
| sg snap_microk8s <<EOF | |
| juju bootstrap microk8s c \ | |
| --constraints "arch=$(go env GOARCH)" \ | |
| --config caas-image-repo="${DOCKER_REGISTRY}/test-repo" \ | |
| --config features="[developer-mode]" | |
| EOF | |
| juju version | |
| juju add-model m | |
| juju set-model-constraints arch=$(go env GOARCH) | |
| juju status | |
| - name: Deploy some applications | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| # On k8s, we have to grant the app access to the cluster. | |
| DEPLOY_FLAGS='' | |
| if [[ ${{ matrix.cloud }} == 'microk8s' ]]; then | |
| DEPLOY_FLAGS='--trust' | |
| fi | |
| juju deploy ${CHARM_${{ matrix.cloud }}} $DEPLOY_FLAGS | |
| juju wait-for application ${CHARM_${{ matrix.cloud }}} | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30 | |
| - name: Update Juju | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| sudo snap remove juju --purge | |
| make go-install | |
| - name: Build jujud image | |
| if: env.RUN_TEST == 'RUN' && matrix.cloud == 'microk8s' | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| make operator-image | |
| # Shim in recognition for our CA to jujud-operator | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM jujusolutions/jujud-operator:${UPSTREAM_JUJU_TAG} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG} | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM jujusolutions/jujud-operator:${CURRENT_STABLE_JUJU_TAG} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG} | |
| - name: Preflight | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| juju status | |
| juju version | |
| - name: Test upgrade controller | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| OUTPUT=$(juju upgrade-controller --debug ${UPGRADE_FLAGS_${{ matrix.cloud }}}) | |
| if [[ $OUTPUT == 'no upgrades available' ]]; then | |
| exit 1 | |
| fi | |
| .github/verify-agent-version.sh ${MODEL_TYPE_${{ matrix.cloud }}} ${UPSTREAM_JUJU_TAG} | |
| PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail -m controller | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30 | |
| - name: Test upgrade model | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| while true; do | |
| juju upgrade-model 2>&1 | tee output.log || true | |
| RES=$(cat output.log | grep "upgrade in progress" || echo "NOT-UPGRADING") | |
| if [ "$RES" = "NOT-UPGRADING" ]; then | |
| break | |
| fi | |
| done | |
| attempt=0 | |
| while true; do | |
| UPDATED=$((juju show-model m --format=json || echo "") | jq -r '.m."agent-version"') | |
| if [[ $UPDATED == $UPSTREAM_JUJU_TAG* ]]; then | |
| break | |
| fi | |
| sleep 10 | |
| attempt=$((attempt+1)) | |
| if [ "$attempt" -eq 48 ]; then | |
| echo "Upgrade model timed out" | |
| exit 1 | |
| fi | |
| done | |
| PANIC=$(juju debug-log --replay --no-tail | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30 | |
| - name: Wrap up | |
| if: env.RUN_TEST == 'RUN' | |
| run: | | |
| set -euxo pipefail | |
| juju version | |
| juju status | |
| sg snap_microk8s "microk8s kubectl get all -A" || true | |
| lxc ls || true |