Merge pull request #17816 from manadart/3.4-peergrouper-status #811
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Upgrade" | |
| on: | |
| push: | |
| branches: [3.*] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths: | |
| - '**.go' | |
| - 'go.mod' | |
| - 'snap/**' | |
| - '.github/workflows/upgrade.yml' | |
| - '.github/setup-lxd/**' | |
| - 'scripts/dqlite/**' | |
| - 'Makefile' | |
| - 'make_functions.sh' | |
| branches-ignore: | |
| - 'main' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| setup: | |
| name: Upgrade pre-check | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.check.outputs.version }} | |
| channel: ${{ steps.check.outputs.channel }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check | |
| id: check | |
| run: | | |
| set -euxo pipefail | |
| version=$(grep -r "const version =" version/version.go | sed -r 's/^const version = \"(.*)\"$/\1/') | |
| majmin=$(echo $version | awk -F'[.-]' '{print $1 "." $2}') | |
| snap info juju > snap-info.txt | |
| channel=$(awk -F':' -v ver=$majmin 'NR == 1, /channels/ {next} {gsub(/ /, "", $1); gsub(/ /, "", $2); if (!match($1,ver)) next; if ($2 == "--") next; print $1; exit}' snap-info.txt) | |
| echo "version=$version" >> $GITHUB_OUTPUT | |
| echo "channel=$channel" >> $GITHUB_OUTPUT | |
| upgrade: | |
| name: Upgrade | |
| runs-on: [self-hosted, linux, x64, aws, xlarge] | |
| timeout-minutes: 30 | |
| needs: setup | |
| if: github.event.pull_request.draft == false && needs.setup.outputs.channel != '' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cloud: ["localhost", "microk8s"] | |
| env: | |
| CHARM_localhost: apache2 | |
| CHARM_microk8s: prometheus-k8s | |
| DOCKER_REGISTRY: 10.152.183.69 | |
| UPGRADE_FLAGS_localhost: --build-agent | |
| UPGRADE_FLAGS_microk8s: --agent-stream=develop | |
| MODEL_TYPE_localhost: iaas | |
| MODEL_TYPE_microk8s: caas | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup LXD | |
| if: matrix.cloud == 'localhost' | |
| uses: canonical/setup-lxd@4e959f8e0d9c5feb27d44c5e4d9a330a782edee0 | |
| - name: Wait for LXD | |
| if: matrix.cloud == 'localhost' | |
| run: | | |
| while ! ip link show lxdbr0; do | |
| echo "Waiting for lxdbr0..." | |
| sleep 10 | |
| done | |
| - name: Set some variables | |
| run: | | |
| set -euxo pipefail | |
| sourceJujuVersion=$(snap info juju | yq ".channels[\"${{ needs.setup.outputs.channel }}\"]" | cut -d' ' -f1) | |
| echo "source-juju-version=${sourceJujuVersion}" >> $GITHUB_OUTPUT | |
| id: vars | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| cache: true | |
| - name: Setup Environment | |
| shell: bash | |
| run: | | |
| echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Install Juju | |
| shell: bash | |
| env: | |
| JUJU_SNAP_CHANNEL: ${{ needs.setup.outputs.channel }} | |
| run: | | |
| set -euxo pipefail | |
| sudo snap install juju --channel=${JUJU_SNAP_CHANNEL} | |
| mkdir -p ~/.local/share | |
| echo "/snap/bin" >> $GITHUB_PATH | |
| - name: Setup Docker Mirror | |
| if: matrix.cloud == 'microk8s' | |
| shell: bash | |
| run: | | |
| (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["https://docker-cache.us-west-2.aws.jujuqa.com:443"]' | sudo tee /etc/docker/daemon.json | |
| sudo systemctl restart docker | |
| docker system info | |
| - name: Setup k8s | |
| if: matrix.cloud == 'microk8s' | |
| uses: balchua/microk8s-actions@13f73436011eb4925c22526f64fb3ecdd81289a9 | |
| with: | |
| channel: "1.28-strict/stable" | |
| addons: '["dns", "hostpath-storage"]' | |
| launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml" | |
| - name: Setup local caas registry | |
| if: matrix.cloud == 'microk8s' | |
| run: | | |
| set -euxo pipefail | |
| # Become a CA | |
| mkdir ~/certs | |
| sudo cp /var/snap/microk8s/current/certs/ca.crt ~/certs/ | |
| sudo cp /var/snap/microk8s/current/certs/ca.key ~/certs/ | |
| sudo chmod a+wr ~/certs/ca.crt | |
| sudo chmod a+wr ~/certs/ca.key | |
| # Recognise CA | |
| sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates | |
| sudo update-ca-certificates | |
| # Generate certs | |
| openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry" | |
| openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \ | |
| -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext | |
| # Deploy registry | |
| cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg snap_microk8s "microk8s kubectl create -f -" | |
| # Wait for registry | |
| sg snap_microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true | |
| sg snap_microk8s "microk8s kubectl describe pod -n container-registry" | |
| curl https://${DOCKER_REGISTRY}/v2/ | |
| - name: Mirror docker images required for juju bootstrap | |
| if: matrix.cloud == 'microk8s' | |
| env: | |
| SOURCE_JUJU_VERSION: ${{ steps.vars.outputs.source-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| # Shim in recognition for our CA to jujud-operator | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM docker.io/jujusolutions/jujud-operator:${SOURCE_JUJU_VERSION} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${SOURCE_JUJU_VERSION} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${SOURCE_JUJU_VERSION} | |
| DOCKER_USERNAME=${DOCKER_REGISTRY}/test-repo make seed-repository | |
| - name: Bootstrap Juju - localhost | |
| if: matrix.cloud == 'localhost' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| juju bootstrap localhost c \ | |
| --constraints "arch=$(go env GOARCH)" | |
| juju version | |
| juju add-model m | |
| juju set-model-constraints arch=$(go env GOARCH) | |
| juju status | |
| - name: Bootstrap Juju - microk8s | |
| if: matrix.cloud == 'microk8s' | |
| # TODO: Enabling developer-mode is a bit of a hack to get this working for now. | |
| # Ideally, we would mock our own simplestream, similar to Jenkins, to select | |
| # and filter with as standard, instead of skipping over them with this flag | |
| run: | | |
| set -euxo pipefail | |
| sg snap_microk8s <<EOF | |
| juju bootstrap microk8s c \ | |
| --constraints "arch=$(go env GOARCH)" \ | |
| --config caas-image-repo="${DOCKER_REGISTRY}/test-repo" \ | |
| --config features="[developer-mode]" | |
| EOF | |
| juju version | |
| juju add-model m | |
| juju set-model-constraints arch=$(go env GOARCH) | |
| juju status | |
| - name: Deploy some applications | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| # On k8s, we have to grant the app access to the cluster. | |
| DEPLOY_FLAGS='' | |
| if [[ ${{ matrix.cloud }} == 'microk8s' ]]; then | |
| DEPLOY_FLAGS='--trust' | |
| fi | |
| juju deploy ${CHARM_${{ matrix.cloud }}} $DEPLOY_FLAGS | |
| juju wait-for application ${CHARM_${{ matrix.cloud }}} | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30 | |
| - name: Update Juju | |
| shell: bash | |
| run: | | |
| sudo snap remove juju --purge | |
| make go-install | |
| - name: Build jujud image | |
| if: matrix.cloud == 'microk8s' | |
| env: | |
| TARGET_JUJU_VERSION: ${{ needs.setup.outputs.version }} | |
| run: | | |
| set -euxo pipefail | |
| make operator-image | |
| # Shim in recognition for our CA to jujud-operator | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM docker.io/jujusolutions/jujud-operator:${TARGET_JUJU_VERSION} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${TARGET_JUJU_VERSION} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${TARGET_JUJU_VERSION} | |
| - name: Preflight | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| juju status | |
| juju version | |
| - name: Test upgrade controller | |
| shell: bash | |
| env: | |
| TARGET_JUJU_VERSION: ${{ needs.setup.outputs.version }} | |
| run: | | |
| set -euxo pipefail | |
| OUTPUT=$(juju upgrade-controller --debug ${UPGRADE_FLAGS_${{ matrix.cloud }}}) | |
| if [[ $OUTPUT == 'no upgrades available' ]]; then | |
| exit 1 | |
| fi | |
| .github/verify-agent-version.sh ${MODEL_TYPE_${{ matrix.cloud }}} ${TARGET_JUJU_VERSION} | |
| PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail -m controller | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30 | |
| - name: Test upgrade model | |
| shell: bash | |
| env: | |
| TARGET_JUJU_VERSION: ${{ needs.setup.outputs.version }} | |
| run: | | |
| set -euxo pipefail | |
| while true; do | |
| juju upgrade-model 2>&1 | tee output.log || true | |
| RES=$(cat output.log | grep "upgrade in progress" || echo "NOT-UPGRADING") | |
| if [ "$RES" = "NOT-UPGRADING" ]; then | |
| break | |
| fi | |
| done | |
| attempt=0 | |
| while true; do | |
| UPDATED=$((juju show-model m --format=json || echo "") | jq -r '.m."agent-version"') | |
| if [[ $UPDATED == $TARGET_JUJU_VERSION* ]]; then | |
| break | |
| fi | |
| sleep 10 | |
| attempt=$((attempt+1)) | |
| if [ "$attempt" -eq 48 ]; then | |
| echo "Upgrade model timed out" | |
| exit 1 | |
| fi | |
| done | |
| PANIC=$(juju debug-log --replay --no-tail | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.cloud }}}.sh 30 | |
| - name: Wrap up | |
| run: | | |
| set -euxo pipefail | |
| juju version | |
| juju status | |
| sg snap_microk8s "microk8s kubectl get all -A" || true | |
| lxc ls || true |