Merge pull request #17380 from jack-w-shaw/bump_os_patch #756
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Upgrade" | |
| on: | |
| push: | |
| branches: [2.9, 3.1, 3.2, 3.3, main] | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths: | |
| - '**.go' | |
| - 'go.mod' | |
| - 'snap/**' | |
| - '.github/workflows/upgrade.yml' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| Upgrade: | |
| name: Upgrade | |
| runs-on: [self-hosted, linux, arm64, aws, large] | |
| if: github.event.pull_request.draft == false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| snap_version: ["2.8/stable"] | |
| model_type: ["localhost", "microk8s"] | |
| env: | |
| CHARM_localhost: apache2 | |
| CHARM_microk8s: prometheus-k8s | |
| DOCKER_REGISTRY: 10.152.183.69 | |
| RUN_TEST: RUN | |
| steps: | |
| - name: Install Dependencies | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| sudo snap install snapcraft --classic | |
| sudo snap install juju --classic --channel=${{ matrix.snap_version }} | |
| echo "/snap/bin" >> $GITHUB_PATH | |
| - name: Checkout | |
| if: env.RUN_TEST == 'RUN' | |
| uses: actions/checkout@v3 | |
| - name: Setup LXD | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost' | |
| uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd | |
| with: | |
| channel: 4.0/candidate | |
| - name: Set some variables | |
| if: env.RUN_TEST == 'RUN' | |
| run: | | |
| set -euxo pipefail | |
| echo "base-juju-version=$(juju version | cut -d '-' -f 1)" >> $GITHUB_OUTPUT | |
| upstreamJujuVersion=$(grep -r "const version =" version/version.go | sed -r 's/^const version = \"(.*)\"$/\1/') | |
| echo "upstream-juju-version=${upstreamJujuVersion}" >> $GITHUB_OUTPUT | |
| currentStableChannel="$(echo $upstreamJujuVersion | cut -d'.' -f1,2)/stable" | |
| currentStableVersion=$(snap info juju | yq ".channels[\"$currentStableChannel\"]" | cut -d' ' -f1) | |
| echo "current-stable-juju-version=$currentStableVersion" >> $GITHUB_OUTPUT | |
| echo "juju-db-version=4.0" >> $GITHUB_OUTPUT | |
| id: vars | |
| - name: Set up Go | |
| if: env.RUN_TEST == 'RUN' | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version-file: 'go.mod' | |
| cache: true | |
| - name: setup env | |
| shell: bash | |
| run: | | |
| echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Setup Docker Mirror | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| shell: bash | |
| run: | | |
| (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["https://docker-cache.us-west-2.aws.jujuqa.com:443"]' | sudo tee /etc/docker/daemon.json | |
| sudo systemctl restart docker | |
| docker system info | |
| - name: Setup k8s | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| uses: balchua/microk8s-actions@e99a1ffcd3bb2682d941104cf6c1a215c657903f | |
| with: | |
| channel: "1.28/stable" | |
| addons: '["dns", "storage"]' | |
| launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml" | |
| - name: Setup local caas registry | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| run: | | |
| set -euxo pipefail | |
| # Become a CA | |
| mkdir ~/certs | |
| sudo cp /var/snap/microk8s/current/certs/ca.crt ~/certs/ | |
| sudo cp /var/snap/microk8s/current/certs/ca.key ~/certs/ | |
| sudo chmod a+wr ~/certs/ca.crt | |
| sudo chmod a+wr ~/certs/ca.key | |
| # Recognise CA | |
| sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates | |
| sudo update-ca-certificates | |
| # Generate certs | |
| openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry" | |
| openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \ | |
| -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext | |
| # Deploy registry | |
| cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg microk8s "microk8s kubectl create -f -" | |
| # Wait for registry | |
| sg microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true | |
| sg microk8s "microk8s kubectl describe pod -n container-registry" | |
| curl https://${DOCKER_REGISTRY}/v2/ | |
| - name: Mirror docker images required for juju bootstrap | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| env: | |
| BASE_JUJU_TAG: ${{ steps.vars.outputs.base-juju-version }} | |
| JUJU_DB_TAG: ${{ steps.vars.outputs.juju-db-version }} | |
| run: | | |
| set -euxo pipefail | |
| # Shim in recognition for our CA to jujud-operator | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM jujusolutions/jujud-operator:${BASE_JUJU_TAG} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${BASE_JUJU_TAG} | |
| docker pull jujusolutions/juju-db:${JUJU_DB_TAG} | |
| docker tag jujusolutions/juju-db:${JUJU_DB_TAG} ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/juju-db:${JUJU_DB_TAG} | |
| - name: Bootstrap Juju - localhost | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| # Juju 2.8 doesn't support IPv6 | |
| lxc network set lxdbr0 ipv6.address none | |
| juju bootstrap localhost c | |
| juju add-model m | |
| juju status | |
| juju version | |
| - name: Bootstrap Juju - microk8s | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| env: | |
| JUJU_DB_TAG: ${{ steps.vars.outputs.juju-db-version }} | |
| # TODO: Enabling developer-mode is a bit of a hack to get this working for now. | |
| # Ideally, we would mock our own simplestream, similar to Jenkins, to select | |
| # and filter with as standard, instead of skipping over them with this flag | |
| run: | | |
| set -euxo pipefail | |
| sg microk8s <<EOF | |
| microk8s.kubectl apply -f "$GITHUB_WORKSPACE/.github/microk8s-juju-2.8-fix.yaml" | |
| juju bootstrap microk8s c \ | |
| --config caas-image-repo="${DOCKER_REGISTRY}/test-repo" \ | |
| --config features="[developer-mode]" | |
| EOF | |
| juju add-model m | |
| juju status | |
| juju version | |
| # The `wait-for` plugin is used after deploying an application | |
| # This was added in Juju 2.9, so it's not installed by the 2.8 snap | |
| # However we just need to install the `wait-for` binary ourselves, | |
| # and then Juju 2.8 can use it (amazing!) | |
| - name: Add `wait-for` plugin | |
| shell: bash | |
| run: | | |
| go install github.com/juju/juju/cmd/plugins/juju-wait-for | |
| - name: Deploy some applications | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| # For prometheus-k8s, we need to use revision 1 - this is the only | |
| # revision with metadata v1, hence can be deployed on 2.8. | |
| # 2.8 which doesn't have the --revision flag, so append the rev no. | |
| DEPLOY_NAME=${CHARM_${{ matrix.model_type }}} | |
| if [[ $DEPLOY_NAME == 'prometheus-k8s' ]]; then | |
| DEPLOY_NAME='prometheus-k8s-1' | |
| fi | |
| juju deploy $DEPLOY_NAME | |
| juju wait-for application ${CHARM_${{ matrix.model_type }}} | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 | |
| - name: Install local juju | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| which juju | |
| sudo snap remove juju --purge | |
| make install | |
| which juju | |
| - name: Build jujud image | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| make operator-image | |
| # Shim in recognition for our CA to jujud-operator | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM jujusolutions/jujud-operator:${UPSTREAM_JUJU_TAG} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${UPSTREAM_JUJU_TAG} | |
| BUILD_TEMP=$(mktemp -d) | |
| cp ~/certs/ca.crt $BUILD_TEMP/ | |
| cat >$BUILD_TEMP/Dockerfile <<EOL | |
| FROM jujusolutions/jujud-operator:${CURRENT_STABLE_JUJU_TAG} | |
| COPY ca.crt /usr/local/share/ca-certificates/ca.crt | |
| RUN update-ca-certificates | |
| EOL | |
| docker build $BUILD_TEMP -t ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG} | |
| docker push ${DOCKER_REGISTRY}/test-repo/jujud-operator:${CURRENT_STABLE_JUJU_TAG} | |
| - name: Preflight | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| juju status | |
| juju version | |
| - name: Test upgrade controller - localhost | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost' | |
| shell: bash | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| # Upgrade to the latest stable. | |
| juju upgrade-controller --debug | |
| $GITHUB_WORKSPACE/.github/verify-agent-version.sh iaas $CURRENT_STABLE_JUJU_TAG | |
| juju upgrade-controller --build-agent --debug | |
| $GITHUB_WORKSPACE/.github/verify-agent-version.sh iaas "${UPSTREAM_JUJU_TAG}.1" | |
| PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail -m controller | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 | |
| - name: Test upgrade controller - microk8s | |
| if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' | |
| shell: bash | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| CURRENT_STABLE_JUJU_TAG: ${{ steps.vars.outputs.current-stable-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| # Upgrade to the latest stable. | |
| juju upgrade-controller --debug | |
| $GITHUB_WORKSPACE/.github/verify-agent-version.sh caas $CURRENT_STABLE_JUJU_TAG | |
| # Upgrade to local built version. | |
| juju upgrade-controller --agent-stream=develop --debug | |
| $GITHUB_WORKSPACE/.github/verify-agent-version.sh caas $UPSTREAM_JUJU_TAG | |
| PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail -m controller | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 | |
| - name: Test upgrade model | |
| if: env.RUN_TEST == 'RUN' | |
| shell: bash | |
| env: | |
| UPSTREAM_JUJU_TAG: ${{ steps.vars.outputs.upstream-juju-version }} | |
| run: | | |
| set -euxo pipefail | |
| while true; do | |
| juju upgrade-model 2>&1 | tee output.log || true | |
| RES=$(cat output.log | grep "upgrade in progress" || echo "NOT-UPGRADING") | |
| if [ "$RES" = "NOT-UPGRADING" ]; then | |
| break | |
| fi | |
| done | |
| attempt=0 | |
| while true; do | |
| UPDATED=$((juju show-model m --format=json || echo "") | jq -r '.m."agent-version"') | |
| if [[ $UPDATED == $UPSTREAM_JUJU_TAG* ]]; then | |
| break | |
| fi | |
| sleep 10 | |
| attempt=$((attempt+1)) | |
| if [ "$attempt" -eq 48 ]; then | |
| echo "Upgrade model timed out" | |
| exit 1 | |
| fi | |
| done | |
| PANIC=$(juju debug-log --replay --no-tail | grep "panic" || true) | |
| if [ "$PANIC" != "" ]; then | |
| echo "Panic found:" | |
| juju debug-log --replay --no-tail | |
| exit 1 | |
| fi | |
| $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 | |
| - name: Wrap up | |
| if: env.RUN_TEST == 'RUN' | |
| run: | | |
| set -euxo pipefail | |
| juju version | |
| juju status | |
| sg microk8s "microk8s kubectl get all -A" || true | |
| lxc ls || true |