From b8ac67647dde5b2ae33749b04cb66ca47f9dcb13 Mon Sep 17 00:00:00 2001 From: Harry Pidcock Date: Fri, 15 Sep 2023 09:31:33 +1000 Subject: [PATCH 1/3] Use docker cache/mirror on aws runners. --- .github/microk8s-juju-2.8-fix.yaml | 52 +++++++++++++++++++++++++ .github/microk8s-launch-config-aws.yaml | 12 ++++++ .github/workflows/microk8s-tests.yml | 10 ++++- .github/workflows/smoke.yml | 10 ++++- .github/workflows/upgrade.yml | 12 +++++- 5 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 .github/microk8s-juju-2.8-fix.yaml create mode 100644 .github/microk8s-launch-config-aws.yaml diff --git a/.github/microk8s-juju-2.8-fix.yaml b/.github/microk8s-juju-2.8-fix.yaml new file mode 100644 index 00000000000..97550fd0dac --- /dev/null +++ b/.github/microk8s-juju-2.8-fix.yaml @@ -0,0 +1,52 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + namespace: kube-system + name: juju-credential-microk8s + labels: + juju-credential: microk8s +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +- nonResourceURLs: ["*"] + verbs: ["*"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: juju-credential-microk8s + labels: + juju-credential: microk8s +secrets: +- kind: Secret + name: juju-credential-microk8s + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + namespace: kube-system + name: juju-credential-microk8s + labels: + juju-credential: microk8s +subjects: +- kind: ServiceAccount + name: juju-credential-microk8s + namespace: kube-system +roleRef: + kind: ClusterRole + name: juju-credential-microk8s + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Secret +metadata: + namespace: kube-system + name: juju-credential-microk8s + labels: + juju-credential: microk8s + annotations: + kubernetes.io/service-account.name: juju-credential-microk8s +type: kubernetes.io/service-account-token diff --git a/.github/microk8s-launch-config-aws.yaml b/.github/microk8s-launch-config-aws.yaml new file mode 100644 index 00000000000..4b55d29ddf2 --- /dev/null +++ b/.github/microk8s-launch-config-aws.yaml @@ -0,0 +1,12 @@ +--- +version: 0.1.0 +extraKubeletArgs: + --cluster-domain: cluster.local + --cluster-dns: 10.152.183.10 +addons: + - name: dns +containerdRegistryConfigs: + docker.io: | + [host."http://10.0.1.123:80"] + capabilities = ["pull", "resolve"] + skip_verify = true diff --git a/.github/workflows/microk8s-tests.yml b/.github/workflows/microk8s-tests.yml index 6e6588e7062..b1a636b75cc 100644 --- a/.github/workflows/microk8s-tests.yml +++ b/.github/workflows/microk8s-tests.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - microk8s: [1.21/stable] + microk8s: [1.28/stable] steps: - name: Checking out repo @@ -37,11 +37,19 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Setup Docker Mirror + shell: bash + run: | + (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["http://10.0.1.123:80"]' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + docker system info + - uses: balchua/microk8s-actions@1e8e626239c2befe7cd5d258c96ae152a7259c74 with: channel: '${{ matrix.microk8s }}' # enable now to give microk8s more time to settle down. addons: '["dns", "storage", "dashboard", "ingress", "metallb:10.64.140.43-10.64.140.49"]' + launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml" - name: Install Dependencies run: | diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 238c5042c02..d5034d06ee6 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -42,12 +42,20 @@ jobs: if: matrix.cloud == 'localhost' uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd + - name: Setup Docker Mirror + shell: bash + run: | + (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["http://10.0.1.123:80"]' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + docker system info + - name: Setup MicroK8s if: matrix.cloud == 'microk8s' uses: balchua/microk8s-actions@1e8e626239c2befe7cd5d258c96ae152a7259c74 with: - channel: "1.25/stable" + channel: "1.28/stable" addons: '["dns", "hostpath-storage", "rbac"]' + launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml" - name: Set up Go uses: actions/setup-go@v3 diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml index 70cf52819a6..6072df8a127 100644 --- a/.github/workflows/upgrade.yml +++ b/.github/workflows/upgrade.yml @@ -78,12 +78,21 @@ jobs: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + - name: Setup Docker Mirror + if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' + shell: bash + run: | + (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["http://10.0.1.123:80"]' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + docker system info + - name: Setup k8s if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' uses: balchua/microk8s-actions@1e8e626239c2befe7cd5d258c96ae152a7259c74 with: - channel: "1.23/stable" + channel: "1.28/stable" addons: '["dns", "storage"]' + launch-configuration: "$GITHUB_WORKSPACE/.github/microk8s-launch-config-aws.yaml" - name: Setup local caas registry if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' @@ -174,6 +183,7 @@ jobs: set -euxo pipefail sg microk8s < Date: Fri, 15 Sep 2023 17:11:35 +1000 Subject: [PATCH 2/3] Fix microk8s upgrade job. --- .github/microk8s-launch-config-aws.yaml | 5 ++ .github/workflows/upgrade.yml | 62 ++++++++----------------- 2 files changed, 24 insertions(+), 43 deletions(-) diff --git a/.github/microk8s-launch-config-aws.yaml b/.github/microk8s-launch-config-aws.yaml index 4b55d29ddf2..98b91e8c8af 100644 --- a/.github/microk8s-launch-config-aws.yaml +++ b/.github/microk8s-launch-config-aws.yaml @@ -10,3 +10,8 @@ containerdRegistryConfigs: [host."http://10.0.1.123:80"] capabilities = ["pull", "resolve"] skip_verify = true + 10.152.183.69: | + [host."https://10.152.183.69:443"] + capabilities = ["pull", "resolve", "push"] + skip_verify = true + diff --git a/.github/workflows/upgrade.yml b/.github/workflows/upgrade.yml index 6072df8a127..0c897346a86 100644 --- a/.github/workflows/upgrade.yml +++ b/.github/workflows/upgrade.yml @@ -37,7 +37,6 @@ jobs: run: | set -euxo pipefail sudo snap install snapcraft --classic - sudo snap install yq sudo snap install juju --classic --channel=${{ matrix.snap_version }} echo "/snap/bin" >> $GITHUB_PATH @@ -45,8 +44,8 @@ jobs: if: env.RUN_TEST == 'RUN' uses: actions/checkout@v3 - # We use LXD in the microk8s test too, to build the snap. - name: Setup LXD + if: env.RUN_TEST == 'RUN' && matrix.model_type == 'localhost' uses: canonical/setup-lxd@90d76101915da56a42a562ba766b1a77019242fd with: channel: 4.0/candidate @@ -83,6 +82,7 @@ jobs: shell: bash run: | (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json '.registry-mirrors += ["http://10.0.1.123:80"]' | sudo tee /etc/docker/daemon.json + (cat /etc/docker/daemon.json 2> /dev/null || echo "{}") | yq -o json ".insecure-registries += [\"${DOCKER_REGISTRY}\"]" | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker system info @@ -109,23 +109,15 @@ jobs: # Recognise CA sudo cp ~/certs/ca.crt /usr/local/share/ca-certificates sudo update-ca-certificates - - sudo mkdir -p /etc/docker/certs.d/${DOCKER_REGISTRY}/ - sudo cp ~/certs/ca.crt /etc/docker/certs.d/${DOCKER_REGISTRY}/ - + # Generate certs openssl req -nodes -newkey rsa:2048 -keyout ~/certs/registry.key -out ~/certs/registry.csr -subj "/CN=registry" openssl x509 -req -in ~/certs/registry.csr -CA ~/certs/ca.crt -CAkey ~/certs/ca.key \ - -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile .github/registry.ext + -out ~/certs/registry.crt -CAcreateserial -days 365 -sha256 -extfile $GITHUB_WORKSPACE/.github/registry.ext # Deploy registry - cat .github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg microk8s "microk8s kubectl create -f -" - - # TODO:(jack-w-shaw) Figure out why we need this and do something nicer - sudo microk8s refresh-certs --cert ca.crt - sudo microk8s refresh-certs --cert server.crt - sg microk8s "microk8s status --wait-ready" + cat $GITHUB_WORKSPACE/.github/reg.yml | CERT_DIR=$HOME/certs envsubst | sg microk8s "microk8s kubectl create -f -" # Wait for registry sg microk8s "microk8s kubectl wait --for condition=available deployment registry -n container-registry --timeout 180s" || true @@ -200,12 +192,7 @@ jobs: - name: Add `wait-for` plugin shell: bash run: | - # Download a stable version of Juju - curl -L -O https://github.com/juju/juju/archive/refs/tags/juju-2.9.29.tar.gz - tar -xf juju-2.9.29.tar.gz - cd juju-juju-2.9.29/ go install github.com/juju/juju/cmd/plugins/juju-wait-for - cd .. - name: Deploy some applications if: env.RUN_TEST == 'RUN' @@ -225,21 +212,17 @@ jobs: juju wait-for application ${CHARM_${{ matrix.model_type }}} - .github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 - - - name: Build snap - if: env.RUN_TEST == 'RUN' - shell: bash - run: | - set -euxo pipefail - snapcraft --use-lxd + $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 - - name: Install snap + - name: Install local juju if: env.RUN_TEST == 'RUN' shell: bash run: | set -euxo pipefail - sudo snap install juju*.snap --dangerous --classic + which juju + sudo snap remove juju --purge + make install + which juju - name: Build jujud image if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' @@ -295,17 +278,10 @@ jobs: # Upgrade to the latest stable. juju upgrade-controller --debug - .github/verify-agent-version.sh $CURRENT_STABLE_JUJU_TAG - - make go-install - $GOPATH/bin/juju upgrade-controller --build-agent --debug - .github/verify-agent-version.sh "${UPSTREAM_JUJU_TAG}.1" - rm -rf $GOPATH/bin/juju* + $GITHUB_WORKSPACE/.github/verify-agent-version.sh $CURRENT_STABLE_JUJU_TAG - # Upgrade to local built snap version - upload snap jujud. - snap_version=$(juju version | cut -d '-' -f 1); - juju upgrade-controller --agent-version $snap_version - .github/verify-agent-version.sh "${UPSTREAM_JUJU_TAG}.2" + juju upgrade-controller --build-agent --debug + $GITHUB_WORKSPACE/.github/verify-agent-version.sh "${UPSTREAM_JUJU_TAG}.1" PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true) if [ "$PANIC" != "" ]; then @@ -314,7 +290,7 @@ jobs: exit 1 fi - .github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 + $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 - name: Test upgrade controller - microk8s if: env.RUN_TEST == 'RUN' && matrix.model_type == 'microk8s' @@ -327,11 +303,11 @@ jobs: # Upgrade to the latest stable. juju upgrade-controller --debug - .github/verify-agent-version.sh $CURRENT_STABLE_JUJU_TAG + $GITHUB_WORKSPACE/.github/verify-agent-version.sh $CURRENT_STABLE_JUJU_TAG # Upgrade to local built version. juju upgrade-controller --agent-stream=develop --debug - .github/verify-agent-version.sh $UPSTREAM_JUJU_TAG + $GITHUB_WORKSPACE/.github/verify-agent-version.sh $UPSTREAM_JUJU_TAG PANIC=$(juju debug-log --replay --no-tail -m controller | grep "panic" || true) if [ "$PANIC" != "" ]; then @@ -340,7 +316,7 @@ jobs: exit 1 fi - .github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 + $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 - name: Test upgrade model if: env.RUN_TEST == 'RUN' @@ -379,7 +355,7 @@ jobs: exit 1 fi - .github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 + $GITHUB_WORKSPACE/.github/verify-${CHARM_${{ matrix.model_type }}}.sh 30 - name: Wrap up if: env.RUN_TEST == 'RUN' From c1782a8a80440bdde977720b55f7e8627b94d0d9 Mon Sep 17 00:00:00 2001 From: Harry Pidcock Date: Mon, 18 Sep 2023 08:26:53 +1000 Subject: [PATCH 3/3] Find relation via id not endpoint names in cleanup job. --- state/cleanup.go | 14 +++++++++++--- state/relation.go | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/state/cleanup.go b/state/cleanup.go index 074ba47b924..d453a6378db 100644 --- a/state/cleanup.go +++ b/state/cleanup.go @@ -5,6 +5,7 @@ package state import ( "fmt" + "strconv" "time" "github.com/juju/charm/v8" @@ -282,13 +283,20 @@ func (st *State) cleanupRelationSettings(prefix string) error { } func (st *State) cleanupForceDestroyedRelation(prefix string) (err error) { - relation, err := st.KeyRelation(prefix) + var relation *Relation + var relId int + if relId, err = strconv.Atoi(prefix); err == nil { + relation, err = st.Relation(relId) + } else if err != nil { + logger.Warningf("handling legacy cleanupForceDestroyedRelation with relation key %q", prefix) + relation, err = st.KeyRelation(prefix) + } if errors.IsNotFound(err) { return nil - } - if err != nil { + } else if err != nil { return errors.Annotatef(err, "getting relation %q", prefix) } + scopes, closer := st.db().GetCollection(relationScopesC) defer closer() diff --git a/state/relation.go b/state/relation.go index 39e4c22772b..c09ff751965 100644 --- a/state/relation.go +++ b/state/relation.go @@ -6,6 +6,7 @@ package state import ( "fmt" "sort" + "strconv" "strings" "time" @@ -484,7 +485,7 @@ func (r *Relation) destroyOps(ignoreApplication string, op *ForcedOperation) (op // Since we are force destroying, life assert should be current relation's life. lifeAssert = bson.D{{"life", r.doc.Life}} deadline := r.st.stateClock.Now().Add(op.MaxWait) - ops = append(ops, newCleanupAtOp(deadline, cleanupForceDestroyedRelation, relationKey(r.Endpoints()))) + ops = append(ops, newCleanupAtOp(deadline, cleanupForceDestroyedRelation, strconv.Itoa(r.Id()))) } ops = append(ops, txn.Op{