Merge pull request juju#16409 from hpidcock/merge-3.1-3.2-20231010

juju#16409

Forward ports:
- juju#16380
- juju#16390
- juju#16365
- juju#16374
- juju#16405
- juju#16406
- juju#16408

Conflicts:
- .github/workflows/static-analysis.yml
- Makefile
- apiserver/facades/controller/caasmodelconfigmanager/facade.go
- cmd/containeragent/initialize/package_test.go
- controller/config.go
- controller/config_test.go
- provider/equinix/environ.go

.github/workflows/static-analysis.yml

@@ -55,7 +55,7 @@ jobs:
           libdqlite-dev \
           libsqlite3-dev \
           sqlite3
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.3
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.54.2
         sudo curl -sSfL https://github.com/mvdan/sh/releases/download/v3.7.0/shfmt_v3.7.0_linux_$(go env GOARCH) -o /usr/bin/shfmt
         sudo chmod +x /usr/bin/shfmt
     

Makefile

@@ -508,8 +508,9 @@ ifeq ($(shell if [ "$(GO_INSTALLED_VERSION)" \< "$(GO_MOD_VERSION)" ]; then echo
 endif
 endif
 
-WAIT_FOR_DPKG=sh -c '. "${PROJECT_DIR}/make_functions.sh"; wait_for_dpkg "$$@"' wait_for_dpkg
-JUJU_DB_CHANNEL=4.4/stable
+WAIT_FOR_DPKG=bash -c '. "${PROJECT_DIR}/make_functions.sh"; wait_for_dpkg "$$@"' wait_for_dpkg
+JUJU_DB_VERSION=4.4
+JUJU_DB_CHANNEL=${JUJU_DB_VERSION}/stable
 
 .PHONY: install-mongo-dependencies
 install-mongo-dependencies:
@@ -552,7 +553,8 @@ check-deps:
 
 
 # CAAS related targets
-DOCKER_USERNAME            ?= jujusolutions
+export OCI_BUILDER         ?= $(shell (which podman 2>&1 > /dev/null && echo podman) || echo docker )
+DOCKER_USERNAME            ?= docker.io/jujusolutions
 DOCKER_BUILDX_CONTEXT      ?= juju-make
 DOCKER_STAGING_DIR         ?= ${BUILD_DIR}/docker-staging
 JUJUD_STAGING_DIR          ?= ${DOCKER_STAGING_DIR}/jujud-operator
@@ -565,6 +567,7 @@ BUILD_OPERATOR_IMAGE=bash -c '. "${PROJECT_DIR}/make_functions.sh"; build_push_o
 OPERATOR_IMAGE_PATH=bash -c '. "${PROJECT_DIR}/make_functions.sh"; operator_image_path "$$@"' operator_image_path
 OPERATOR_IMAGE_RELEASE_PATH=bash -c '. "${PROJECT_DIR}/make_functions.sh"; operator_image_release_path "$$@"' operator_image_release_path
 UPDATE_MICROK8S_OPERATOR=bash -c '. "${PROJECT_DIR}/make_functions.sh"; microk8s_operator_update "$$@"' microk8s_operator_update
+SEED_REPOSITORY=bash -c '. "${PROJECT_DIR}/make_functions.sh"; seed_repository "$$@"' seed_repository
 
 image_check_prereq=image-check-build
 ifneq ($(OPERATOR_IMAGE_BUILD_SRC),true)
@@ -585,7 +588,9 @@ image-check-build-skip:
 .PHONY: docker-builder
 docker-builder:
 ## docker-builder: Makes sure that there is a buildx context for building the oci images
+ifeq ($(OCI_BUILDER),docker)
 	-@docker buildx create --name ${DOCKER_BUILDX_CONTEXT}
+endif
 
 .PHONY: image-check
 operator-image: image-check docker-builder
@@ -609,12 +614,16 @@ push-operator-image-undefined:
 push-operator-image: $(push_operator_image_prereq)
 ## push-operator-image: Push up the newly built operator image via docker
 
-
 .PHONY: push-release-operator-image
 push-release-operator-image: PUSH_IMAGE=true
 push-release-operator-image: operator-image
 ## push-release-operator-image: Push up the newly built release operator image via docker
 
+.PHONY: seed-repository
+seed-repository:
+## seed-repository: Copy required juju images from docker.io/jujusolutions
+	JUJU_DB_VERSION=$(JUJU_DB_VERSION) $(SEED_REPOSITORY)
+
 
 .PHONY: host-install
 host-install:
@@ -623,18 +632,19 @@ host-install:
 
 .PHONY: minikube-operator-update
 minikube-operator-update: host-install operator-image
-## minikube-operator-update: Push up the newly built operator image for use with minikube
-	docker save "$(shell ${OPERATOR_IMAGE_PATH})" | minikube image load --overwrite=true -
+## minikube-operator-update: Inject the newly built operator image into minikube
+	$(OCI_BUILDER) save "$(shell ${OPERATOR_IMAGE_PATH})" | minikube image load --overwrite=true -
 
 .PHONY: microk8s-operator-update
 microk8s-operator-update: host-install operator-image
-## microk8s-operator-update: Push up the newly built operator image for use with microk8s
+## microk8s-operator-update: Inject the newly built operator image into microk8s
 	@${UPDATE_MICROK8S_OPERATOR}
 
 .PHONY: k3s-operator-update
 k3s-operator-update: host-install operator-image
-## k3s-operator-update: Push up the newly built operator image for use with k3s
-	docker save "$(shell ${OPERATOR_IMAGE_PATH})" | sudo k3s ctr images import -
+## k3s-operator-update: Inject the newly built operator image into k3s
+	$(OCI_BUILDER) save "$(shell ${OPERATOR_IMAGE_PATH})" | sudo k3s ctr images import -
+
 
 .PHONY: check-k8s-model
 check-k8s-model:
@@ -646,7 +656,7 @@ check-k8s-model:
 local-operator-update: check-k8s-model operator-image
 ## local-operator-update: Build then update local operator image
 	$(eval kubeworkers != juju status -m ${JUJU_K8S_MODEL} kubernetes-worker --format json | jq -c '.machines | keys' | tr  -c '[:digit:]' ' ' 2>&1)
-	docker save "$(shell ${OPERATOR_IMAGE_PATH})" | gzip > ${DOCKER_STAGING_DIR}/jujud-operator-image.tar.gz
+	$(OCI_BUILDER) save "$(shell ${OPERATOR_IMAGE_PATH})" | gzip > ${DOCKER_STAGING_DIR}/jujud-operator-image.tar.gz
 	$(foreach wm,$(kubeworkers), juju scp -m ${JUJU_K8S_MODEL} ${DOCKER_STAGING_DIR}/jujud-operator-image.tar.gz $(wm):/tmp/jujud-operator-image.tar.gz ; )
 	$(foreach wm,$(kubeworkers), juju ssh -m ${JUJU_K8S_MODEL} $(wm) -- "zcat /tmp/jujud-operator-image.tar.gz | docker load" ; )
 

api/controller/caasmodelconfigmanager/client.go

@@ -8,6 +8,9 @@ import (
 
 	"github.com/juju/juju/api/base"
 	"github.com/juju/juju/api/common"
+	apiwatcher "github.com/juju/juju/api/watcher"
+	"github.com/juju/juju/core/watcher"
+	"github.com/juju/juju/rpc/params"
 )
 
 // Client allows access to the CAAS model config manager API endpoint.
@@ -28,3 +31,15 @@ func NewClient(caller base.APICaller) (*Client, error) {
 		ControllerConfigAPI: common.NewControllerConfig(facadeCaller),
 	}, nil
 }
+
+// WatchControllerConfig provides a watcher for changes on controller config.
+func (c *Client) WatchControllerConfig() (watcher.NotifyWatcher, error) {
+	var result params.NotifyWatchResult
+	if err := c.facade.FacadeCall("WatchControllerConfig", nil, &result); err != nil {
+		return nil, err
+	}
+	if result.Error != nil {
+		return nil, result.Error
+	}
+	return apiwatcher.NewNotifyWatcher(c.facade.RawAPICaller(), result), nil
+}

apiserver/facades/client/client/client.go

@@ -15,6 +15,7 @@ import (
 	apiservererrors "github.com/juju/juju/apiserver/errors"
 	"github.com/juju/juju/apiserver/facade"
 	"github.com/juju/juju/cloudconfig/podcfg"
+	"github.com/juju/juju/controller"
 	"github.com/juju/juju/core/cache"
 	"github.com/juju/juju/core/leadership"
 	"github.com/juju/juju/core/multiwatcher"
@@ -292,13 +293,15 @@ func (c *Client) toolVersionsForCAAS(args params.FindToolsParams, streamsVersion
 	if err != nil {
 		return result, errors.Trace(err)
 	}
-	imageRepoDetails := controllerCfg.CAASImageRepo()
+	imageRepoDetails, err := docker.NewImageRepoDetails(controllerCfg.CAASImageRepo())
+	if err != nil {
+		return result, errors.Annotatef(err, "parsing %s", controller.CAASImageRepo)
+	}
 	if imageRepoDetails.Empty() {
-		repoDetails, err := docker.NewImageRepoDetails(podcfg.JujudOCINamespace)
+		imageRepoDetails, err = docker.NewImageRepoDetails(podcfg.JujudOCINamespace)
 		if err != nil {
 			return result, errors.Trace(err)
 		}
-		imageRepoDetails = *repoDetails
 	}
 	reg, err := c.registryAPIFunc(imageRepoDetails)
 	if err != nil {

apiserver/facades/client/controller/controller.go

@@ -32,6 +32,7 @@ import (
 	"github.com/juju/juju/core/model"
 	"github.com/juju/juju/core/multiwatcher"
 	"github.com/juju/juju/core/permission"
+	"github.com/juju/juju/docker"
 	"github.com/juju/juju/environs/bootstrap"
 	"github.com/juju/juju/migration"
 	"github.com/juju/juju/pubsub/controller"
@@ -728,6 +729,50 @@ func (c *ControllerAPI) ConfigSet(args params.ControllerConfigSet) error {
 	if err := c.checkIsSuperUser(); err != nil {
 		return errors.Trace(err)
 	}
+
+	currentCfg, err := c.state.ControllerConfig()
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	// TODO(dqlite): move this business logic out of the facade.
+	if newValue, ok := args.Config[corecontroller.CAASImageRepo]; ok {
+		var newCAASImageRepo docker.ImageRepoDetails
+		if v, ok := newValue.(string); ok {
+			newCAASImageRepo, err = docker.NewImageRepoDetails(v)
+			if err != nil {
+				return fmt.Errorf("cannot parse %s: %s%w", corecontroller.CAASImageRepo, err.Error(),
+					errors.Hide(errors.NotValid))
+			}
+		} else {
+			return fmt.Errorf("%s expected a string got %v%w", corecontroller.CAASImageRepo, v,
+				errors.Hide(errors.NotValid))
+		}
+
+		var currentCAASImageRepo docker.ImageRepoDetails
+		if currentValue, ok := currentCfg[corecontroller.CAASImageRepo]; !ok {
+			return fmt.Errorf("cannot change %s as it is not currently set%w", corecontroller.CAASImageRepo,
+				errors.Hide(errors.NotValid))
+		} else if v, ok := currentValue.(string); !ok {
+			return fmt.Errorf("existing %s expected a string", corecontroller.CAASImageRepo)
+		} else {
+			currentCAASImageRepo, err = docker.NewImageRepoDetails(v)
+			if err != nil {
+				return fmt.Errorf("cannot parse existing %s: %w", corecontroller.CAASImageRepo, err)
+			}
+		}
+		// TODO: when podspec is removed, implement changing caas-image-repo.
+		if newCAASImageRepo.Repository != currentCAASImageRepo.Repository {
+			return fmt.Errorf("cannot change %s: repository read-only, only authentication can be updated", corecontroller.CAASImageRepo)
+		}
+		if !newCAASImageRepo.IsPrivate() && currentCAASImageRepo.IsPrivate() {
+			return fmt.Errorf("cannot change %s: unable to remove authentication details", corecontroller.CAASImageRepo)
+		}
+		if newCAASImageRepo.IsPrivate() && !currentCAASImageRepo.IsPrivate() {
+			return fmt.Errorf("cannot change %s: unable to add authentication details", corecontroller.CAASImageRepo)
+		}
+	}
+
 	if err := c.state.UpdateControllerConfig(args.Config, nil); err != nil {
 		return errors.Trace(err)
 	}

apiserver/facades/client/controller/controller_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/juju/juju/core/cache"
 	coremultiwatcher "github.com/juju/juju/core/multiwatcher"
 	"github.com/juju/juju/core/permission"
+	"github.com/juju/juju/docker"
 	"github.com/juju/juju/environs"
 	environscloudspec "github.com/juju/juju/environs/cloudspec"
 	"github.com/juju/juju/environs/config"
@@ -1020,6 +1021,54 @@ func (s *controllerSuite) TestConfigSetPublishesEvent(c *gc.C) {
 	c.Assert(config.Features().SortedValues(), jc.DeepEquals, []string{"bar", "foo"})
 }
 
+func (s *controllerSuite) TestConfigSetCAASImageRepo(c *gc.C) {
+	config, err := s.State.ControllerConfig()
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(config.CAASImageRepo(), gc.Equals, "")
+
+	err = s.controller.ConfigSet(params.ControllerConfigSet{Config: map[string]interface{}{
+		"caas-image-repo": "juju-repo.local",
+	}})
+	c.Assert(err, gc.ErrorMatches, `cannot change caas-image-repo as it is not currently set`)
+
+	err = s.State.UpdateControllerConfig(map[string]interface{}{
+		"caas-image-repo": "jujusolutions",
+	}, nil)
+	c.Assert(err, jc.ErrorIsNil)
+
+	err = s.controller.ConfigSet(params.ControllerConfigSet{Config: map[string]interface{}{
+		"caas-image-repo": "juju-repo.local",
+	}})
+	c.Assert(err, gc.ErrorMatches, `cannot change caas-image-repo: repository read-only, only authentication can be updated`)
+
+	err = s.controller.ConfigSet(params.ControllerConfigSet{Config: map[string]interface{}{
+		"caas-image-repo": `{"repository":"jujusolutions","username":"foo","password":"bar"}`,
+	}})
+	c.Assert(err, gc.ErrorMatches, `cannot change caas-image-repo: unable to add authentication details`)
+
+	err = s.State.UpdateControllerConfig(map[string]interface{}{
+		"caas-image-repo": `{"repository":"jujusolutions","username":"bar","password":"foo"}`,
+	}, nil)
+	c.Assert(err, jc.ErrorIsNil)
+
+	err = s.controller.ConfigSet(params.ControllerConfigSet{Config: map[string]interface{}{
+		"caas-image-repo": `{"repository":"jujusolutions","username":"foo","password":"bar"}`,
+	}})
+	c.Assert(err, jc.ErrorIsNil)
+
+	config, err = s.State.ControllerConfig()
+	c.Assert(err, jc.ErrorIsNil)
+	repoDetails, err := docker.NewImageRepoDetails(config.CAASImageRepo())
+	c.Assert(err, jc.ErrorIsNil)
+	c.Assert(repoDetails, gc.DeepEquals, docker.ImageRepoDetails{
+		Repository: "jujusolutions",
+		BasicAuthConfig: docker.BasicAuthConfig{
+			Username: "foo",
+			Password: "bar",
+		},
+	})
+}
+
 func (s *controllerSuite) TestMongoVersion(c *gc.C) {
 	result, err := s.controller.MongoVersion()
 	c.Assert(err, jc.ErrorIsNil)

apiserver/facades/client/modelupgrader/findagents.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/juju/juju/apiserver/common"
 	"github.com/juju/juju/cloudconfig/podcfg"
+	"github.com/juju/juju/controller"
 	coreos "github.com/juju/juju/core/os"
 	"github.com/juju/juju/docker"
 	envtools "github.com/juju/juju/environs/tools"
@@ -95,13 +96,15 @@ func (m *ModelUpgraderAPI) agentVersionsForCAAS(
 	streamsAgents coretools.List,
 ) (coretools.Versions, error) {
 	result := coretools.Versions{}
-	imageRepoDetails := args.ControllerCfg.CAASImageRepo()
+	imageRepoDetails, err := docker.NewImageRepoDetails(args.ControllerCfg.CAASImageRepo())
+	if err != nil {
+		return nil, errors.Annotatef(err, "parsing %s", controller.CAASImageRepo)
+	}
 	if imageRepoDetails.Empty() {
-		repoDetails, err := docker.NewImageRepoDetails(podcfg.JujudOCINamespace)
+		imageRepoDetails, err = docker.NewImageRepoDetails(podcfg.JujudOCINamespace)
 		if err != nil {
 			return nil, errors.Trace(err)
 		}
-		imageRepoDetails = *repoDetails
 	}
 	reg, err := m.registryAPIFunc(imageRepoDetails)
 	if err != nil {

apiserver/facades/controller/caasapplicationprovisioner/provisioner.go

@@ -31,6 +31,7 @@ import (
 	"github.com/juju/juju/core/network"
 	"github.com/juju/juju/core/resources"
 	"github.com/juju/juju/core/status"
+	"github.com/juju/juju/docker"
 	"github.com/juju/juju/environs/bootstrap"
 	"github.com/juju/juju/environs/config"
 	"github.com/juju/juju/environs/tags"
@@ -353,6 +354,10 @@ func (a *API) provisioningInfo(appName names.ApplicationTag) (*params.CAASApplic
 		return nil, errors.Annotatef(err, "getting application config")
 	}
 	base := app.Base()
+	imageRepoDetails, err := docker.NewImageRepoDetails(cfg.CAASImageRepo())
+	if err != nil {
+		return nil, errors.Annotatef(err, "parsing %s", controller.CAASImageRepo)
+	}
 	return &params.CAASApplicationProvisioningInfo{
 		Version:              vers,
 		APIAddresses:         addrs,
@@ -362,7 +367,7 @@ func (a *API) provisioningInfo(appName names.ApplicationTag) (*params.CAASApplic
 		Devices:              devices,
 		Constraints:          mergedCons,
 		Base:                 params.Base{Name: base.OS, Channel: base.Channel},
-		ImageRepo:            params.NewDockerImageInfo(cfg.CAASImageRepo(), imagePath),
+		ImageRepo:            params.NewDockerImageInfo(imageRepoDetails, imagePath),
 		CharmModifiedVersion: app.CharmModifiedVersion(),
 		CharmURL:             *charmURL,
 		Trust:                appConfig.GetBool(application.TrustConfigOptionName, false),

apiserver/facades/controller/caasmodelconfigmanager/facade.go

@@ -7,14 +7,37 @@ import (
 	"github.com/juju/juju/apiserver/common"
 	"github.com/juju/juju/apiserver/facade"
 	"github.com/juju/juju/rpc/params"
+	"github.com/juju/juju/state"
+	"github.com/juju/juju/state/watcher"
 )
 
+//go:generate go run go.uber.org/mock/mockgen -package mocks -destination mocks/context_mock.go github.com/juju/juju/apiserver/facade Authorizer,Context,Resources
+
+// State provides required state for the Facade.
+type State interface {
+	WatchControllerConfig() state.NotifyWatcher
+}
+
 // Facade allows model config manager clients to watch controller config changes and fetch controller config.
 type Facade struct {
-	auth                facade.Authorizer
+	auth      facade.Authorizer
+	resources facade.Resources
+
+	ctrlState           State
 	controllerConfigAPI *common.ControllerConfigAPI
 }
 
 func (f *Facade) ControllerConfig() (params.ControllerConfigResult, error) {
 	return f.controllerConfigAPI.ControllerConfig()
 }
+
+func (f *Facade) WatchControllerConfig() (params.NotifyWatchResult, error) {
+	result := params.NotifyWatchResult{}
+	w := f.ctrlState.WatchControllerConfig()
+	if _, ok := <-w.Changes(); ok {
+		result.NotifyWatcherId = f.resources.Register(w)
+	} else {
+		return result, watcher.EnsureErr(w)
+	}
+	return result, nil
+}

apiserver/facades/controller/caasmodelconfigmanager/register.go

@@ -33,6 +33,8 @@ func newFacade(ctx facade.Context) (*Facade, error) {
 	}
 	return &Facade{
 		auth:                authorizer,
+		resources:           ctx.Resources(),
 		controllerConfigAPI: common.NewStateControllerConfig(systemState),
+		ctrlState:           systemState,
 	}, nil
 }