Merge pull request juju#16588 from wallyworld/merge-3.4-20231116

juju#16588

Merge 3.4

Conflicts were due to adding a context to calls in main.

juju#16552 [from barrettj12/minimal](juju@0ec38c2)
juju#16551 [from barrettj12/bump-cmd](juju@6c4a15e)
juju#16558 [from barrettj12/bootstrap-doc](juju@189de77)
juju#16559 [from ycliuhw/fix-vault-setup](juju@737ad81)
juju#16520 [from jack-w-shaw/JUJU-4732_use_charm_name_i…](juju@f835120)
juju#16555 [from masnax/3.1](juju@bc48204)
juju#16566 [from barrettj12/chrev](juju@e2c0e11)
juju#16565 [from jack-w-shaw/write_caas_application_pro…](juju@27628d1)
juju#16569 [from SimonRichardson/remove-lxd-retry-logic](juju@c17d277)

```
# Conflicts:
# apiserver/facades/client/application/repository_mocks_test.go
# apiserver/facades/client/charms/client.go
# apiserver/facades/client/charms/client_test.go
# apiserver/facades/client/charms/mocks/repository.go
# cmd/jujud/agent/bootstrap_test.go
# cmd/jujud/agent/controllercharm.go
# core/charm/downloader/downloader.go
# core/charm/downloader/downloader_test.go
# core/charm/downloader/export_test.go
# core/charm/downloader/mocks/charm_mocks.go
# core/charm/repository.go
# core/charm/repository/charmhub.go
# core/charm/repository/charmhub_test.go
# scripts/win-installer/setup.iss
# snap/snapcraft.yaml
# version/version.go
```

cmd/juju/commands/bootstrap.go

@@ -75,7 +75,7 @@ Used without arguments, bootstrap will step you through the process of
 initializing a Juju cloud environment. Initialization consists of creating
 a 'controller' model and provisioning a machine to act as controller.
 
-We recommend you call your controller ‘username-region’ e.g. ‘fred-us-east-1’
+We recommend you call your controller ‘username-region’ e.g. ‘fred-us-east-1’.
 See --clouds for a list of clouds and credentials.
 See --regions <cloud> for a list of available regions for a given cloud.
 
@@ -103,23 +103,23 @@ used with the MAAS provider ('--to <host>.maas').
 You can change the default timeout and retry delays used during the
 bootstrap by changing the following settings in your configuration
 (all values represent number of seconds):
+
     # How long to wait for a connection to the controller
-    bootstrap-timeout: 1200 # default: 20 minutes
-    # How long to wait between connection attempts to a controller
-address.
-    bootstrap-retry-delay: 5 # default: 5 seconds
+    bootstrap-timeout: 1200  # default: 20 minutes
+    # How long to wait between connection attempts to a controller address.
+    bootstrap-retry-delay: 5  # default: 5 seconds
     # How often to refresh controller addresses from the API server.
-    bootstrap-addresses-delay: 10 # default: 10 seconds
+    bootstrap-addresses-delay: 10  # default: 10 seconds
 
 It is possible to override the base e.g. [email protected], Juju attempts 
 to bootstrap on to, by supplying a base argument to '--bootstrap-base'.
 
 An error is emitted if the determined base is not supported. Using the
 '--force' option to override this check:
 
-	juju bootstrap [email protected] --force
+    juju bootstrap [email protected] --force
 
-The '--bootstrap-series' can be still used, but is deprecated in favour
+The '--bootstrap-series' flag can be still used, but is deprecated in favour
 of '--bootstrap-base'.
 
 Private clouds may need to specify their own custom image metadata and
@@ -137,9 +137,9 @@ The agent version can be specified a simple numeric version, e.g. 2.2.4.
 
 For example, at the time when 2.3.0, 2.3.1 and 2.3.2 are released and your
 agent stream is 'released' (default), then a 2.3.1 client can bootstrap:
-    * 2.3.0 controller by running '... bootstrap --agent-version=2.3.0 ...';
-    * 2.3.1 controller by running '... bootstrap ...';
-    * 2.3.2 controller by running 'bootstrap --auto-upgrade'.
+   * 2.3.0 controller by running '... bootstrap --agent-version=2.3.0 ...';
+   * 2.3.1 controller by running '... bootstrap ...';
+   * 2.3.2 controller by running 'bootstrap --auto-upgrade'.
 However, if this client has a copy of codebase, then a local copy of Juju
 will be built and bootstrapped - 2.3.1.1.
 

environs/bootstrap/bootstrap.go

@@ -215,10 +215,16 @@ func withDefaultControllerConstraints(cons constraints.Value) constraints.Value
 		// A default of 3.5GiB will result in machines with up to 4GiB of memory, eg
 		// - 3.75GiB on AWS, Google
 		// - 3.5GiB on Azure
-		// - 4GiB on Rackspace etc
 		var mem uint64 = 3.5 * 1024
 		cons.Mem = &mem
 	}
+	// If we're bootstrapping a controller on a lxd virtual machine, we want to
+	// ensure that it has at least 2 cores. Less than 2 cores can cause the
+	// controller to become unresponsive when installing.
+	if !cons.HasCpuCores() && cons.HasVirtType() && *cons.VirtType == "virtual-machine" {
+		var cores = uint64(2)
+		cons.CpuCores = &cores
+	}
 	return cons
 }
 
@@ -442,7 +448,7 @@ func bootstrapIAAS(
 	// The follow is used to determine if we should apply the default
 	// constraints when we bootstrap. Generally speaking this should always be
 	// applied, but there are exceptions to the rule e.g. local LXD
-	if checker, ok := environ.(environs.DefaultConstraintsChecker); !ok || checker.ShouldApplyControllerConstraints() {
+	if checker, ok := environ.(environs.DefaultConstraintsChecker); !ok || checker.ShouldApplyControllerConstraints(bootstrapConstraints) {
 		bootstrapConstraints = withDefaultControllerConstraints(bootstrapConstraints)
 	}
 	bootstrapParams.BootstrapConstraints = bootstrapConstraints

environs/interface.go

@@ -627,7 +627,7 @@ type PrecheckJujuUpgradeStep interface {
 type DefaultConstraintsChecker interface {
 	// ShouldApplyControllerConstraints returns if bootstrapping logic should
 	// use default constraints
-	ShouldApplyControllerConstraints() bool
+	ShouldApplyControllerConstraints(constraints.Value) bool
 }
 
 // HardwareCharacteristicsDetector is implemented by environments that can

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/juju/blobstore/v3 v3.0.2
 	github.com/juju/charm/v11 v11.0.2
 	github.com/juju/clock v1.0.3
-	github.com/juju/cmd/v3 v3.0.13
+	github.com/juju/cmd/v3 v3.0.14
 	github.com/juju/collections v1.0.4
 	github.com/juju/description/v4 v4.0.11
 	github.com/juju/errors v1.0.0

go.sum

@@ -801,8 +801,8 @@ github.com/juju/clock v1.0.3 h1:yJHIsWXeU8j3QcBdiess09SzfiXRRrsjKPn2whnMeds=
 github.com/juju/clock v1.0.3/go.mod h1:HIBvJ8kiV/n7UHwKuCkdYL4l/MDECztHR2sAvWDxxf0=
 github.com/juju/cmd v0.0.0-20171107070456-e74f39857ca0/go.mod h1:yWJQHl73rdSX4DHVKGqkAip+huBslxRwS8m9CrOLq18=
 github.com/juju/cmd/v3 v3.0.0-20220202061353-b1cc80b193b0/go.mod h1:EoGJiEG+vbMwO9l+Es0SDTlaQPjH6nLcnnc4NfZB3cY=
-github.com/juju/cmd/v3 v3.0.13 h1:lNK/dPmlZsh/9DZBGTjLrEv1AD75K3Nsnyo8WxJpNYE=
-github.com/juju/cmd/v3 v3.0.13/go.mod h1:lGtDvm2BG+FKnIS8yY/vrhxQNX9imnL6bPIYGSTchuI=
+github.com/juju/cmd/v3 v3.0.14 h1:KuuamArSH7vQ6SdQKEHYK2scEMkJTEZKLs8abrlW3XE=
+github.com/juju/cmd/v3 v3.0.14/go.mod h1:lGtDvm2BG+FKnIS8yY/vrhxQNX9imnL6bPIYGSTchuI=
 github.com/juju/collections v0.0.0-20200605021417-0d0ec82b7271/go.mod h1:5XgO71dV1JClcOJE+4dzdn4HrI5LiyKd7PlVG6eZYhY=
 github.com/juju/collections v0.0.0-20220203020748-febd7cad8a7a/go.mod h1:JWeZdyttIEbkR51z2S13+J+aCuHVe0F6meRy+P0YGDo=
 github.com/juju/collections v1.0.0/go.mod h1:JWeZdyttIEbkR51z2S13+J+aCuHVe0F6meRy+P0YGDo=

internal/container/lxd/container.go

@@ -199,42 +199,6 @@ func (s *Server) AliveContainers(prefix string) ([]Container, error) {
 // FilterContainers retrieves the list of containers from the server and filters
 // them based on the input namespace prefix and any supplied statuses.
 func (s *Server) FilterContainers(prefix string, statuses ...string) ([]Container, error) {
-	// The retry here is required here because when creating a virtual machine
-	// container type, it does not always appear in the list of containers.
-	// There doesn't seem to be a status that's available to us that indicates
-	// that the machine is being created, but not yet quite alive.
-	//
-	// As we're trying to not have any logic that differentiates between
-	// containers and virtual machines, at a higher level, we retry here to
-	// prevent leaking the difference.
-	var containers []Container
-	err := retry.Call(retry.CallArgs{
-		Func: func() error {
-			var err error
-			containers, err = s.filterContainers(prefix, statuses)
-			return err
-		},
-		IsFatalError: func(err error) bool {
-			return !errors.Is(err, errors.NotFound)
-		},
-		NotifyFunc: func(err error, attempt int) {
-			logger.Debugf("failed to retrieve containers from LXD, attempt %d: %v", attempt, err)
-		},
-		Attempts:    10,
-		Delay:       1 * time.Second,
-		MaxDelay:    5 * time.Second,
-		MaxDuration: 30 * time.Second,
-		Clock:       s.clock,
-		BackoffFunc: retry.ExpBackoff(1*time.Second, 10*time.Second, 2.0, true),
-	})
-	if err != nil && !errors.Is(err, errors.NotFound) {
-		// No containers found is not an error.
-		return nil, errors.Trace(err)
-	}
-	return containers, nil
-}
-
-func (s *Server) filterContainers(prefix string, statuses []string) ([]Container, error) {
 	instances, err := s.GetInstances(api.InstanceTypeAny)
 	if err != nil {
 		return nil, errors.Trace(err)
@@ -250,9 +214,6 @@ func (s *Server) filterContainers(prefix string, statuses []string) ([]Container
 		}
 		results = append(results, Container{c})
 	}
-	if len(results) == 0 {
-		return nil, errors.NotFoundf("containers with prefix %q", prefix)
-	}
 	return results, nil
 }
 

internal/container/lxd/container_test.go

@@ -131,76 +131,6 @@ func (s *containerSuite) TestFilterContainers(c *gc.C) {
 	c.Check(filtered, gc.DeepEquals, expected)
 }
 
-func (s *containerSuite) TestFilterContainersRetry(c *gc.C) {
-	ctrl := gomock.NewController(c)
-	defer ctrl.Finish()
-	cSvr := s.NewMockServer(ctrl)
-
-	for i := 0; i < 3; i++ {
-		ret := []api.Instance{
-			{
-				Name:       "prefix-c1",
-				StatusCode: api.Pending,
-			},
-			{
-				Name:       "prefix-c2",
-				StatusCode: api.Pending,
-			},
-			{
-				Name:       "prefix-c3",
-				StatusCode: api.Pending,
-			},
-			{
-				Name:       "not-prefix-c4",
-				StatusCode: api.Pending,
-			},
-		}
-		cSvr.EXPECT().GetInstances(api.InstanceTypeAny).Return(ret, nil)
-	}
-
-	matching := []api.Instance{
-		{
-			Name:       "prefix-c1",
-			StatusCode: api.Starting,
-		},
-		{
-			Name:       "prefix-c2",
-			StatusCode: api.Stopped,
-		},
-	}
-	ret := append(matching, []api.Instance{
-		{
-			Name:       "prefix-c3",
-			StatusCode: api.Started,
-		},
-		{
-			Name:       "not-prefix-c4",
-			StatusCode: api.Stopped,
-		},
-	}...)
-	cSvr.EXPECT().GetInstances(api.InstanceTypeAny).Return(ret, nil)
-
-	clock := mocks.NewMockClock(ctrl)
-	clock.EXPECT().Now().Return(time.Now()).AnyTimes()
-	clock.EXPECT().After(gomock.Any()).DoAndReturn(func(d time.Duration) <-chan time.Time {
-		c := make(chan time.Time, 1)
-		defer close(c)
-		return c
-	}).AnyTimes()
-
-	jujuSvr, err := lxd.NewTestingServer(cSvr, clock)
-	c.Assert(err, jc.ErrorIsNil)
-
-	filtered, err := jujuSvr.FilterContainers("prefix", "Starting", "Stopped")
-	c.Assert(err, jc.ErrorIsNil)
-
-	expected := make([]lxd.Container, len(matching))
-	for i, v := range matching {
-		expected[i] = lxd.Container{v}
-	}
-	c.Check(filtered, gc.DeepEquals, expected)
-}
-
 func (s *containerSuite) TestAliveContainers(c *gc.C) {
 	ctrl := gomock.NewController(c)
 	defer ctrl.Finish()

internal/container/lxd/image.go

@@ -60,7 +60,7 @@ func (s *Server) FindImage(
 		// We already have an image with the given alias, so just use that.
 		target = entry.Target
 		image, _, err := s.GetImage(target)
-		if isCompatibleVirtType(virtType, image.Type) && err == nil {
+		if err == nil && isCompatibleVirtType(virtType, image.Type) {
 			logger.Debugf("Found image locally - %q %q", image.Filename, target)
 			return SourcedImage{
 				Image:     image,

internal/container/lxd/manager_test.go

@@ -6,7 +6,6 @@ package lxd_test
 import (
 	"context"
 	"errors"
-	stdtesting "testing"
 
 	lxdclient "github.com/canonical/lxd/client"
 	lxdapi "github.com/canonical/lxd/shared/api"
@@ -33,10 +32,6 @@ import (
 	coretesting "github.com/juju/juju/testing"
 )
 
-func Test(t *stdtesting.T) {
-	gc.TestingT(t)
-}
-
 type managerSuite struct {
 	lxdtesting.BaseSuite
 

provider/all/none.go

@@ -0,0 +1,8 @@
+// Copyright 2023 Canonical Ltd.
+// Licensed under the AGPLv3, see LICENCE file for details.
+
+package all
+
+// This file is here so that you can compile Juju with no providers
+// (i.e. BUILD_TAGS='minimal'). Otherwise, Go will complain that the build
+// constraints exclude all Go files in this package.

provider/lxd/environ_policy.go

@@ -48,7 +48,10 @@ func (env *environ) ConstraintsValidator(ctx envcontext.ProviderCallContext) (co
 
 // ShouldApplyControllerConstraints returns if bootstrapping logic should use
 // default constraints
-func (env *environ) ShouldApplyControllerConstraints() bool {
+func (env *environ) ShouldApplyControllerConstraints(cons constraints.Value) bool {
+	if cons.HasVirtType() && *cons.VirtType == "virtual-machine" {
+		return true
+	}
 	return false
 }
 

provider/lxd/environ_policy_test.go

@@ -263,6 +263,33 @@ func (s *environPolicySuite) TestSupportNetworks(c *gc.C) {
 	c.Check(isSupported, jc.IsFalse)
 }
 
+func (s *environPolicySuite) TestShouldApplyControllerConstraints(c *gc.C) {
+	defer s.setupMocks(c).Finish()
+
+	cons := constraints.MustParse("")
+
+	ok := s.env.(environs.DefaultConstraintsChecker).ShouldApplyControllerConstraints(cons)
+	c.Assert(ok, jc.IsFalse)
+}
+
+func (s *environPolicySuite) TestShouldApplyControllerConstraintsInvalid(c *gc.C) {
+	defer s.setupMocks(c).Finish()
+
+	cons := constraints.MustParse("virt-type=invalid")
+
+	ok := s.env.(environs.DefaultConstraintsChecker).ShouldApplyControllerConstraints(cons)
+	c.Assert(ok, jc.IsFalse)
+}
+
+func (s *environPolicySuite) TestShouldApplyControllerConstraintsForVirtualMachine(c *gc.C) {
+	defer s.setupMocks(c).Finish()
+
+	cons := constraints.MustParse("virt-type=virtual-machine")
+
+	ok := s.env.(environs.DefaultConstraintsChecker).ShouldApplyControllerConstraints(cons)
+	c.Assert(ok, jc.IsTrue)
+}
+
 func (s *environPolicySuite) setupMocks(c *gc.C) *gomock.Controller {
 	ctrl := gomock.NewController(c)
 

tests/suites/secrets_k8s/k8s.sh

@@ -301,7 +301,16 @@ prepare_vault() {
 	export VAULT_ADDR="http://${ip}:8200"
 	export VAULT_TOKEN="$root_token"
 
-	vault status
+	# wait for vault server to be ready.
+	attempt=0
+	until [[ $(vault status -format yaml 2>/dev/null | yq .initialized | grep -i 'true') ]]; do
+		if [[ ${attempt} -ge 30 ]]; then
+			echo "Failed: vault server was not able to be ready."
+			exit 1
+		fi
+		sleep 2
+		attempt=$((attempt + 1))
+	done
 }
 
 test_secrets() {

worker/caasapplicationprovisioner/doc.go

@@ -0,0 +1,12 @@
+// Copyright 2023 Canonical Ltd.
+// Licensed under the AGPLv3, see LICENCE file for details.
+
+// Package caasapplicationprovisioner defines two types of worker:
+//   - provisioner: Watches a Kubernetes model and starts a new worker
+//     of the appWorker type whenever an application is created.
+//   - appWorker: Drives the Kubernetes provider to create, manage,
+//     and destroy Kubernetes resources to match a requested state. Also
+//     writes the state of created resources (application/unit status,
+//     application/unit IP addresses & ports, filesystem info, etc.)
+//     back into the database.
+package caasapplicationprovisioner

worker/charmrevision/doc.go

@@ -0,0 +1,13 @@
+// Copyright 2023 Canonical Ltd.
+// Licensed under the AGPLv3, see LICENCE file for details.
+
+// Package charmrevision defines the charm revision updater worker. This worker
+// is responsible for polling Charmhub every 24 hours to check if there are new
+// revisions available of any repository charm deployed in the model. If so, it
+// will put a document in the Juju database, so that the next time the user runs
+// `juju status`, they can see that there is an update available. This worker
+// also sends anonymised usage metrics to Charmhub when it polls.
+//
+// This worker doesn't contain much business logic - most of the work is
+// delegated to the facade call.
+package charmrevision