From b93257621c14e589f65a484961cdeb702885d5c5 Mon Sep 17 00:00:00 2001 From: Eric Wollesen Date: Thu, 9 Feb 2023 08:14:34 -0700 Subject: [PATCH] Curated packages PBC full cluster lifecycle integration This change introduces the creation and deletion of a namespaced package bundle controller for workload clusters created via the full cluster lifecycle. Future commits will add packages installation/deletion via full cluster lifecycle. --- Makefile | 4 +- .../cmd/installpackagecontroller.go | 2 +- config/manifest/eksa-components.yaml | 12 + config/rbac/role.yaml | 12 + controllers/cluster_controller.go | 142 ++++++++- controllers/cluster_controller_legacy.go | 1 + controllers/cluster_controller_test.go | 297 ++++++++++++++++- controllers/cluster_controller_test_test.go | 101 +++++- controllers/factory.go | 23 +- controllers/mocks/cluster_controller.go | 111 +++++++ pkg/api/v1alpha1/cluster_types.go | 6 + pkg/curatedpackages/mocks/installer.go | 37 +++ pkg/curatedpackages/mocks/packageinstaller.go | 12 +- .../packagecontrollerclient.go | 107 ++++++- .../packagecontrollerclient_test.go | 298 ++++++++++++++---- pkg/curatedpackages/packageinstaller.go | 5 +- pkg/curatedpackages/packageinstaller_test.go | 6 +- pkg/dependencies/factory.go | 1 + pkg/executables/helm.go | 34 ++ pkg/executables/helm_test.go | 52 +++ 20 files changed, 1169 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index 6e28e907b0df2..d1f775fdfcd7a 100644 --- a/Makefile +++ b/Makefile @@ -575,7 +575,7 @@ mocks: ## Generate mocks ${MOCKGEN} -destination=pkg/curatedpackages/mocks/bundlemanager.go -package=mocks -source "pkg/curatedpackages/bundlemanager.go" Manager ${MOCKGEN} -destination=pkg/clients/kubernetes/mocks/kubectl.go -package=mocks -source "pkg/clients/kubernetes/unauth.go" ${MOCKGEN} -destination=pkg/clients/kubernetes/mocks/kubeconfig.go -package=mocks -source "pkg/clients/kubernetes/kubeconfig.go" - ${MOCKGEN} -destination=pkg/curatedpackages/mocks/installer.go -package=mocks -source "pkg/curatedpackages/packagecontrollerclient.go" ChartInstaller + ${MOCKGEN} -destination=pkg/curatedpackages/mocks/installer.go -package=mocks -source "pkg/curatedpackages/packagecontrollerclient.go" ChartInstaller ChartInstallationDeleter ${MOCKGEN} -destination=pkg/cluster/mocks/client_builder.go -package=mocks -source "pkg/cluster/client_builder.go" ${MOCKGEN} -destination=controllers/mocks/factory.go -package=mocks "github.com/aws/eks-anywhere/controllers" Manager ${MOCKGEN} -destination=pkg/networking/cilium/reconciler/mocks/templater.go -package=mocks -source "pkg/networking/cilium/reconciler/reconciler.go" @@ -585,7 +585,7 @@ mocks: ## Generate mocks ${MOCKGEN} -destination=pkg/providers/docker/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/docker/reconciler/reconciler.go" ${MOCKGEN} -destination=pkg/providers/tinkerbell/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/tinkerbell/reconciler/reconciler.go" ${MOCKGEN} -destination=pkg/awsiamauth/reconciler/mocks/reconciler.go -package=mocks -source "pkg/awsiamauth/reconciler/reconciler.go" - ${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator + ${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator PackageControllerClient ${MOCKGEN} -destination=pkg/workflow/task_mock_test.go -package=workflow_test -source "pkg/workflow/task.go" ${MOCKGEN} -destination=pkg/validations/createcluster/mocks/createcluster.go -package=mocks -source "pkg/validations/createcluster/createcluster.go" ${MOCKGEN} -destination=pkg/awsiamauth/mock_test.go -package=awsiamauth_test -source "pkg/awsiamauth/installer.go" diff --git a/cmd/eksctl-anywhere/cmd/installpackagecontroller.go b/cmd/eksctl-anywhere/cmd/installpackagecontroller.go index b7f97fc514694..769503a3b671d 100644 --- a/cmd/eksctl-anywhere/cmd/installpackagecontroller.go +++ b/cmd/eksctl-anywhere/cmd/installpackagecontroller.go @@ -72,7 +72,7 @@ func installPackageController(ctx context.Context) error { } curatedpackages.PrintLicense() - err = ctrlClient.EnableCuratedPackages(ctx) + err = ctrlClient.Enable(ctx) if err != nil { return err } diff --git a/config/manifest/eksa-components.yaml b/config/manifest/eksa-components.yaml index 76601db316ed9..f2c5326ad0fe3 100644 --- a/config/manifest/eksa-components.yaml +++ b/config/manifest/eksa-components.yaml @@ -5919,6 +5919,18 @@ rules: - patch - update - watch +- apiGroups: + - packages.eks.amazonaws.com + resources: + - packagebundlecontrollers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - tinkerbell.org resources: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 4c81fa1423dd2..841ba435400b1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -210,6 +210,18 @@ rules: - patch - update - watch +- apiGroups: + - packages.eks.amazonaws.com + resources: + - packagebundlecontrollers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - tinkerbell.org resources: diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go index 02a1452cf6c42..1838d7b3268df 100644 --- a/controllers/cluster_controller.go +++ b/controllers/cluster_controller.go @@ -3,10 +3,13 @@ package controllers import ( "context" "fmt" + "os" "time" "github.com/go-logr/logr" "github.com/pkg/errors" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -27,7 +30,10 @@ import ( "github.com/aws/eks-anywhere/pkg/controller/clientutil" "github.com/aws/eks-anywhere/pkg/controller/clusters" "github.com/aws/eks-anywhere/pkg/controller/handlers" + "github.com/aws/eks-anywhere/pkg/curatedpackages" + "github.com/aws/eks-anywhere/pkg/registrymirror" "github.com/aws/eks-anywhere/pkg/utils/ptr" + "github.com/aws/eks-anywhere/release/api/v1alpha1" ) const ( @@ -42,6 +48,7 @@ type ClusterReconciler struct { providerReconcilerRegistry ProviderClusterReconcilerRegistry awsIamAuth AWSIamConfigReconciler clusterValidator ClusterValidator + packagesClient PackagesClient } type ProviderClusterReconcilerRegistry interface { @@ -61,12 +68,13 @@ type ClusterValidator interface { } // NewClusterReconciler constructs a new ClusterReconciler. -func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator) *ClusterReconciler { +func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator, pkgs PackagesClient) *ClusterReconciler { return &ClusterReconciler{ client: client, providerReconcilerRegistry: registry, awsIamAuth: awsIamAuth, clusterValidator: clusterValidator, + packagesClient: pkgs, } } @@ -262,9 +270,135 @@ func (r *ClusterReconciler) postClusterProviderReconcile(ctx context.Context, lo } } + if cluster.IsPackagesEnabled() { + if err := r.postReconcilePackagesForWorkloadCluster(ctx, log, cluster); err != nil { + return controller.Result{}, err + } + } + return controller.Result{}, nil } +// PackagesClient handles curated packages operations from within the cluster +// controller. +type PackagesClient interface { + EnableFullLifecycle(context.Context, logr.Logger, string, string, *v1alpha1.Image, *registrymirror.RegistryMirror, ...curatedpackages.PackageControllerClientOpt) error + ReconcileDelete(context.Context, logr.Logger, client.Client, *anywherev1.Cluster) error +} + +func (r *ClusterReconciler) postReconcilePackagesForWorkloadCluster(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (err error) { + image, err := r.getBundleFromCluster(ctx, cluster) + if err != nil { + return err + } + + kubeConfig, err := r.writeKubeConfigSecret(ctx, cluster) + if err != nil { + return err + } + defer func() { + if err := kubeConfig.Remove(); err != nil { + log.Error(err, "removing kubeconfig file", "filename", kubeConfig.Name()) + } + }() + + registry := registrymirror.FromCluster(cluster) + + if err := r.packagesClient.EnableFullLifecycle(ctx, log, cluster.Name, kubeConfig.Name(), image, registry); err != nil { + return fmt.Errorf("packages client error: %w", err) + } + + log.V(6).Info("Installed curated packages on workload cluster", "cluster", cluster.Name) + + return nil +} + +// getBundleFromCluster based on the cluster's k8s version. +func (r *ClusterReconciler) getBundleFromCluster(ctx context.Context, cluster *anywherev1.Cluster) (*v1alpha1.Image, error) { + bundles := &v1alpha1.Bundles{} + nn := types.NamespacedName{ + Name: cluster.Spec.BundlesRef.Name, + Namespace: cluster.Spec.BundlesRef.Namespace, + } + if err := r.client.Get(ctx, nn, bundles); err != nil { + return nil, fmt.Errorf("retrieving bundle: %w", err) + } + + verBundle, err := r.findMatchingBundle(bundles, string(cluster.Spec.KubernetesVersion)) + if err != nil { + return nil, err + } + + return &verBundle.PackageController.HelmChart, nil +} + +func (r *ClusterReconciler) findMatchingBundle(bundles *v1alpha1.Bundles, kubeVersion string) (*v1alpha1.VersionsBundle, error) { + var verBundle *v1alpha1.VersionsBundle + for _, b := range bundles.Spec.VersionsBundles { + if b.KubeVersion == string(kubeVersion) { + verBundle = &b + break + } + } + if verBundle == nil { + return nil, fmt.Errorf("no bundle for kube version %q", kubeVersion) + } + + return verBundle, nil +} + +// writeKubeConfigSecret so that the helm executable can pick it up to install +// curated packages helm charts. +func (r *ClusterReconciler) writeKubeConfigSecret(ctx context.Context, cluster *anywherev1.Cluster) (kubeconfigFile, error) { + kubeConfigSecret := &corev1.Secret{} + kubeConfigNN := types.NamespacedName{ + Namespace: constants.EksaSystemNamespace, + Name: cluster.Name + "-kubeconfig", + } + if err := r.client.Get(ctx, kubeConfigNN, kubeConfigSecret); err != nil { + return nil, fmt.Errorf("getting kubeconfig secret: %w", err) + } + secretBytes, err := yaml.Marshal(kubeConfigSecret) + if err != nil { + return nil, fmt.Errorf("marshaling secret %w", err) + } + f, err := os.CreateTemp("", "kubeconfig-*.yaml") + if err != nil { + return nil, fmt.Errorf("opening kubeconfig file %w", err) + } + defer f.Close() + if _, err := f.Write(secretBytes); err != nil { + return nil, fmt.Errorf("writing kubeconfig file %w", err) + } + + return &removeableFile{f}, nil +} + +// kubeconfigFile adds utility operations to delete a filename after its no +// longer needed. +type kubeconfigFile interface { + // Name is the file's name on disk. + Name() string + // Remove is a best-effort deletion of the file on disk. + Remove() error +} + +// removeableFile wraps os.File to implement kubeconfigFile. +type removeableFile struct { + *os.File +} + +// Remove the file from disk. +func (f *removeableFile) Remove() error { + f.Close() + if err := os.Remove(f.Name()); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("removing removeableFile: %w", err) + } + } + return nil +} + func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (ctrl.Result, error) { if cluster.IsSelfManaged() { return ctrl.Result{}, errors.New("deleting self-managed clusters is not supported") @@ -310,6 +444,12 @@ func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger } } + if !cluster.IsSelfManaged() { + if err := r.packagesClient.ReconcileDelete(ctx, log, r.client, cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("deleting packages for cluster %q: %w", cluster.Name, err) + } + } + return ctrl.Result{}, nil } diff --git a/controllers/cluster_controller_legacy.go b/controllers/cluster_controller_legacy.go index 40715fdf7f38c..3e6d69ca6a7b7 100644 --- a/controllers/cluster_controller_legacy.go +++ b/controllers/cluster_controller_legacy.go @@ -60,6 +60,7 @@ func NewClusterReconcilerLegacy(client client.Client, log logr.Logger, scheme *r // +kubebuilder:rbac:groups="",namespace=eksa-system,resources=secrets,verbs=delete; // +kubebuilder:rbac:groups=tinkerbell.org,resources=hardware;hardware/status,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=machines;machines/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=packages.eks.amazonaws.com,resources=packagebundlecontrollers,verbs=get;list;watch;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/controllers/cluster_controller_test.go b/controllers/cluster_controller_test.go index 954d89b34dac2..538427faa09f0 100644 --- a/controllers/cluster_controller_test.go +++ b/controllers/cluster_controller_test.go @@ -3,10 +3,12 @@ package controllers_test import ( "context" "fmt" + "strings" "testing" "time" "github.com/go-logr/logr" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" apiv1 "k8s.io/api/core/v1" @@ -24,6 +26,7 @@ import ( "github.com/aws/eks-anywhere/controllers/mocks" "github.com/aws/eks-anywhere/internal/test/envtest" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/controller/clusters" "github.com/aws/eks-anywhere/pkg/govmomi" "github.com/aws/eks-anywhere/pkg/providers/vsphere" @@ -68,7 +71,12 @@ func newVsphereClusterReconcilerTest(t *testing.T, objs ...runtime.Object) *vsph Add(anywherev1.VSphereDatacenterKind, reconciler). Build() - r := controllers.NewClusterReconciler(cl, ®istry, iam, clusterValidator) + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockPkgs.EXPECT(). + ReconcileDelete(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil).AnyTimes() + + r := controllers.NewClusterReconciler(cl, ®istry, iam, clusterValidator, mockPkgs) return &vsphereClusterReconcilerTest{ govcClient: govcClient, @@ -98,10 +106,10 @@ func TestClusterReconcilerReconcileSelfManagedCluster(t *testing.T) { clusterValidator := mocks.NewMockClusterValidator(controller) registry := newRegistryMock(providerReconciler) c := fake.NewClientBuilder().WithRuntimeObjects(selfManagedCluster).Build() - + mockPkgs := mocks.NewMockPackagesClient(controller) providerReconciler.EXPECT().ReconcileWorkerNodes(ctx, gomock.AssignableToTypeOf(logr.Logger{}), sameName(selfManagedCluster)) - r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator) + r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, mockPkgs) result, err := r.Reconcile(ctx, clusterRequest(selfManagedCluster)) g.Expect(err).ToNot(HaveOccurred()) g.Expect(result).To(Equal(ctrl.Result{})) @@ -128,7 +136,7 @@ func TestClusterReconcilerReconcilePausedCluster(t *testing.T) { iam := mocks.NewMockAWSIamConfigReconciler(ctrl) clusterValidator := mocks.NewMockClusterValidator(ctrl) registry := newRegistryMock(providerReconciler) - r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator) + r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil) g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{})) api := envtest.NewAPIExpecter(t, c) @@ -164,7 +172,7 @@ func TestClusterReconcilerReconcileDeletedSelfManagedCluster(t *testing.T) { registry := newRegistryMock(providerReconciler) c := fake.NewClientBuilder().WithRuntimeObjects(selfManagedCluster).Build() - r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator) + r := controllers.NewClusterReconciler(c, registry, iam, clusterValidator, nil) _, err := r.Reconcile(ctx, clusterRequest(selfManagedCluster)) g.Expect(err).To(MatchError(ContainSubstring("deleting self-managed clusters is not supported"))) } @@ -233,7 +241,7 @@ func TestClusterReconcilerReconcileDeletePausedCluster(t *testing.T) { managementCluster, cluster, capiCluster, ).Build() - r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator) + r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil) g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{})) api := envtest.NewAPIExpecter(t, c) @@ -276,7 +284,7 @@ func TestClusterReconcilerReconcileDeleteClusterManagedByCLI(t *testing.T) { iam := mocks.NewMockAWSIamConfigReconciler(controller) clusterValidator := mocks.NewMockClusterValidator(controller) - r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator) + r := controllers.NewClusterReconciler(c, newRegistryForDummyProviderReconciler(), iam, clusterValidator, nil) g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).To(Equal(reconcile.Result{})) api := envtest.NewAPIExpecter(t, c) @@ -335,6 +343,281 @@ func TestClusterReconcilerDeleteNoCAPIClusterSuccess(t *testing.T) { } } +func TestClusterReconcilerSkipDontInstallPackagesOnSelfManaged(t *testing.T) { + ctx := context.Background() + cluster := &anywherev1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "my-namespace", + }, + Spec: anywherev1.ClusterSpec{ + KubernetesVersion: "v1.25", + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + Namespace: "my-namespace", + }, + ManagementCluster: anywherev1.ManagementCluster{ + Name: "", + }, + }, + } + objs := []runtime.Object{cluster} + cb := fake.NewClientBuilder() + mockClient := cb.WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + + ctrl := gomock.NewController(t) + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockPkgs.EXPECT().ReconcileDelete(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + r := controllers.NewClusterReconciler(mockClient, nullRegistry, nil, nil, mockPkgs) + _, err := r.Reconcile(ctx, clusterRequest(cluster)) + if err != nil { + t.Fatalf("expected err to be nil, got %s", err) + } +} + +func TestClusterReconcilerDontDeletePackagesOnSelfManaged(t *testing.T) { + ctx := context.Background() + deleteTime := metav1.NewTime(time.Now().Add(-1 * time.Second)) + cluster := &anywherev1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster", + Namespace: "my-namespace", + DeletionTimestamp: &deleteTime, + }, + Spec: anywherev1.ClusterSpec{ + KubernetesVersion: "v1.25", + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + Namespace: "my-namespace", + }, + ManagementCluster: anywherev1.ManagementCluster{ + Name: "", + }, + }, + } + objs := []runtime.Object{cluster} + cb := fake.NewClientBuilder() + mockClient := cb.WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + + ctrl := gomock.NewController(t) + // At the moment, Reconcile won't get this far, but if the time comes when + // deleting self-managed clusters via full cluster lifecycle happens, we + // need to be aware and adapt appropriately. + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockPkgs.EXPECT().ReconcileDelete(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + r := controllers.NewClusterReconciler(mockClient, nullRegistry, nil, nil, mockPkgs) + _, err := r.Reconcile(ctx, clusterRequest(cluster)) + if err == nil || !strings.Contains(err.Error(), "deleting self-managed clusters is not supported") { + t.Fatalf("unexpected error %s", err) + } +} + +func TestClusterReconcilerPackagesDeletion(s *testing.T) { + newTestCluster := func() *anywherev1.Cluster { + deleteTime := metav1.NewTime(time.Now().Add(-1 * time.Second)) + return &anywherev1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-workload-cluster", + Namespace: "my-namespace", + DeletionTimestamp: &deleteTime, + }, + Spec: anywherev1.ClusterSpec{ + KubernetesVersion: "v1.25", + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + Namespace: "my-namespace", + }, + ManagementCluster: anywherev1.ManagementCluster{ + Name: "my-management-cluster", + }, + }, + } + } + + s.Run("errors when packages client errors", func(t *testing.T) { + ctx := context.Background() + log := testr.New(t) + logCtx := ctrl.LoggerInto(ctx, log) + cluster := newTestCluster() + cluster.Spec.BundlesRef.Name = "non-existent" + ctrl := gomock.NewController(t) + objs := []runtime.Object{cluster} + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockPkgs.EXPECT().ReconcileDelete(logCtx, log, gomock.Any(), gomock.Any()).Return(fmt.Errorf("test error")) + mockIAM := mocks.NewMockAWSIamConfigReconciler(ctrl) + mockValid := mocks.NewMockClusterValidator(ctrl) + + r := controllers.NewClusterReconciler(fakeClient, nullRegistry, mockIAM, mockValid, mockPkgs) + _, err := r.Reconcile(logCtx, clusterRequest(cluster)) + if err == nil || !strings.Contains(err.Error(), "test error") { + t.Errorf("expected packages client deletion error, got %s", err) + } + }) +} + +func TestClusterReconcilerPackagesInstall(s *testing.T) { + newTestCluster := func() *anywherev1.Cluster { + return &anywherev1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-workload-cluster", + Namespace: "my-namespace", + }, + Spec: anywherev1.ClusterSpec{ + KubernetesVersion: "v1.25", + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + Namespace: "my-namespace", + }, + ManagementCluster: anywherev1.ManagementCluster{ + Name: "my-management-cluster", + }, + }, + } + } + + s.Run("errors when bundles aren't found", func(t *testing.T) { + ctx := context.Background() + log := testr.New(t) + logCtx := ctrl.LoggerInto(ctx, log) + cluster := newTestCluster() + ctrl := gomock.NewController(t) + objs := []runtime.Object{cluster} + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockIAM := mocks.NewMockAWSIamConfigReconciler(ctrl) + mockValid := mocks.NewMockClusterValidator(ctrl) + mockValid.EXPECT().ValidateManagementClusterName(logCtx, log, gomock.Any()).Return(nil) + + r := controllers.NewClusterReconciler(fakeClient, nullRegistry, mockIAM, mockValid, mockPkgs) + _, err := r.Reconcile(logCtx, clusterRequest(cluster)) + if err == nil || !apierrors.IsNotFound(err) { + t.Errorf("expected not found err getting cluster resource, got %s", err) + } + }) + + s.Run("errors when a matching k8s bundle version isn't found", func(t *testing.T) { + ctx := context.Background() + log := testr.New(t) + logCtx := ctrl.LoggerInto(ctx, log) + cluster := newTestCluster() + cluster.Spec.KubernetesVersion = "non-existent" + ctrl := gomock.NewController(t) + bundles := createBundle(cluster) + bundles.ObjectMeta.Name = cluster.Spec.BundlesRef.Name + bundles.ObjectMeta.Namespace = cluster.Spec.BundlesRef.Namespace + objs := []runtime.Object{cluster, bundles} + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockIAM := mocks.NewMockAWSIamConfigReconciler(ctrl) + mockValid := mocks.NewMockClusterValidator(ctrl) + mockValid.EXPECT().ValidateManagementClusterName(logCtx, log, gomock.Any()).Return(nil) + + r := controllers.NewClusterReconciler(fakeClient, nullRegistry, mockIAM, mockValid, mockPkgs) + _, err := r.Reconcile(logCtx, clusterRequest(cluster)) + if err == nil || !strings.Contains(err.Error(), "no bundle for kube version") { + t.Errorf("expected no bundle for kube version error, got %s", err) + } + }) + + s.Run("errors when the kube config secret isn't found", func(t *testing.T) { + ctx := context.Background() + log := testr.New(t) + logCtx := ctrl.LoggerInto(ctx, log) + cluster := newTestCluster() + ctrl := gomock.NewController(t) + bundles := createBundle(cluster) + bundles.Spec.VersionsBundles[0].KubeVersion = string(cluster.Spec.KubernetesVersion) + bundles.ObjectMeta.Name = cluster.Spec.BundlesRef.Name + bundles.ObjectMeta.Namespace = cluster.Spec.BundlesRef.Namespace + objs := []runtime.Object{cluster, bundles} + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockIAM := mocks.NewMockAWSIamConfigReconciler(ctrl) + mockValid := mocks.NewMockClusterValidator(ctrl) + mockValid.EXPECT().ValidateManagementClusterName(logCtx, log, gomock.Any()).Return(nil) + + r := controllers.NewClusterReconciler(fakeClient, nullRegistry, mockIAM, mockValid, mockPkgs) + _, err := r.Reconcile(logCtx, clusterRequest(cluster)) + if err == nil || !apierrors.IsNotFound(err) { + t.Errorf("expected no kubeconfig secret error, got %s", err) + } + }) + + s.Run("errors when enable fails", func(t *testing.T) { + ctx := context.Background() + log := testr.New(t) + logCtx := ctrl.LoggerInto(ctx, log) + cluster := newTestCluster() + ctrl := gomock.NewController(t) + bundles := createBundle(cluster) + bundles.Spec.VersionsBundles[0].KubeVersion = string(cluster.Spec.KubernetesVersion) + bundles.ObjectMeta.Name = cluster.Spec.BundlesRef.Name + bundles.ObjectMeta.Namespace = cluster.Spec.BundlesRef.Namespace + secret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: constants.EksaSystemNamespace, + Name: cluster.Name + "-kubeconfig", + }, + } + objs := []runtime.Object{cluster, bundles, secret} + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + mockIAM := mocks.NewMockAWSIamConfigReconciler(ctrl) + mockValid := mocks.NewMockClusterValidator(ctrl) + mockValid.EXPECT().ValidateManagementClusterName(logCtx, log, gomock.Any()).Return(nil) + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockPkgs.EXPECT().EnableFullLifecycle(logCtx, log, cluster.Name, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("test error")) + + r := controllers.NewClusterReconciler(fakeClient, nullRegistry, mockIAM, mockValid, mockPkgs) + _, err := r.Reconcile(logCtx, clusterRequest(cluster)) + if err == nil || !strings.Contains(err.Error(), "packages client error: test error") { + t.Errorf("expected packages client error, got %s", err) + } + }) + + s.Run("skips installation when disabled via cluster spec", func(t *testing.T) { + ctx := context.Background() + log := testr.New(t) + logCtx := ctrl.LoggerInto(ctx, log) + cluster := newTestCluster() + cluster.Spec.Packages = &anywherev1.PackageConfiguration{Disable: true} + ctrl := gomock.NewController(t) + bundles := createBundle(cluster) + bundles.Spec.VersionsBundles[0].KubeVersion = string(cluster.Spec.KubernetesVersion) + bundles.ObjectMeta.Name = cluster.Spec.BundlesRef.Name + bundles.ObjectMeta.Namespace = cluster.Spec.BundlesRef.Namespace + secret := &apiv1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: constants.EksaSystemNamespace, + Name: cluster.Name + "-kubeconfig", + }, + } + objs := []runtime.Object{cluster, bundles, secret} + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() + nullRegistry := newRegistryForDummyProviderReconciler() + mockIAM := mocks.NewMockAWSIamConfigReconciler(ctrl) + mockValid := mocks.NewMockClusterValidator(ctrl) + mockValid.EXPECT().ValidateManagementClusterName(logCtx, log, gomock.Any()).Return(nil) + mockPkgs := mocks.NewMockPackagesClient(ctrl) + mockPkgs.EXPECT(). + EnableFullLifecycle(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Times(0) + + r := controllers.NewClusterReconciler(fakeClient, nullRegistry, mockIAM, mockValid, mockPkgs) + _, err := r.Reconcile(logCtx, clusterRequest(cluster)) + if err != nil { + t.Errorf("expected nil error, got %s", err) + } + }) +} + func createWNMachineConfig() *anywherev1.VSphereMachineConfig { return &anywherev1.VSphereMachineConfig{ TypeMeta: metav1.TypeMeta{ diff --git a/controllers/cluster_controller_test_test.go b/controllers/cluster_controller_test_test.go index 2bb91c6a07269..84b7f5e37f72c 100644 --- a/controllers/cluster_controller_test_test.go +++ b/controllers/cluster_controller_test_test.go @@ -8,6 +8,7 @@ import ( "github.com/go-logr/logr" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -21,8 +22,10 @@ import ( "github.com/aws/eks-anywhere/internal/test/envtest" anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/cluster" + "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/controller" "github.com/aws/eks-anywhere/pkg/controller/clusters" + "github.com/aws/eks-anywhere/release/api/v1alpha1" ) func TestClusterReconcilerEnsureOwnerReferences(t *testing.T) { @@ -34,6 +37,11 @@ func TestClusterReconcilerEnsureOwnerReferences(t *testing.T) { Name: "my-management-cluster", Namespace: "my-namespace", }, + Spec: anywherev1.ClusterSpec{ + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + }, + }, } cluster := &anywherev1.Cluster{ @@ -41,6 +49,13 @@ func TestClusterReconcilerEnsureOwnerReferences(t *testing.T) { Name: "my-cluster", Namespace: "my-namespace", }, + Spec: anywherev1.ClusterSpec{ + KubernetesVersion: "v1.25", + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + Namespace: "my-namespace", + }, + }, } cluster.Spec.IdentityProviderRefs = []anywherev1.Ref{ { @@ -73,7 +88,29 @@ func TestClusterReconcilerEnsureOwnerReferences(t *testing.T) { }, }, } - objs := []runtime.Object{cluster, managementCluster, oidc, awsIAM} + bundles := &v1alpha1.Bundles{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bundles-ref", + Namespace: cluster.Namespace, + }, + Spec: v1alpha1.BundlesSpec{ + VersionsBundles: []v1alpha1.VersionsBundle{ + { + KubeVersion: "v1.25", + PackageController: v1alpha1.PackageBundle{ + HelmChart: v1alpha1.Image{}, + }, + }, + }, + }, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-kubeconfig", + Namespace: constants.EksaSystemNamespace, + }, + } + objs := []runtime.Object{cluster, managementCluster, oidc, awsIAM, bundles, secret} cb := fake.NewClientBuilder() cl := cb.WithRuntimeObjects(objs...).Build() @@ -84,8 +121,15 @@ func TestClusterReconcilerEnsureOwnerReferences(t *testing.T) { validator := newMockClusterValidator(t) validator.EXPECT().ValidateManagementClusterName(ctx, gomock.AssignableToTypeOf(logr.Logger{}), gomock.AssignableToTypeOf(cluster)).Return(nil) - r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), iam, validator) + pcc := newMockPackagesClient(t) + pcc.EXPECT().EnableFullLifecycle(ctx, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + + r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), iam, validator, pcc) _, err := r.Reconcile(ctx, clusterRequest(cluster)) + + g.Expect(cl.Get(ctx, client.ObjectKey{Namespace: cluster.Spec.BundlesRef.Namespace, Name: cluster.Spec.BundlesRef.Name}, bundles)).To(Succeed()) + g.Expect(cl.Get(ctx, client.ObjectKey{Namespace: constants.EksaSystemNamespace, Name: cluster.Name + "-kubeconfig"}, secret)).To(Succeed()) + g.Expect(err).NotTo(HaveOccurred()) newOidc := &anywherev1.OIDCConfig{} @@ -133,7 +177,7 @@ func TestClusterReconcilerReconcileChildObjectNotFound(t *testing.T) { cl := cb.WithRuntimeObjects(objs...).Build() api := envtest.NewAPIExpecter(t, cl) - r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), newMockClusterValidator(t)) + r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), newMockClusterValidator(t), nil) g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).Error().To(MatchError(ContainSubstring("not found"))) c := envtest.CloneNameNamespace(cluster) api.ShouldEventuallyMatch(ctx, c, func(g Gomega) { @@ -145,7 +189,7 @@ func TestClusterReconcilerReconcileChildObjectNotFound(t *testing.T) { func TestClusterReconcilerSetupWithManager(t *testing.T) { client := env.Client() - r := controllers.NewClusterReconciler(client, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), newMockClusterValidator(t)) + r := controllers.NewClusterReconciler(client, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), newMockClusterValidator(t), nil) g := NewWithT(t) g.Expect(r.SetupWithManager(env.Manager(), env.Manager().GetLogger())).To(Succeed()) @@ -174,7 +218,7 @@ func TestClusterReconcilerManagementClusterNotFound(t *testing.T) { cl := cb.WithRuntimeObjects(objs...).Build() api := envtest.NewAPIExpecter(t, cl) - r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), newMockClusterValidator(t)) + r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), newMockClusterValidator(t), nil) g.Expect(r.Reconcile(ctx, clusterRequest(cluster))).Error().To(MatchError(ContainSubstring("\"my-management-cluster\" not found"))) c := envtest.CloneNameNamespace(cluster) api.ShouldEventuallyMatch(ctx, c, func(g Gomega) { @@ -192,7 +236,8 @@ func TestClusterReconcilerSetBundlesRef(t *testing.T) { }, Spec: anywherev1.ClusterSpec{ BundlesRef: &anywherev1.BundlesRef{ - Name: "my-bundles-ref", + Name: "my-bundles-ref", + Namespace: "my-namespace", }, }, } @@ -201,20 +246,53 @@ func TestClusterReconcilerSetBundlesRef(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "my-cluster", }, + Spec: anywherev1.ClusterSpec{ + KubernetesVersion: "v1.25", + BundlesRef: &anywherev1.BundlesRef{ + Name: "my-bundles-ref", + Namespace: "my-namespace", + }, + }, } cluster.SetManagedBy("my-management-cluster") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-cluster-kubeconfig", + Namespace: constants.EksaSystemNamespace, + }, + } + bundles := &v1alpha1.Bundles{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-bundles-ref", + Namespace: cluster.Spec.BundlesRef.Namespace, + }, + Spec: v1alpha1.BundlesSpec{ + VersionsBundles: []v1alpha1.VersionsBundle{ + { + KubeVersion: "v1.25", + PackageController: v1alpha1.PackageBundle{ + HelmChart: v1alpha1.Image{}, + }, + }, + }, + }, + } - objs := []runtime.Object{cluster, managementCluster} + objs := []runtime.Object{cluster, managementCluster, secret, bundles} cb := fake.NewClientBuilder() cl := cb.WithRuntimeObjects(objs...).Build() mgmtCluster := &anywherev1.Cluster{} g.Expect(cl.Get(ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: managementCluster.Name}, mgmtCluster)).To(Succeed()) + g.Expect(cl.Get(ctx, client.ObjectKey{Namespace: cluster.Spec.BundlesRef.Namespace, Name: cluster.Spec.BundlesRef.Name}, bundles)).To(Succeed()) + g.Expect(cl.Get(ctx, client.ObjectKey{Namespace: constants.EksaSystemNamespace, Name: cluster.Name + "-kubeconfig"}, secret)).To(Succeed()) + pcc := newMockPackagesClient(t) + pcc.EXPECT().EnableFullLifecycle(ctx, gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) validator := newMockClusterValidator(t) validator.EXPECT().ValidateManagementClusterName(ctx, gomock.AssignableToTypeOf(logr.Logger{}), gomock.AssignableToTypeOf(cluster)).Return(nil) - r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), validator) + r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), validator, pcc) _, err := r.Reconcile(ctx, clusterRequest(cluster)) g.Expect(err).ToNot(HaveOccurred()) @@ -250,7 +328,7 @@ func TestClusterReconcilerWorkloadClusterMgmtClusterNameFail(t *testing.T) { validator.EXPECT().ValidateManagementClusterName(ctx, gomock.AssignableToTypeOf(logr.Logger{}), gomock.AssignableToTypeOf(cluster)). Return(errors.New("test error")) - r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), validator) + r := controllers.NewClusterReconciler(cl, newRegistryForDummyProviderReconciler(), newMockAWSIamConfigReconciler(t), validator, nil) _, err := r.Reconcile(ctx, clusterRequest(cluster)) g.Expect(err).To(HaveOccurred()) } @@ -309,3 +387,8 @@ func newMockClusterValidator(t *testing.T) *mocks.MockClusterValidator { ctrl := gomock.NewController(t) return mocks.NewMockClusterValidator(ctrl) } + +func newMockPackagesClient(t *testing.T) *mocks.MockPackagesClient { + ctrl := gomock.NewController(t) + return mocks.NewMockPackagesClient(ctrl) +} diff --git a/controllers/factory.go b/controllers/factory.go index 37f8ed7d8e0ba..0abe8792f303e 100644 --- a/controllers/factory.go +++ b/controllers/factory.go @@ -13,7 +13,9 @@ import ( awsiamconfigreconciler "github.com/aws/eks-anywhere/pkg/awsiamauth/reconciler" "github.com/aws/eks-anywhere/pkg/controller/clusters" "github.com/aws/eks-anywhere/pkg/crypto" + "github.com/aws/eks-anywhere/pkg/curatedpackages" "github.com/aws/eks-anywhere/pkg/dependencies" + "github.com/aws/eks-anywhere/pkg/executables" ciliumreconciler "github.com/aws/eks-anywhere/pkg/networking/cilium/reconciler" cnireconciler "github.com/aws/eks-anywhere/pkg/networking/reconciler" dockerreconciler "github.com/aws/eks-anywhere/pkg/providers/docker/reconciler" @@ -42,6 +44,7 @@ type Factory struct { awsIamConfigReconciler *awsiamconfigreconciler.Reconciler logger logr.Logger deps *dependencies.Dependencies + packageControllerClient *curatedpackages.PackageControllerClient } type Reconcilers struct { @@ -89,7 +92,10 @@ func (f *Factory) Close(ctx context.Context) error { func (f *Factory) WithClusterReconciler(capiProviders []clusterctlv1.Provider) *Factory { f.dependencyFactory.WithGovc() - f.withTracker().WithProviderClusterReconcilerRegistry(capiProviders).withAWSIamConfigReconciler() + f.withTracker(). + WithProviderClusterReconcilerRegistry(capiProviders). + withAWSIamConfigReconciler(). + withPackageControllerClient() f.buildSteps = append(f.buildSteps, func(ctx context.Context) error { if f.reconcilers.ClusterReconciler != nil { @@ -101,6 +107,7 @@ func (f *Factory) WithClusterReconciler(capiProviders []clusterctlv1.Provider) * f.registry, f.awsIamConfigReconciler, clusters.NewClusterValidator(f.manager.GetClient()), + f.packageControllerClient, ) return nil @@ -385,3 +392,17 @@ func (f *Factory) withAWSIamConfigReconciler() *Factory { return f } + +func (f *Factory) withPackageControllerClient() *Factory { + f.dependencyFactory.WithHelm(executables.WithSkipCRDs()).WithKubectl() + + f.buildSteps = append(f.buildSteps, func(ctx context.Context) error { + if f.packageControllerClient != nil { + return nil + } + f.packageControllerClient = curatedpackages.NewPackageControllerClientFullLifecycle(f.logger, f.deps.Helm, f.deps.Helm, f.deps.Kubectl) + return nil + }) + + return f +} diff --git a/controllers/mocks/cluster_controller.go b/controllers/mocks/cluster_controller.go index 64d1e5f75d53b..a929ffcfb16dc 100644 --- a/controllers/mocks/cluster_controller.go +++ b/controllers/mocks/cluster_controller.go @@ -11,8 +11,12 @@ import ( v1alpha1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" controller "github.com/aws/eks-anywhere/pkg/controller" clusters "github.com/aws/eks-anywhere/pkg/controller/clusters" + curatedpackages "github.com/aws/eks-anywhere/pkg/curatedpackages" + registrymirror "github.com/aws/eks-anywhere/pkg/registrymirror" + v1alpha10 "github.com/aws/eks-anywhere/release/api/v1alpha1" logr "github.com/go-logr/logr" gomock "github.com/golang/mock/gomock" + client "sigs.k8s.io/controller-runtime/pkg/client" ) // MockProviderClusterReconcilerRegistry is a mock of ProviderClusterReconcilerRegistry interface. @@ -155,3 +159,110 @@ func (mr *MockClusterValidatorMockRecorder) ValidateManagementClusterName(ctx, l mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateManagementClusterName", reflect.TypeOf((*MockClusterValidator)(nil).ValidateManagementClusterName), ctx, log, cluster) } + +// MockPackagesClient is a mock of PackagesClient interface. +type MockPackagesClient struct { + ctrl *gomock.Controller + recorder *MockPackagesClientMockRecorder +} + +// MockPackagesClientMockRecorder is the mock recorder for MockPackagesClient. +type MockPackagesClientMockRecorder struct { + mock *MockPackagesClient +} + +// NewMockPackagesClient creates a new mock instance. +func NewMockPackagesClient(ctrl *gomock.Controller) *MockPackagesClient { + mock := &MockPackagesClient{ctrl: ctrl} + mock.recorder = &MockPackagesClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPackagesClient) EXPECT() *MockPackagesClientMockRecorder { + return m.recorder +} + +// EnableFullLifecycle mocks base method. +func (m *MockPackagesClient) EnableFullLifecycle(arg0 context.Context, arg1 logr.Logger, arg2, arg3 string, arg4 *v1alpha10.Image, arg5 *registrymirror.RegistryMirror, arg6 ...curatedpackages.PackageControllerClientOpt) error { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3, arg4, arg5} + for _, a := range arg6 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "EnableFullLifecycle", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnableFullLifecycle indicates an expected call of EnableFullLifecycle. +func (mr *MockPackagesClientMockRecorder) EnableFullLifecycle(arg0, arg1, arg2, arg3, arg4, arg5 interface{}, arg6 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3, arg4, arg5}, arg6...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableFullLifecycle", reflect.TypeOf((*MockPackagesClient)(nil).EnableFullLifecycle), varargs...) +} + +// ReconcileDelete mocks base method. +func (m *MockPackagesClient) ReconcileDelete(arg0 context.Context, arg1 logr.Logger, arg2 client.Client, arg3 *v1alpha1.Cluster) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReconcileDelete", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReconcileDelete indicates an expected call of ReconcileDelete. +func (mr *MockPackagesClientMockRecorder) ReconcileDelete(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconcileDelete", reflect.TypeOf((*MockPackagesClient)(nil).ReconcileDelete), arg0, arg1, arg2, arg3) +} + +// MockkubeconfigFile is a mock of kubeconfigFile interface. +type MockkubeconfigFile struct { + ctrl *gomock.Controller + recorder *MockkubeconfigFileMockRecorder +} + +// MockkubeconfigFileMockRecorder is the mock recorder for MockkubeconfigFile. +type MockkubeconfigFileMockRecorder struct { + mock *MockkubeconfigFile +} + +// NewMockkubeconfigFile creates a new mock instance. +func NewMockkubeconfigFile(ctrl *gomock.Controller) *MockkubeconfigFile { + mock := &MockkubeconfigFile{ctrl: ctrl} + mock.recorder = &MockkubeconfigFileMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockkubeconfigFile) EXPECT() *MockkubeconfigFileMockRecorder { + return m.recorder +} + +// Name mocks base method. +func (m *MockkubeconfigFile) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockkubeconfigFileMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockkubeconfigFile)(nil).Name)) +} + +// Remove mocks base method. +func (m *MockkubeconfigFile) Remove() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Remove") + ret0, _ := ret[0].(error) + return ret0 +} + +// Remove indicates an expected call of Remove. +func (mr *MockkubeconfigFileMockRecorder) Remove() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Remove", reflect.TypeOf((*MockkubeconfigFile)(nil).Remove)) +} diff --git a/pkg/api/v1alpha1/cluster_types.go b/pkg/api/v1alpha1/cluster_types.go index 2cd1983c7b158..7442c6009db4f 100644 --- a/pkg/api/v1alpha1/cluster_types.go +++ b/pkg/api/v1alpha1/cluster_types.go @@ -70,6 +70,12 @@ func (c *Cluster) HasAWSIamConfig() bool { return false } +// IsPackagesEnabled checks if the configuration supports curated packages +// installation. +func (c *Cluster) IsPackagesEnabled() bool { + return !c.IsSelfManaged() && (c.Spec.Packages == nil || !c.Spec.Packages.Disable) +} + func (n *Cluster) Equal(o *Cluster) bool { if n == o { return true diff --git a/pkg/curatedpackages/mocks/installer.go b/pkg/curatedpackages/mocks/installer.go index efcf494f3fc4e..334263b1b6d46 100644 --- a/pkg/curatedpackages/mocks/installer.go +++ b/pkg/curatedpackages/mocks/installer.go @@ -47,3 +47,40 @@ func (mr *MockChartInstallerMockRecorder) InstallChart(ctx, chart, ociURI, versi mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallChart", reflect.TypeOf((*MockChartInstaller)(nil).InstallChart), ctx, chart, ociURI, version, kubeconfigFilePath, namespace, valueFilePath, values) } + +// MockChartInstallationDeleter is a mock of ChartInstallationDeleter interface. +type MockChartInstallationDeleter struct { + ctrl *gomock.Controller + recorder *MockChartInstallationDeleterMockRecorder +} + +// MockChartInstallationDeleterMockRecorder is the mock recorder for MockChartInstallationDeleter. +type MockChartInstallationDeleterMockRecorder struct { + mock *MockChartInstallationDeleter +} + +// NewMockChartInstallationDeleter creates a new mock instance. +func NewMockChartInstallationDeleter(ctrl *gomock.Controller) *MockChartInstallationDeleter { + mock := &MockChartInstallationDeleter{ctrl: ctrl} + mock.recorder = &MockChartInstallationDeleterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockChartInstallationDeleter) EXPECT() *MockChartInstallationDeleterMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockChartInstallationDeleter) Delete(ctx context.Context, kubeconfigFilePath, installName, namespace string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, kubeconfigFilePath, installName, namespace) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockChartInstallationDeleterMockRecorder) Delete(ctx, kubeconfigFilePath, installName, namespace interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockChartInstallationDeleter)(nil).Delete), ctx, kubeconfigFilePath, installName, namespace) +} diff --git a/pkg/curatedpackages/mocks/packageinstaller.go b/pkg/curatedpackages/mocks/packageinstaller.go index a36ef0ec3c241..50609050885ba 100644 --- a/pkg/curatedpackages/mocks/packageinstaller.go +++ b/pkg/curatedpackages/mocks/packageinstaller.go @@ -34,18 +34,18 @@ func (m *MockPackageController) EXPECT() *MockPackageControllerMockRecorder { return m.recorder } -// EnableCuratedPackages mocks base method. -func (m *MockPackageController) EnableCuratedPackages(ctx context.Context) error { +// Enable mocks base method. +func (m *MockPackageController) Enable(ctx context.Context) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EnableCuratedPackages", ctx) + ret := m.ctrl.Call(m, "Enable", ctx) ret0, _ := ret[0].(error) return ret0 } -// EnableCuratedPackages indicates an expected call of EnableCuratedPackages. -func (mr *MockPackageControllerMockRecorder) EnableCuratedPackages(ctx interface{}) *gomock.Call { +// Enable indicates an expected call of Enable. +func (mr *MockPackageControllerMockRecorder) Enable(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnableCuratedPackages", reflect.TypeOf((*MockPackageController)(nil).EnableCuratedPackages), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enable", reflect.TypeOf((*MockPackageController)(nil).Enable), ctx) } // IsInstalled mocks base method. diff --git a/pkg/curatedpackages/packagecontrollerclient.go b/pkg/curatedpackages/packagecontrollerclient.go index 18c6f4f4cc967..6e770d887f9aa 100644 --- a/pkg/curatedpackages/packagecontrollerclient.go +++ b/pkg/curatedpackages/packagecontrollerclient.go @@ -9,8 +9,14 @@ import ( "strings" "time" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + packagesv1 "github.com/aws/eks-anywhere-packages/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + anywherev1 "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/cluster" "github.com/aws/eks-anywhere/pkg/config" "github.com/aws/eks-anywhere/pkg/constants" @@ -32,9 +38,11 @@ const ( type PackageControllerClientOpt func(client *PackageControllerClient) type PackageControllerClient struct { - kubeConfig string - chart *releasev1.Image - chartInstaller ChartInstaller + kubeConfig string + chart *releasev1.Image + chartInstaller ChartInstaller + // deleter handles helm chart install deletion. + deleter ChartInstallationDeleter clusterName string clusterSpec *v1alpha1.ClusterSpec managementClusterName string @@ -49,19 +57,79 @@ type PackageControllerClient struct { // activeBundleTimeout is the timeout to activate a bundle on installation. activeBundleTimeout time.Duration valuesFileWriter filewriter.FileWriter + // skipWaitForPackageBundle indicates whether the installer should wait + // until a package bundle is activated. + // + // Skipping the wait is desirable for full cluster lifecycle use cases, + // where resource creation and error reporting are asynchronous in nature. + skipWaitForPackageBundle bool } type ChartInstaller interface { InstallChart(ctx context.Context, chart, ociURI, version, kubeconfigFilePath, namespace, valueFilePath string, values []string) error } +// ChartInstallationDeleter handles deleting chart installations. +type ChartInstallationDeleter interface { + Delete(ctx context.Context, kubeconfigFilePath, installName, namespace string) error +} + +// NewPackageControllerClientFullLifecycle creates a PackageControllerClient +// for the Full Cluster Lifecycle controller. +// +// It differs because the CLI use case has far more information available at +// instantiation, while the FCL use case has less information at +// instantiation, and the rest when cluster creation is triggered. +func NewPackageControllerClientFullLifecycle(logger logr.Logger, chartInstaller ChartInstaller, deleter ChartInstallationDeleter, kubectl KubectlRunner) *PackageControllerClient { + return &PackageControllerClient{ + chartInstaller: chartInstaller, + deleter: deleter, + kubectl: kubectl, + skipWaitForPackageBundle: true, + eksaRegion: eksaDefaultRegion, + } +} + +// EnableFullLifecycle wraps Enable to handle run-time arguments. +// +// This method fills in the gaps between the original CLI use case, where all +// information is known at PackageControllerClient initialization, and the +// Full Cluster Lifecycle use case, where there's limited information at +// initialization. +func (pc *PackageControllerClient) EnableFullLifecycle(ctx context.Context, log logr.Logger, clusterName, kubeConfig string, chart *releasev1.Image, registryMirror *registrymirror.RegistryMirror, options ...PackageControllerClientOpt) (err error) { + log.V(6).Info("enabling curated package full lifecycle", "clusterName", clusterName) + defer func(err *error) { + if err != nil && *err != nil { + log.V(6).Error(*err, "enabling curated packages full lifecycle", "clusterName", clusterName) + } else { + log.V(6).Info("success enabling curated packages full lifecycle") + } + }(&err) + pc.skipWaitForPackageBundle = true + pc.clusterName = clusterName + pc.kubeConfig = kubeConfig + pc.chart = chart + pc.registryMirror = registryMirror + writer, err := filewriter.NewWriter(clusterName) + if err != nil { + return fmt.Errorf("creating file writer for helm values: %w", err) + } + options = append(options, WithValuesFileWriter(writer)) + for _, o := range options { + o(pc) + } + + return pc.Enable(ctx) +} + // NewPackageControllerClient instantiates a new instance of PackageControllerClient. -func NewPackageControllerClient(chartInstaller ChartInstaller, kubectl KubectlRunner, clusterName string, kubeConfig string, chart *releasev1.Image, registryMirror *registrymirror.RegistryMirror, options ...PackageControllerClientOpt) *PackageControllerClient { +func NewPackageControllerClient(chartInstaller ChartInstaller, deleter ChartInstallationDeleter, kubectl KubectlRunner, clusterName, kubeConfig string, chart *releasev1.Image, registryMirror *registrymirror.RegistryMirror, options ...PackageControllerClientOpt) *PackageControllerClient { pcc := &PackageControllerClient{ kubeConfig: kubeConfig, clusterName: clusterName, chart: chart, chartInstaller: chartInstaller, + deleter: deleter, kubectl: kubectl, registryMirror: registryMirror, eksaRegion: eksaDefaultRegion, @@ -73,7 +141,8 @@ func NewPackageControllerClient(chartInstaller ChartInstaller, kubectl KubectlRu return pcc } -// EnableCuratedPackages enables curated packages in a cluster +// Enable curated packages in a cluster +// // In case the cluster is management cluster, it performs the following actions: // - Installation of Package Controller through helm chart installation // - Creation of secret credentials @@ -82,7 +151,7 @@ func NewPackageControllerClient(chartInstaller ChartInstaller, kubectl KubectlRu // // In case the cluster is a workload cluster, it performs the following actions: // - Creation of package bundle controller (PBC) custom resource in management cluster -func (pc *PackageControllerClient) EnableCuratedPackages(ctx context.Context) error { +func (pc *PackageControllerClient) Enable(ctx context.Context) error { ociURI := fmt.Sprintf("%s%s", "oci://", pc.registryMirror.ReplaceRegistry(pc.chart.Image())) clusterName := fmt.Sprintf("clusterName=%s", pc.clusterName) sourceRegistry, defaultRegistry, defaultImageRegistry := pc.GetCuratedPackagesRegistries() @@ -116,11 +185,15 @@ func (pc *PackageControllerClient) EnableCuratedPackages(ctx context.Context) er chartName = chartName + "-" + pc.clusterName } - if err := pc.chartInstaller.InstallChart(ctx, chartName, ociURI, pc.chart.Tag(), pc.kubeConfig, "eksa-packages", valueFilePath, values); err != nil { + if err := pc.chartInstaller.InstallChart(ctx, chartName, ociURI, pc.chart.Tag(), pc.kubeConfig, constants.EksaPackagesName, valueFilePath, values); err != nil { return err } - return pc.waitForActiveBundle(ctx) + if !pc.skipWaitForPackageBundle { + return pc.waitForActiveBundle(ctx) + } + + return nil } // GetCuratedPackagesRegistries gets value for configurable registries from PBC. @@ -352,6 +425,24 @@ func (pc *PackageControllerClient) GetPackageControllerConfiguration() (result s return result, err } +// ReconcileDelete removes resources after a full cluster lifecycle cluster is deleted. +func (pc *PackageControllerClient) ReconcileDelete(ctx context.Context, logger logr.Logger, client client.Client, cluster *anywherev1.Cluster) error { + namespace := "eksa-packages-" + cluster.Name + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}} + if err := client.Delete(ctx, ns); err != nil { + return fmt.Errorf("deleting workload cluster curated packages namespace %q %w", namespace, err) + } + + name := "eks-anywhere-packages-" + pc.clusterName + if err := pc.deleter.Delete(ctx, pc.kubeConfig, name, constants.EksaPackagesName); err != nil { + return err + } + + logger.V(6).Info("Removed curated packages installation", "clusterName", cluster.Name) + + return nil +} + func WithEksaAccessKeyId(eksaAccessKeyId string) func(client *PackageControllerClient) { return func(config *PackageControllerClient) { config.eksaAccessKeyID = eksaAccessKeyId diff --git a/pkg/curatedpackages/packagecontrollerclient_test.go b/pkg/curatedpackages/packagecontrollerclient_test.go index 08d51eb6f6a72..c3d9472ef239e 100644 --- a/pkg/curatedpackages/packagecontrollerclient_test.go +++ b/pkg/curatedpackages/packagecontrollerclient_test.go @@ -11,8 +11,12 @@ import ( "testing" "time" + "github.com/go-logr/logr/testr" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" packagesv1 "github.com/aws/eks-anywhere-packages/api/v1alpha1" "github.com/aws/eks-anywhere/internal/test" @@ -37,6 +41,7 @@ type packageControllerTest struct { ctx context.Context kubectl *mocks.MockKubectlRunner chartInstaller *mocks.MockChartInstaller + deleter *mocks.MockChartInstallationDeleter command *curatedpackages.PackageControllerClient clusterName string kubeConfig string @@ -56,6 +61,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { ctrl := gomock.NewController(t) k := mocks.NewMockKubectlRunner(ctrl) ci := mocks.NewMockChartInstaller(ctrl) + del := mocks.NewMockChartInstallationDeleter(ctrl) kubeConfig := "kubeconfig.kubeconfig" chart := &artifactsv1.Image{ Name: "test_controller", @@ -108,7 +114,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { kubectl: k, chartInstaller: ci, command: curatedpackages.NewPackageControllerClient( - ci, k, clusterName, kubeConfig, chart, registryMirror, + ci, del, k, clusterName, kubeConfig, chart, registryMirror, curatedpackages.WithEksaSecretAccessKey(eksaAccessKey), curatedpackages.WithEksaRegion(eksaRegion), curatedpackages.WithEksaAccessKeyId(eksaAccessId), @@ -135,7 +141,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { kubectl: k, chartInstaller: ci, command: curatedpackages.NewPackageControllerClient( - ci, k, clusterName, kubeConfig, chart, + ci, del, k, clusterName, kubeConfig, chart, nil, curatedpackages.WithEksaSecretAccessKey(eksaAccessKey), curatedpackages.WithEksaRegion(eksaRegion), @@ -162,7 +168,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { kubectl: k, chartInstaller: ci, command: curatedpackages.NewPackageControllerClient( - ci, k, clusterName, kubeConfig, chartDev, + ci, del, k, clusterName, kubeConfig, chartDev, nil, curatedpackages.WithEksaSecretAccessKey(eksaAccessKey), curatedpackages.WithEksaRegion(eksaRegion), @@ -189,7 +195,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { kubectl: k, chartInstaller: ci, command: curatedpackages.NewPackageControllerClient( - ci, k, clusterName, kubeConfig, chartStaging, + ci, del, k, clusterName, kubeConfig, chartStaging, nil, curatedpackages.WithEksaSecretAccessKey(eksaAccessKey), curatedpackages.WithEksaRegion(eksaRegion), @@ -216,7 +222,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { kubectl: k, chartInstaller: ci, command: curatedpackages.NewPackageControllerClient( - ci, k, clusterName, kubeConfig, chart, registryMirrorInsecure, + ci, del, k, clusterName, kubeConfig, chart, registryMirrorInsecure, curatedpackages.WithManagementClusterName(clusterName), curatedpackages.WithValuesFileWriter(writer), ), @@ -239,7 +245,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { kubectl: k, chartInstaller: ci, command: curatedpackages.NewPackageControllerClient( - ci, k, clusterName, kubeConfig, chart, nil, + ci, del, k, clusterName, kubeConfig, chart, nil, curatedpackages.WithManagementClusterName(clusterName), curatedpackages.WithValuesFileWriter(writer), ), @@ -259,7 +265,7 @@ func newPackageControllerTests(t *testing.T) []*packageControllerTest { } } -func TestEnableCuratedPackagesSuccess(t *testing.T) { +func TestEnableSuccess(t *testing.T) { for _, tt := range newPackageControllerTests(t) { clusterName := fmt.Sprintf("clusterName=%s", "billy") valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName) @@ -276,7 +282,7 @@ func TestEnableCuratedPackagesSuccess(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -286,17 +292,17 @@ func TestEnableCuratedPackagesSuccess(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err != nil { t.Errorf("Install Controller Should succeed when installation passes") } } } -func TestEnableCuratedPackagesSucceedInWorkloadCluster(t *testing.T) { +func TestEnableSucceedInWorkloadCluster(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, tt.clusterName, tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, tt.clusterName, tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion("us-west-2"), @@ -320,7 +326,7 @@ func TestEnableCuratedPackagesSucceedInWorkloadCluster(t *testing.T) { values = append(values, "cronjob.suspend=true") } values = append(values, "workloadOnly=true") - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name+"-billy", ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name+"-billy", ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -330,7 +336,7 @@ func TestEnableCuratedPackagesSucceedInWorkloadCluster(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) tt.Expect(err).To(BeNil()) } } @@ -353,10 +359,10 @@ func getPBCFail(t *testing.T) func(context.Context, string, string, string, stri } } -func TestEnableCuratedPackagesWithProxy(t *testing.T) { +func TestEnableWithProxy(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion(tt.eksaRegion), @@ -390,7 +396,7 @@ func TestEnableCuratedPackagesWithProxy(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -400,17 +406,17 @@ func TestEnableCuratedPackagesWithProxy(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err != nil { t.Errorf("Install Controller Should succeed when installation passes") } } } -func TestEnableCuratedPackagesWithEmptyProxy(t *testing.T) { +func TestEnableWithEmptyProxy(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion(tt.eksaRegion), @@ -441,7 +447,7 @@ func TestEnableCuratedPackagesWithEmptyProxy(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -451,14 +457,14 @@ func TestEnableCuratedPackagesWithEmptyProxy(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err != nil { t.Errorf("Install Controller Should succeed when installation passes") } } } -func TestEnableCuratedPackagesFail(t *testing.T) { +func TestEnableFail(t *testing.T) { for _, tt := range newPackageControllerTests(t) { clusterName := fmt.Sprintf("clusterName=%s", "billy") valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName) @@ -475,20 +481,20 @@ func TestEnableCuratedPackagesFail(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(errors.New("login failed")) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(errors.New("login failed")) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err == nil { t.Errorf("Install Controller Should fail when installation fails") } } } -func TestEnableCuratedPackagesFailNoActiveBundle(t *testing.T) { +func TestEnableFailNoActiveBundle(t *testing.T) { for _, tt := range newPackageControllerTests(t) { clusterName := fmt.Sprintf("clusterName=%s", "billy") valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName) @@ -505,20 +511,20 @@ func TestEnableCuratedPackagesFailNoActiveBundle(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCFail(t)). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err == nil { t.Errorf("expected error, got nil") } } } -func TestEnableCuratedPackagesSuccessWhenCronJobFails(t *testing.T) { +func TestEnableSuccessWhenCronJobFails(t *testing.T) { for _, tt := range newPackageControllerTests(t) { clusterName := fmt.Sprintf("clusterName=%s", "billy") valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName) @@ -535,7 +541,7 @@ func TestEnableCuratedPackagesSuccessWhenCronJobFails(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -545,7 +551,7 @@ func TestEnableCuratedPackagesSuccessWhenCronJobFails(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err != nil { t.Errorf("Install Controller Should succeed when cron job fails") } @@ -576,10 +582,10 @@ func TestIsInstalledFalse(t *testing.T) { } } -func TestEnableCuratedPackagesActiveBundleCustomTimeout(t *testing.T) { +func TestEnableActiveBundleCustomTimeout(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion(tt.eksaRegion), @@ -608,7 +614,7 @@ func TestEnableCuratedPackagesActiveBundleCustomTimeout(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -618,14 +624,14 @@ func TestEnableCuratedPackagesActiveBundleCustomTimeout(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err != nil { t.Errorf("Install Controller Should succeed when installation passes") } } } -func TestEnableCuratedPackagesActiveBundleWaitLoops(t *testing.T) { +func TestEnableActiveBundleWaitLoops(t *testing.T) { for _, tt := range newPackageControllerTests(t) { clusterName := fmt.Sprintf("clusterName=%s", "billy") valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName) @@ -642,7 +648,7 @@ func TestEnableCuratedPackagesActiveBundleWaitLoops(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCLoops(t, 3)). @@ -652,7 +658,7 @@ func TestEnableCuratedPackagesActiveBundleWaitLoops(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) if err != nil { t.Errorf("expected no error, got %v", err) } @@ -675,10 +681,10 @@ func getPBCLoops(t *testing.T, loops int) func(context.Context, string, string, } } -func TestEnableCuratedPackagesActiveBundleTimesOut(t *testing.T) { +func TestEnableActiveBundleTimesOut(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion(tt.eksaRegion), @@ -707,13 +713,13 @@ func TestEnableCuratedPackagesActiveBundleTimesOut(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCDelay(t, time.Second)). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) expectedErr := fmt.Errorf("timed out finding an active package bundle / eksa-packages-billy namespace for the current cluster: %v", context.DeadlineExceeded) if err.Error() != expectedErr.Error() { t.Errorf("expected %v, got %v", expectedErr, err) @@ -721,10 +727,10 @@ func TestEnableCuratedPackagesActiveBundleTimesOut(t *testing.T) { } } -func TestEnableCuratedPackagesActiveBundleNamespaceTimesOut(t *testing.T) { +func TestEnableActiveBundleNamespaceTimesOut(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion(tt.eksaRegion), @@ -753,7 +759,7 @@ func TestEnableCuratedPackagesActiveBundleNamespaceTimesOut(t *testing.T) { if (tt.eksaAccessID == "" || tt.eksaAccessKey == "") && tt.registryMirror == nil { values = append(values, "cronjob.suspend=true") } - tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, "eksa-packages", valueFilePath, values).Return(nil) + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, values).Return(nil) tt.kubectl.EXPECT(). GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). DoAndReturn(getPBCSuccess(t)). @@ -763,7 +769,7 @@ func TestEnableCuratedPackagesActiveBundleNamespaceTimesOut(t *testing.T) { DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return false, nil }). AnyTimes() - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) expectedErr := fmt.Errorf("timed out finding an active package bundle / eksa-packages-billy namespace for the current cluster: %v", context.DeadlineExceeded) if err.Error() != expectedErr.Error() { t.Errorf("expected %v, got %v", expectedErr, err) @@ -810,7 +816,7 @@ func TestCreateHelmOverrideValuesYamlFail(t *testing.T) { func TestCreateHelmOverrideValuesYamlFailWithNoWriter(t *testing.T) { for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithEksaSecretAccessKey(tt.eksaAccessKey), curatedpackages.WithEksaRegion(tt.eksaRegion), @@ -823,7 +829,7 @@ func TestCreateHelmOverrideValuesYamlFailWithNoWriter(t *testing.T) { t.Setenv("REGISTRY_PASSWORD", "password") } - err := tt.command.EnableCuratedPackages(tt.ctx) + err := tt.command.Enable(tt.ctx) expectedErr := fmt.Errorf("valuesFileWriter is nil") if err.Error() != expectedErr.Error() { t.Errorf("expected %v, got %v", expectedErr, err) @@ -836,7 +842,7 @@ func TestCreateHelmOverrideValuesYamlFailWithWriteError(t *testing.T) { writer := writermocks.NewMockFileWriter(ctrl) for _, tt := range newPackageControllerTests(t) { tt.command = curatedpackages.NewPackageControllerClient( - tt.chartInstaller, tt.kubectl, "billy", tt.kubeConfig, tt.chart, + tt.chartInstaller, tt.deleter, tt.kubectl, "billy", tt.kubeConfig, tt.chart, tt.registryMirror, curatedpackages.WithValuesFileWriter(writer), ) @@ -855,7 +861,7 @@ func TestCreateHelmOverrideValuesYamlFailWithWriteError(t *testing.T) { func TestGetPackageControllerConfigurationNil(t *testing.T) { g := NewWithT(t) - sut := curatedpackages.NewPackageControllerClient(nil, nil, "billy", "", nil, nil) + sut := curatedpackages.NewPackageControllerClient(nil, nil, nil, "billy", "", nil, nil) result, err := sut.GetPackageControllerConfiguration() g.Expect(result).To(Equal("")) g.Expect(err).To(BeNil()) @@ -892,7 +898,7 @@ func TestGetPackageControllerConfigurationAll(t *testing.T) { } cluster := cluster.Spec{Config: &cluster.Config{Cluster: &v1alpha1.Cluster{Spec: clusterSpec}}} g := NewWithT(t) - sut := curatedpackages.NewPackageControllerClient(nil, nil, "billy", "", nil, nil, curatedpackages.WithClusterSpec(&cluster)) + sut := curatedpackages.NewPackageControllerClient(nil, nil, nil, "billy", "", nil, nil, curatedpackages.WithClusterSpec(&cluster)) result, err := sut.GetPackageControllerConfiguration() g.Expect(result).To(Equal(expectedAllValues)) g.Expect(err).To(BeNil()) @@ -906,7 +912,7 @@ func TestGetPackageControllerConfigurationNothing(t *testing.T) { } g := NewWithT(t) cluster := cluster.Spec{Config: &cluster.Config{Cluster: &v1alpha1.Cluster{Spec: clusterSpec}}} - sut := curatedpackages.NewPackageControllerClient(nil, nil, "billy", "", nil, nil, curatedpackages.WithClusterSpec(&cluster)) + sut := curatedpackages.NewPackageControllerClient(nil, nil, nil, "billy", "", nil, nil, curatedpackages.WithClusterSpec(&cluster)) result, err := sut.GetPackageControllerConfiguration() g.Expect(result).To(Equal("")) g.Expect(err).To(BeNil()) @@ -924,7 +930,7 @@ func TestGetCuratedPackagesRegistriesDefaultRegion(t *testing.T) { } g := NewWithT(t) cluster := cluster.Spec{Config: &cluster.Config{Cluster: &v1alpha1.Cluster{Spec: clusterSpec}}} - sut := curatedpackages.NewPackageControllerClient(nil, nil, "billy", "", chart, nil, curatedpackages.WithClusterSpec(&cluster)) + sut := curatedpackages.NewPackageControllerClient(nil, nil, nil, "billy", "", chart, nil, curatedpackages.WithClusterSpec(&cluster)) _, _, img := sut.GetCuratedPackagesRegistries() g.Expect(img).To(Equal("783794618700.dkr.ecr.us-west-2.amazonaws.com")) } @@ -941,7 +947,7 @@ func TestGetCuratedPackagesRegistriesCustomRegion(t *testing.T) { } g := NewWithT(t) cluster := cluster.Spec{Config: &cluster.Config{Cluster: &v1alpha1.Cluster{Spec: clusterSpec}}} - sut := curatedpackages.NewPackageControllerClient(nil, nil, "billy", "", chart, nil, curatedpackages.WithClusterSpec(&cluster), curatedpackages.WithEksaRegion("test")) + sut := curatedpackages.NewPackageControllerClient(nil, nil, nil, "billy", "", chart, nil, curatedpackages.WithClusterSpec(&cluster), curatedpackages.WithEksaRegion("test")) _, _, img := sut.GetCuratedPackagesRegistries() g.Expect(img).To(Equal("783794618700.dkr.ecr.test.amazonaws.com")) } @@ -957,8 +963,192 @@ func TestGetPackageControllerConfigurationError(t *testing.T) { } g := NewWithT(t) cluster := cluster.Spec{Config: &cluster.Config{Cluster: &v1alpha1.Cluster{Spec: clusterSpec}}} - sut := curatedpackages.NewPackageControllerClient(nil, nil, "billy", "", nil, nil, curatedpackages.WithClusterSpec(&cluster)) + sut := curatedpackages.NewPackageControllerClient(nil, nil, nil, "billy", "", nil, nil, curatedpackages.WithClusterSpec(&cluster)) _, err := sut.GetPackageControllerConfiguration() g.Expect(err).NotTo(BeNil()) g.Expect(err.Error()).To(Equal("invalid environment in specification ")) } + +func TestReconcileDeleteGoldenPath(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + log := testr.New(t) + + cluster := &v1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "billy"}} + kubeconfig := "test.kubeconfig" + nsName := constants.EksaPackagesName + "-" + cluster.Name + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} + client := fake.NewClientBuilder().WithRuntimeObjects(ns).Build() + ctrl := gomock.NewController(t) + deleter := mocks.NewMockChartInstallationDeleter(ctrl) + deleter.EXPECT().Delete(ctx, kubeconfig, "eks-anywhere-packages-"+cluster.Name, constants.EksaPackagesName) + + sut := curatedpackages.NewPackageControllerClient(nil, deleter, nil, "billy", kubeconfig, nil, nil) + + err := sut.ReconcileDelete(ctx, log, client, cluster) + g.Expect(err).To(BeNil()) +} + +func TestReconcileDeleteNamespaceErrorsOut(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + log := testr.New(t) + + cluster := &v1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "billy"}} + kubeconfig := "test.kubeconfig" + // There's no namespace defined in the fake client; deleting will fail + // with not found. + client := fake.NewClientBuilder().Build() + ctrl := gomock.NewController(t) + deleter := mocks.NewMockChartInstallationDeleter(ctrl) + + sut := curatedpackages.NewPackageControllerClient(nil, deleter, nil, "billy", kubeconfig, nil, nil) + + err := sut.ReconcileDelete(ctx, log, client, cluster) + g.Expect(err).Should(HaveOccurred()) + g.Expect(err.Error()).Should(ContainSubstring("\"eksa-packages-billy\" not found")) +} + +func TestReconcileDeleteHelmErrorsOut(t *testing.T) { + g := NewWithT(t) + ctx := context.Background() + log := testr.New(t) + + cluster := &v1alpha1.Cluster{ObjectMeta: metav1.ObjectMeta{Name: "billy"}} + kubeconfig := "test.kubeconfig" + nsName := constants.EksaPackagesName + "-" + cluster.Name + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: nsName}} + client := fake.NewClientBuilder().WithRuntimeObjects(ns).Build() + ctrl := gomock.NewController(t) + deleter := mocks.NewMockChartInstallationDeleter(ctrl) + // TODO this namespace should no longer be empty, following PR 5081 + testErr := fmt.Errorf("test error") + deleter.EXPECT(). + Delete(ctx, kubeconfig, "eks-anywhere-packages-"+cluster.Name, constants.EksaPackagesName). + Return(testErr) + + sut := curatedpackages.NewPackageControllerClient(nil, deleter, nil, "billy", kubeconfig, nil, nil) + + err := sut.ReconcileDelete(ctx, log, client, cluster) + g.Expect(err).Should(HaveOccurred()) + g.Expect(err.Error()).Should(Equal("test error")) +} + +func TestEnableFullLifecyclePath(t *testing.T) { + log := testr.New(t) + ctrl := gomock.NewController(t) + k := mocks.NewMockKubectlRunner(ctrl) + ci := mocks.NewMockChartInstaller(ctrl) + del := mocks.NewMockChartInstallationDeleter(ctrl) + kubeConfig := "kubeconfig.kubeconfig" + chart := &artifactsv1.Image{ + Name: "test_controller", + URI: "test_registry/eks-anywhere/eks-anywhere-packages:v1", + } + clusterName := "billy" + writer, _ := filewriter.NewWriter(clusterName) + + tt := packageControllerTest{ + WithT: NewWithT(t), + ctx: context.Background(), + kubectl: k, + chartInstaller: ci, + command: curatedpackages.NewPackageControllerClientFullLifecycle(log, ci, del, k), + clusterName: clusterName, + kubeConfig: kubeConfig, + chart: chart, + registryMirror: nil, + writer: writer, + wantValueFile: "testdata/values_empty.yaml", + } + + valueFilePath := filepath.Join("billy", filewriter.DefaultTmpFolder, valueFileName) + ociURI := fmt.Sprintf("%s%s", "oci://", tt.registryMirror.ReplaceRegistry(tt.chart.Image())) + // GetCuratedPackagesRegistries can't be used here, as when initialized + // via full cluster lifecycle the package controller client hasn't yet + // determined its chart. + values := []string{ + "clusterName=" + clusterName, + "workloadOnly=true", + "sourceRegistry=public.ecr.aws/eks-anywhere", + "defaultRegistry=public.ecr.aws/eks-anywhere", + "defaultImageRegistry=783794618700.dkr.ecr.us-west-2.amazonaws.com", + "cronjob.suspend=true", + } + + tt.chartInstaller.EXPECT().InstallChart(tt.ctx, tt.chart.Name+"-"+clusterName, ociURI, tt.chart.Tag(), tt.kubeConfig, constants.EksaPackagesName, valueFilePath, gomock.InAnyOrder(values)).Return(nil) + tt.kubectl.EXPECT(). + GetObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(getPBCSuccess(t)). + AnyTimes() + tt.kubectl.EXPECT(). + HasResource(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_, _, _, _, _ interface{}) (bool, error) { return true, nil }). + AnyTimes() + chartImage := &artifactsv1.Image{ + Name: "test_controller", + URI: "test_registry/eks-anywhere/eks-anywhere-packages:v1", + } + + err := tt.command.EnableFullLifecycle(tt.ctx, log, clusterName, kubeConfig, chartImage, tt.registryMirror, curatedpackages.WithEksaRegion("us-west-2")) + if err != nil { + t.Errorf("Install Controller Should succeed when installation passes") + } +} + +func TestGetCuratedPackagesRegistries(s *testing.T) { + s.Run("substitutes a region if set", func(t *testing.T) { + ctrl := gomock.NewController(t) + k := mocks.NewMockKubectlRunner(ctrl) + ci := mocks.NewMockChartInstaller(ctrl) + del := mocks.NewMockChartInstallationDeleter(ctrl) + kubeConfig := "kubeconfig.kubeconfig" + chart := &artifactsv1.Image{ + Name: "test_controller", + URI: "test_registry/eks-anywhere/eks-anywhere-packages:v1", + } + // eksaRegion := "test-region" + clusterName := "billy" + writer, _ := filewriter.NewWriter(clusterName) + client := curatedpackages.NewPackageControllerClient( + ci, del, k, clusterName, kubeConfig, chart, nil, + curatedpackages.WithManagementClusterName(clusterName), + curatedpackages.WithValuesFileWriter(writer), + curatedpackages.WithEksaRegion("testing"), + ) + + expected := "783794618700.dkr.ecr.testing.amazonaws.com" + _, _, got := client.GetCuratedPackagesRegistries() + + if got != expected { + t.Errorf("expected %q, got %q", expected, got) + } + }) + + s.Run("won't substitute a blank region", func(t *testing.T) { + ctrl := gomock.NewController(t) + k := mocks.NewMockKubectlRunner(ctrl) + ci := mocks.NewMockChartInstaller(ctrl) + del := mocks.NewMockChartInstallationDeleter(ctrl) + kubeConfig := "kubeconfig.kubeconfig" + chart := &artifactsv1.Image{ + Name: "test_controller", + URI: "test_registry/eks-anywhere/eks-anywhere-packages:v1", + } + // eksaRegion := "test-region" + clusterName := "billy" + writer, _ := filewriter.NewWriter(clusterName) + client := curatedpackages.NewPackageControllerClient( + ci, del, k, clusterName, kubeConfig, chart, nil, + curatedpackages.WithManagementClusterName(clusterName), + curatedpackages.WithValuesFileWriter(writer), + ) + + expected := "783794618700.dkr.ecr.us-west-2.amazonaws.com" + _, _, got := client.GetCuratedPackagesRegistries() + + if got != expected { + t.Errorf("expected %q, got %q", expected, got) + } + }) +} diff --git a/pkg/curatedpackages/packageinstaller.go b/pkg/curatedpackages/packageinstaller.go index 669eaee661d51..affcffbf9e62c 100644 --- a/pkg/curatedpackages/packageinstaller.go +++ b/pkg/curatedpackages/packageinstaller.go @@ -9,7 +9,8 @@ import ( ) type PackageController interface { - EnableCuratedPackages(ctx context.Context) error + // Enable curated packages support. + Enable(ctx context.Context) error IsInstalled(ctx context.Context) bool } @@ -68,7 +69,7 @@ func (pi *Installer) installPackagesController(ctx context.Context) error { logger.Info(" Package controller disabled") return nil } - err := pi.packageController.EnableCuratedPackages(ctx) + err := pi.packageController.Enable(ctx) if err != nil { return err } diff --git a/pkg/curatedpackages/packageinstaller_test.go b/pkg/curatedpackages/packageinstaller_test.go index 8db6140759c7b..c2a70180e372d 100644 --- a/pkg/curatedpackages/packageinstaller_test.go +++ b/pkg/curatedpackages/packageinstaller_test.go @@ -72,7 +72,7 @@ func TestPackageInstallerSuccess(t *testing.T) { tt := newPackageInstallerTest(t) tt.packageClient.EXPECT().CreatePackages(tt.ctx, tt.packagePath, tt.kubeConfigPath).Return(nil) - tt.packageControllerClient.EXPECT().EnableCuratedPackages(tt.ctx).Return(nil) + tt.packageControllerClient.EXPECT().Enable(tt.ctx).Return(nil) tt.command.InstallCuratedPackages(tt.ctx) } @@ -80,7 +80,7 @@ func TestPackageInstallerSuccess(t *testing.T) { func TestPackageInstallerFailWhenControllerFails(t *testing.T) { tt := newPackageInstallerTest(t) - tt.packageControllerClient.EXPECT().EnableCuratedPackages(tt.ctx).Return(errors.New("controller installation failed")) + tt.packageControllerClient.EXPECT().Enable(tt.ctx).Return(errors.New("controller installation failed")) tt.command.InstallCuratedPackages(tt.ctx) } @@ -89,7 +89,7 @@ func TestPackageInstallerFailWhenPackageFails(t *testing.T) { tt := newPackageInstallerTest(t) tt.packageClient.EXPECT().CreatePackages(tt.ctx, tt.packagePath, tt.kubeConfigPath).Return(errors.New("path doesn't exist")) - tt.packageControllerClient.EXPECT().EnableCuratedPackages(tt.ctx).Return(nil) + tt.packageControllerClient.EXPECT().Enable(tt.ctx).Return(nil) tt.command.InstallCuratedPackages(tt.ctx) } diff --git a/pkg/dependencies/factory.go b/pkg/dependencies/factory.go index c2feb8245f4d0..1a34d9a740f99 100644 --- a/pkg/dependencies/factory.go +++ b/pkg/dependencies/factory.go @@ -1003,6 +1003,7 @@ func (f *Factory) WithPackageControllerClient(spec *cluster.Spec, kubeConfig str return err } f.dependencies.PackageControllerClient = curatedpackages.NewPackageControllerClient( + f.dependencies.Helm, f.dependencies.Helm, f.dependencies.Kubectl, spec.Cluster.Name, diff --git a/pkg/executables/helm.go b/pkg/executables/helm.go index c2f0bd2c29462..83348ce3ab133 100644 --- a/pkg/executables/helm.go +++ b/pkg/executables/helm.go @@ -21,6 +21,8 @@ type Helm struct { registryMirror *registrymirror.RegistryMirror env map[string]string insecure bool + // skipCRDs passes the --skip-crds flag to the helm executable. + skipCRDs bool } type HelmOpt func(*Helm) @@ -32,6 +34,16 @@ func WithRegistryMirror(mirror *registrymirror.RegistryMirror) HelmOpt { } } +// WithSkipCRDs configures helm to skip the creation of CRDs when installing a +// chart. +func WithSkipCRDs() HelmOpt { + return func(h *Helm) { + h.skipCRDs = true + } +} + +// WithInsecure configures helm to skip validating TLS certificates when +// communicating with the Kubernetes API. func WithInsecure() HelmOpt { return func(h *Helm) { h.insecure = true @@ -127,6 +139,9 @@ func (h *Helm) InstallChartFromName(ctx context.Context, ociURI, kubeConfig, nam func (h *Helm) InstallChart(ctx context.Context, chart, ociURI, version, kubeconfigFilePath, namespace, valueFilePath string, values []string) error { valueArgs := GetHelmValueArgs(values) params := []string{"upgrade", "--install", chart, ociURI, "--version", version} + if h.skipCRDs { + params = append(params, "--skip-crds") + } params = append(params, valueArgs...) params = append(params, "--kubeconfig", kubeconfigFilePath) if len(namespace) > 0 { @@ -151,6 +166,25 @@ func (h *Helm) InstallChartWithValuesFile(ctx context.Context, chart, ociURI, ve return err } +// Delete removes an installation. +func (h *Helm) Delete(ctx context.Context, kubeconfigFilePath, installName, namespace string) error { + params := []string{ + "delete", installName, + "--kubeconfig", kubeconfigFilePath, + } + if namespace != "" { + params = append(params, "--namespace", namespace) + } + + params = h.addInsecureFlagIfProvided(params) + if _, err := h.executable.Command(ctx, params...).WithEnvVars(h.env).Run(); err != nil { + return fmt.Errorf("deleting helm installation %w", err) + } + logger.V(6).Info("Deleted helm installation", "name", installName, "namespace", namespace) + + return nil +} + func (h *Helm) ListCharts(ctx context.Context, kubeconfigFilePath string) ([]string, error) { params := []string{"list", "-q", "--kubeconfig", kubeconfigFilePath} out, err := h.executable.Command(ctx, params...).WithEnvVars(h.env).Run() diff --git a/pkg/executables/helm_test.go b/pkg/executables/helm_test.go index 8276fc9beeb28..eeb3451a1f762 100644 --- a/pkg/executables/helm_test.go +++ b/pkg/executables/helm_test.go @@ -4,11 +4,13 @@ import ( "bytes" "context" "errors" + "fmt" "testing" "github.com/golang/mock/gomock" . "github.com/onsi/gomega" + "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/executables" "github.com/aws/eks-anywhere/pkg/executables/mocks" "github.com/aws/eks-anywhere/pkg/registrymirror" @@ -140,6 +142,19 @@ func TestHelmSaveChartSuccessWithInsecure(t *testing.T) { tt.Expect(tt.h.SaveChart(tt.ctx, url, version, destinationFolder)).To(Succeed()) } +func TestHelmSkipCRDs(t *testing.T) { + tt := newHelmTest(t, executables.WithSkipCRDs()) + url := "url" + version := "1.1" + kubeconfig := "kubeconfig" + chart := "chart" + expectCommand( + tt.e, tt.ctx, "upgrade", "--install", chart, url, "--version", version, "--skip-crds", "--kubeconfig", kubeconfig, "--create-namespace", "--namespace", constants.EksaPackagesName, + ).withEnvVars(tt.envVars).to().Return(bytes.Buffer{}, nil) + + tt.Expect(tt.h.InstallChart(tt.ctx, chart, url, version, kubeconfig, constants.EksaPackagesName, "", nil)).To(Succeed()) +} + func TestHelmInstallChartSuccess(t *testing.T) { tt := newHelmTest(t) chart := "chart" @@ -277,3 +292,40 @@ func TestHelmListCharts(t *testing.T) { tt.Expect(result).To(Equal(expected)) }) } + +func TestHelmDelete(s *testing.T) { + kubeconfig := "/root/.kube/config" + + s.Run("Success", func(t *testing.T) { + tt := newHelmTest(s) + installName := "test-install" + expectCommand(tt.e, tt.ctx, "delete", installName, "--kubeconfig", kubeconfig).withEnvVars(tt.envVars).to().Return(bytes.Buffer{}, nil) + err := tt.h.Delete(tt.ctx, kubeconfig, installName, "") + tt.Expect(err).NotTo(HaveOccurred()) + }) + + s.Run("passes the namespace, if present", func(t *testing.T) { + tt := newHelmTest(s) + testNamespace := "testing" + installName := "test-install" + expectCommand(tt.e, tt.ctx, "delete", installName, "--kubeconfig", kubeconfig, "--namespace", testNamespace).withEnvVars(tt.envVars).to().Return(bytes.Buffer{}, nil) + err := tt.h.Delete(tt.ctx, kubeconfig, installName, testNamespace) + tt.Expect(err).NotTo(HaveOccurred()) + }) + + s.Run("passes the insecure skip flag", func(t *testing.T) { + tt := newHelmTest(t, executables.WithInsecure()) + installName := "test-install" + expectCommand(tt.e, tt.ctx, "delete", installName, "--kubeconfig", kubeconfig, "--insecure-skip-tls-verify").withEnvVars(tt.envVars).to().Return(bytes.Buffer{}, nil) + err := tt.h.Delete(tt.ctx, kubeconfig, installName, "") + tt.Expect(err).NotTo(HaveOccurred()) + }) + + s.Run("returns errors from the helm executable", func(t *testing.T) { + tt := newHelmTest(s) + installName := "test-install" + expectCommand(tt.e, tt.ctx, "delete", installName, "--kubeconfig", kubeconfig).withEnvVars(tt.envVars).to().Return(bytes.Buffer{}, fmt.Errorf("test error")) + err := tt.h.Delete(tt.ctx, kubeconfig, installName, "") + tt.Expect(err).To(HaveOccurred()) + }) +}