Curated packages PBC full cluster lifecycle integration

This change introduces the creation and deletion of a namespaced package
bundle controller for workload clusters created via the full cluster
lifecycle.

Future commits will add packages installation/deletion via full cluster
lifecycle.

Makefile

@@ -575,7 +575,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/curatedpackages/mocks/bundlemanager.go -package=mocks -source "pkg/curatedpackages/bundlemanager.go" Manager
 	${MOCKGEN} -destination=pkg/clients/kubernetes/mocks/kubectl.go -package=mocks -source "pkg/clients/kubernetes/unauth.go"
 	${MOCKGEN} -destination=pkg/clients/kubernetes/mocks/kubeconfig.go -package=mocks -source "pkg/clients/kubernetes/kubeconfig.go"
-	${MOCKGEN} -destination=pkg/curatedpackages/mocks/installer.go -package=mocks -source "pkg/curatedpackages/packagecontrollerclient.go" ChartInstaller
+	${MOCKGEN} -destination=pkg/curatedpackages/mocks/installer.go -package=mocks -source "pkg/curatedpackages/packagecontrollerclient.go" ChartInstaller ChartInstallationDeleter
 	${MOCKGEN} -destination=pkg/cluster/mocks/client_builder.go -package=mocks -source "pkg/cluster/client_builder.go"
 	${MOCKGEN} -destination=controllers/mocks/factory.go -package=mocks "github.com/aws/eks-anywhere/controllers" Manager
 	${MOCKGEN} -destination=pkg/networking/cilium/reconciler/mocks/templater.go -package=mocks -source "pkg/networking/cilium/reconciler/reconciler.go"
@@ -585,7 +585,7 @@ mocks: ## Generate mocks
 	${MOCKGEN} -destination=pkg/providers/docker/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/docker/reconciler/reconciler.go"
 	${MOCKGEN} -destination=pkg/providers/tinkerbell/reconciler/mocks/reconciler.go -package=mocks -source "pkg/providers/tinkerbell/reconciler/reconciler.go"
 	${MOCKGEN} -destination=pkg/awsiamauth/reconciler/mocks/reconciler.go -package=mocks -source "pkg/awsiamauth/reconciler/reconciler.go"
-	${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator
+	${MOCKGEN} -destination=controllers/mocks/cluster_controller.go -package=mocks -source "controllers/cluster_controller.go" AWSIamConfigReconciler ClusterValidator PackageControllerClient
 	${MOCKGEN} -destination=pkg/workflow/task_mock_test.go -package=workflow_test -source "pkg/workflow/task.go"
 	${MOCKGEN} -destination=pkg/validations/createcluster/mocks/createcluster.go -package=mocks -source "pkg/validations/createcluster/createcluster.go"
 	${MOCKGEN} -destination=pkg/awsiamauth/mock_test.go -package=awsiamauth_test -source "pkg/awsiamauth/installer.go"

cmd/eksctl-anywhere/cmd/installpackagecontroller.go

@@ -72,7 +72,7 @@ func installPackageController(ctx context.Context) error {
 	}
 
 	curatedpackages.PrintLicense()
-	err = ctrlClient.EnableCuratedPackages(ctx)
+	err = ctrlClient.Enable(ctx)
 	if err != nil {
 		return err
 	}

config/manifest/eksa-components.yaml

@@ -5919,6 +5919,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - packages.eks.amazonaws.com
+  resources:
+  - packagebundlecontrollers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - tinkerbell.org
   resources:

config/rbac/role.yaml

@@ -210,6 +210,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - packages.eks.amazonaws.com
+  resources:
+  - packagebundlecontrollers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - tinkerbell.org
   resources:

controllers/cluster_controller.go

@@ -3,10 +3,13 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -27,7 +30,10 @@ import (
 	"github.com/aws/eks-anywhere/pkg/controller/clientutil"
 	"github.com/aws/eks-anywhere/pkg/controller/clusters"
 	"github.com/aws/eks-anywhere/pkg/controller/handlers"
+	"github.com/aws/eks-anywhere/pkg/curatedpackages"
+	"github.com/aws/eks-anywhere/pkg/registrymirror"
 	"github.com/aws/eks-anywhere/pkg/utils/ptr"
+	"github.com/aws/eks-anywhere/release/api/v1alpha1"
 )
 
 const (
@@ -42,6 +48,7 @@ type ClusterReconciler struct {
 	providerReconcilerRegistry ProviderClusterReconcilerRegistry
 	awsIamAuth                 AWSIamConfigReconciler
 	clusterValidator           ClusterValidator
+	packagesClient             PackagesClient
 }
 
 type ProviderClusterReconcilerRegistry interface {
@@ -61,12 +68,13 @@ type ClusterValidator interface {
 }
 
 // NewClusterReconciler constructs a new ClusterReconciler.
-func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator) *ClusterReconciler {
+func NewClusterReconciler(client client.Client, registry ProviderClusterReconcilerRegistry, awsIamAuth AWSIamConfigReconciler, clusterValidator ClusterValidator, pkgs PackagesClient) *ClusterReconciler {
 	return &ClusterReconciler{
 		client:                     client,
 		providerReconcilerRegistry: registry,
 		awsIamAuth:                 awsIamAuth,
 		clusterValidator:           clusterValidator,
+		packagesClient:             pkgs,
 	}
 }
 
@@ -262,9 +270,135 @@ func (r *ClusterReconciler) postClusterProviderReconcile(ctx context.Context, lo
 		}
 	}
 
+	if cluster.IsPackagesEnabled() {
+		if err := r.postReconcilePackagesForWorkloadCluster(ctx, log, cluster); err != nil {
+			return controller.Result{}, err
+		}
+	}
+
 	return controller.Result{}, nil
 }
 
+// PackagesClient handles curated packages operations from within the cluster
+// controller.
+type PackagesClient interface {
+	EnableFullLifecycle(context.Context, logr.Logger, string, string, *v1alpha1.Image, *registrymirror.RegistryMirror, ...curatedpackages.PackageControllerClientOpt) error
+	ReconcileDelete(context.Context, logr.Logger, client.Client, *anywherev1.Cluster) error
+}
+
+func (r *ClusterReconciler) postReconcilePackagesForWorkloadCluster(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (err error) {
+	image, err := r.getBundleFromCluster(ctx, cluster)
+	if err != nil {
+		return err
+	}
+
+	kubeConfig, err := r.writeKubeConfigSecret(ctx, cluster)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := kubeConfig.Remove(); err != nil {
+			log.Error(err, "removing kubeconfig file", "filename", kubeConfig.Name())
+		}
+	}()
+
+	registry := registrymirror.FromCluster(cluster)
+
+	if err := r.packagesClient.EnableFullLifecycle(ctx, log, cluster.Name, kubeConfig.Name(), image, registry); err != nil {
+		return fmt.Errorf("packages client error: %w", err)
+	}
+
+	log.V(6).Info("Installed curated packages on workload cluster", "cluster", cluster.Name)
+
+	return nil
+}
+
+// getBundleFromCluster based on the cluster's k8s version.
+func (r *ClusterReconciler) getBundleFromCluster(ctx context.Context, cluster *anywherev1.Cluster) (*v1alpha1.Image, error) {
+	bundles := &v1alpha1.Bundles{}
+	nn := types.NamespacedName{
+		Name:      cluster.Spec.BundlesRef.Name,
+		Namespace: cluster.Spec.BundlesRef.Namespace,
+	}
+	if err := r.client.Get(ctx, nn, bundles); err != nil {
+		return nil, fmt.Errorf("retrieving bundle: %w", err)
+	}
+
+	verBundle, err := r.findMatchingBundle(bundles, string(cluster.Spec.KubernetesVersion))
+	if err != nil {
+		return nil, err
+	}
+
+	return &verBundle.PackageController.HelmChart, nil
+}
+
+func (r *ClusterReconciler) findMatchingBundle(bundles *v1alpha1.Bundles, kubeVersion string) (*v1alpha1.VersionsBundle, error) {
+	var verBundle *v1alpha1.VersionsBundle
+	for _, b := range bundles.Spec.VersionsBundles {
+		if b.KubeVersion == string(kubeVersion) {
+			verBundle = &b
+			break
+		}
+	}
+	if verBundle == nil {
+		return nil, fmt.Errorf("no bundle for kube version %q", kubeVersion)
+	}
+
+	return verBundle, nil
+}
+
+// writeKubeConfigSecret so that the helm executable can pick it up to install
+// curated packages helm charts.
+func (r *ClusterReconciler) writeKubeConfigSecret(ctx context.Context, cluster *anywherev1.Cluster) (kubeconfigFile, error) {
+	kubeConfigSecret := &corev1.Secret{}
+	kubeConfigNN := types.NamespacedName{
+		Namespace: constants.EksaSystemNamespace,
+		Name:      cluster.Name + "-kubeconfig",
+	}
+	if err := r.client.Get(ctx, kubeConfigNN, kubeConfigSecret); err != nil {
+		return nil, fmt.Errorf("getting kubeconfig secret: %w", err)
+	}
+	secretBytes, err := yaml.Marshal(kubeConfigSecret)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling secret %w", err)
+	}
+	f, err := os.CreateTemp("", "kubeconfig-*.yaml")
+	if err != nil {
+		return nil, fmt.Errorf("opening kubeconfig file %w", err)
+	}
+	defer f.Close()
+	if _, err := f.Write(secretBytes); err != nil {
+		return nil, fmt.Errorf("writing kubeconfig file %w", err)
+	}
+
+	return &removeableFile{f}, nil
+}
+
+// kubeconfigFile adds utility operations to delete a filename after its no
+// longer needed.
+type kubeconfigFile interface {
+	// Name is the file's name on disk.
+	Name() string
+	// Remove is a best-effort deletion of the file on disk.
+	Remove() error
+}
+
+// removeableFile wraps os.File to implement kubeconfigFile.
+type removeableFile struct {
+	*os.File
+}
+
+// Remove the file from disk.
+func (f *removeableFile) Remove() error {
+	f.Close()
+	if err := os.Remove(f.Name()); err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("removing removeableFile: %w", err)
+		}
+	}
+	return nil
+}
+
 func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger, cluster *anywherev1.Cluster) (ctrl.Result, error) {
 	if cluster.IsSelfManaged() {
 		return ctrl.Result{}, errors.New("deleting self-managed clusters is not supported")
@@ -310,6 +444,12 @@ func (r *ClusterReconciler) reconcileDelete(ctx context.Context, log logr.Logger
 		}
 	}
 
+	if !cluster.IsSelfManaged() {
+		if err := r.packagesClient.ReconcileDelete(ctx, log, r.client, cluster); err != nil {
+			return ctrl.Result{}, fmt.Errorf("deleting packages for cluster %q: %w", cluster.Name, err)
+		}
+	}
+
 	return ctrl.Result{}, nil
 }
 

controllers/cluster_controller_legacy.go

@@ -60,6 +60,7 @@ func NewClusterReconcilerLegacy(client client.Client, log logr.Logger, scheme *r
 // +kubebuilder:rbac:groups="",namespace=eksa-system,resources=secrets,verbs=delete;
 // +kubebuilder:rbac:groups=tinkerbell.org,resources=hardware;hardware/status,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=machines;machines/status,verbs=get;list;watch
+// +kubebuilder:rbac:groups=packages.eks.amazonaws.com,resources=packagebundlecontrollers,verbs=get;list;watch;create;update;patch;delete
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.