Skip to content

Commit

Permalink
Use Cluster Mover in create and delete workflows (#7909)
Browse files Browse the repository at this point in the history 
* Use Cluster Mover in create and delete workflows

* update tests and mocks

* additional test coverage
  • Loading branch information
@tatlat
tatlat authored Apr 4, 2024
1 parent 96d504b commit 4328ced
Show file tree
Hide file tree
Showing 18 changed files with 473 additions and 174 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ mocks: ## Generate mocks
${MOCKGEN} -destination=pkg/bootstrapper/mocks/bootstrapper.go -package=mocks "github.com/aws/eks-anywhere/pkg/bootstrapper" ClusterClient
${MOCKGEN} -destination=pkg/git/providers/github/mocks/github.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/providers/github" GithubClient
${MOCKGEN} -destination=pkg/git/mocks/git.go -package=mocks "github.com/aws/eks-anywhere/pkg/git" Client,ProviderClient
${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter
${MOCKGEN} -destination=pkg/workflows/interfaces/mocks/clients.go -package=mocks "github.com/aws/eks-anywhere/pkg/workflows/interfaces" Bootstrapper,ClusterManager,GitOpsManager,Validator,CAPIManager,EksdInstaller,EksdUpgrader,PackageInstaller,ClusterUpgrader,ClusterCreator,ClientFactory,EksaInstaller,ClusterDeleter,ClusterMover
${MOCKGEN} -destination=pkg/git/gogithub/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gogithub" Client
${MOCKGEN} -destination=pkg/git/gitclient/mocks/client.go -package=mocks "github.com/aws/eks-anywhere/pkg/git/gitclient" GoGit
${MOCKGEN} -destination=pkg/validations/mocks/docker.go -package=mocks "github.com/aws/eks-anywhere/pkg/validations" DockerExecutable
Expand Down
4 changes: 3 additions & 1 deletion cmd/eksctl-anywhere/cmd/createcluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,8 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
WithCreateClusterDefaulter(createCLIConfig).
WithClusterApplier().
WithKubeconfigWriter(clusterSpec.Cluster).
WithClusterCreator(clusterSpec.Cluster)
WithClusterCreator(clusterSpec.Cluster).
WithClusterMover()

if cc.timeoutOptions.noTimeouts {
factory.WithNoTimeouts()
Expand Down Expand Up @@ -274,6 +275,7 @@ func (cc *createClusterOptions) createCluster(cmd *cobra.Command, _ []string) er
deps.PackageInstaller,
deps.ClusterCreator,
deps.EksaInstaller,
deps.ClusterMover,
)

err = createMgmtCluster.Run(ctx, clusterSpec, createValidations)
Expand Down
3 changes: 2 additions & 1 deletion cmd/eksctl-anywhere/cmd/deletecluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
WithEksdInstaller().
WithEKSAInstaller().
WithUnAuthKubeClient().
WithClusterMover().
Build(ctx)
if err != nil {
return err
Expand Down Expand Up @@ -154,7 +155,7 @@ func (dc *deleteClusterOptions) deleteCluster(ctx context.Context) error {
deleteWorkload := workload.NewDelete(deps.Provider, deps.Writer, deps.ClusterManager, deps.ClusterDeleter, deps.GitOpsFlux)
err = deleteWorkload.Run(ctx, cluster, clusterSpec)
} else {
deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient)
deleteManagement := management.NewDelete(deps.Bootstrapper, deps.Provider, deps.Writer, deps.ClusterManager, deps.GitOpsFlux, deps.ClusterDeleter, deps.EksdInstaller, deps.EksaInstaller, deps.UnAuthKubeClient, deps.ClusterMover)
err = deleteManagement.Run(ctx, cluster, clusterSpec)
}
cleanup(deps, &err)
Expand Down
82 changes: 82 additions & 0 deletions pkg/clustermanager/cluster_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,73 @@ func (c *ClusterManager) PauseCAPIWorkloadClusters(ctx context.Context, manageme
return nil
}

func (c *ClusterManager) resumeEksaReconcileForManagementAndWorkloadClusters(ctx context.Context, managementCluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
clusters := &v1alpha1.ClusterList{}
err := c.clusterClient.ListObjects(ctx, eksaClusterResourceType, clusterSpec.Cluster.Namespace, managementCluster.KubeconfigFile, clusters)
if err != nil {
return err
}

for _, w := range clusters.Items {
if w.ManagedBy() != clusterSpec.Cluster.Name {
continue
}

if err := c.resumeReconcileForCluster(ctx, managementCluster, &w, provider); err != nil {
return err
}
}

return nil
}

// ResumeEKSAControllerReconcile resumes a paused EKS-Anywhere cluster.
func (c *ClusterManager) ResumeEKSAControllerReconcile(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
// clear pause annotation
clusterSpec.Cluster.ClearPauseAnnotation()
provider.DatacenterConfig(clusterSpec).ClearPauseAnnotation()

if clusterSpec.Cluster.IsSelfManaged() {
return c.resumeEksaReconcileForManagementAndWorkloadClusters(ctx, cluster, clusterSpec, provider)
}

return c.resumeReconcileForCluster(ctx, cluster, clusterSpec.Cluster, provider)
}

func (c *ClusterManager) resumeReconcileForCluster(ctx context.Context, clusterCreds *types.Cluster, cluster *v1alpha1.Cluster, provider providers.Provider) error {
pausedAnnotation := cluster.PausedAnnotation()
err := c.clusterClient.RemoveAnnotationInNamespace(ctx, provider.DatacenterResourceType(), cluster.Spec.DatacenterRef.Name, pausedAnnotation, clusterCreds, cluster.Namespace)
if err != nil {
return fmt.Errorf("removing paused annotation when resuming datacenterconfig reconciliation: %v", err)
}

if provider.MachineResourceType() != "" {
for _, machineConfigRef := range cluster.MachineConfigRefs() {
err = c.clusterClient.RemoveAnnotationInNamespace(ctx, provider.MachineResourceType(), machineConfigRef.Name, pausedAnnotation, clusterCreds, cluster.Namespace)
if err != nil {
return fmt.Errorf("removing paused annotation when resuming reconciliation for machine config %s: %v", machineConfigRef.Name, err)
}
}
}

err = c.clusterClient.RemoveAnnotationInNamespace(ctx, cluster.ResourceType(), cluster.Name, pausedAnnotation, clusterCreds, cluster.Namespace)
if err != nil {
return fmt.Errorf("removing paused annotation when resuming cluster reconciliation: %v", err)
}

if err = c.clusterClient.RemoveAnnotationInNamespace(ctx,
cluster.ResourceType(),
cluster.Name,
v1alpha1.ManagedByCLIAnnotation,
clusterCreds,
cluster.Namespace,
); err != nil {
return fmt.Errorf("removing managed by CLI annotation when resuming cluster reconciliation: %v", err)
}

return nil
}

// ResumeCAPIWorkloadClusters resumes all workload CAPI clusters except the management cluster.
func (c *ClusterManager) ResumeCAPIWorkloadClusters(ctx context.Context, managementCluster *types.Cluster) error {
clusters, err := c.clusterClient.GetClusters(ctx, managementCluster)
Expand All @@ -693,6 +760,21 @@ func (c *ClusterManager) ResumeCAPIWorkloadClusters(ctx context.Context, managem
return nil
}

// AllowDeleteWhilePaused allows the deletion of paused clusters.
func (c *ClusterManager) AllowDeleteWhilePaused(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec) error {
return c.allowDeleteWhilePaused(ctx, cluster, clusterSpec.Cluster)
}

func (c *ClusterManager) allowDeleteWhilePaused(ctx context.Context, clusterCreds *types.Cluster, cluster *v1alpha1.Cluster) error {
allowDelete := map[string]string{v1alpha1.AllowDeleteWhenPausedAnnotation: "true"}

if err := c.clusterClient.UpdateAnnotationInNamespace(ctx, cluster.ResourceType(), cluster.Name, allowDelete, clusterCreds, cluster.Namespace); err != nil {
return fmt.Errorf("updating paused annotation in cluster reconciliation: %v", err)
}

return nil
}

func (c *ClusterManager) PauseEKSAControllerReconcile(ctx context.Context, cluster *types.Cluster, clusterSpec *cluster.Spec, provider providers.Provider) error {
if clusterSpec.Cluster.IsSelfManaged() {
return c.pauseEksaReconcileForManagementAndWorkloadClusters(ctx, cluster, clusterSpec, provider)
Expand Down
97 changes: 78 additions & 19 deletions pkg/clustermanager/cluster_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,7 +771,7 @@ func TestPauseEKSAControllerReconcileWorkloadCluster(t *testing.T) {
tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).To(Succeed())
}

func TestPauseEKSAControllerReconcileWorkloadClusterUpdateAnnotationError(t *testing.T) {
func TestResumeEKSAControllerReconcileWorkloadClusterUpdateAnnotationError(t *testing.T) {
tt := newTest(t, clustermanager.WithRetrier(retrier.NewWithMaxRetries(1, 0)))
tt.clusterSpec.Cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -788,15 +788,26 @@ func TestPauseEKSAControllerReconcileWorkloadClusterUpdateAnnotationError(t *tes
},
}

datacenterConfig := &v1alpha1.VSphereDatacenterConfig{
ObjectMeta: metav1.ObjectMeta{
Name: tt.clusterName,
},
Spec: v1alpha1.VSphereDatacenterConfigSpec{
Insecure: true,
},
}
pauseAnnotation := "anywhere.eks.amazonaws.com/paused"

tt.mocks.provider.EXPECT().DatacenterResourceType().Return(eksaVSphereDatacenterResourceType)
tt.mocks.provider.EXPECT().MachineResourceType().Return("")
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, expectedPauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, expectedPauseAnnotation, tt.cluster, "").Return(errors.New("pause eksa cluster error"))
tt.mocks.provider.EXPECT().DatacenterConfig(tt.clusterSpec).Return(datacenterConfig)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, pauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, pauseAnnotation, tt.cluster, "").Return(errors.New("pause eksa cluster error"))

tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
tt.Expect(tt.clusterManager.ResumeEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
}

func TestPauseEKSAControllerReconcileManagementCluster(t *testing.T) {
func TestResumeEKSAControllerReconcileManagementCluster(t *testing.T) {
tt := newTest(t)
tt.clusterSpec.Cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -813,6 +824,18 @@ func TestPauseEKSAControllerReconcileManagementCluster(t *testing.T) {
},
}

tt.clusterSpec.Cluster.PauseReconcile()

datacenterConfig := &v1alpha1.VSphereDatacenterConfig{
ObjectMeta: metav1.ObjectMeta{
Name: tt.clusterName,
},
Spec: v1alpha1.VSphereDatacenterConfigSpec{
Insecure: true,
},
}
pauseAnnotation := "anywhere.eks.amazonaws.com/paused"

tt.mocks.client.EXPECT().
ListObjects(tt.ctx, eksaClusterResourceType, "", "", &v1alpha1.ClusterList{}).
DoAndReturn(func(_ context.Context, _, _, _ string, obj *v1alpha1.ClusterList) error {
Expand Down Expand Up @@ -851,34 +874,31 @@ func TestPauseEKSAControllerReconcileManagementCluster(t *testing.T) {
})
tt.mocks.provider.EXPECT().DatacenterResourceType().Return(eksaVSphereDatacenterResourceType).Times(2)
tt.mocks.provider.EXPECT().MachineResourceType().Return("").Times(2)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, expectedPauseAnnotation, tt.cluster, "").Return(nil).Times(2)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, expectedPauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(
tt.mocks.provider.EXPECT().DatacenterConfig(tt.clusterSpec).Return(datacenterConfig)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaVSphereDatacenterResourceType, tt.clusterSpec.Cluster.Spec.DatacenterRef.Name, pauseAnnotation, tt.cluster, "").Return(nil).Times(2)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaClusterResourceType, tt.clusterSpec.Cluster.Name, pauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(
tt.ctx,
eksaClusterResourceType,
tt.clusterSpec.Cluster.Name,
map[string]string{
v1alpha1.ManagedByCLIAnnotation: "true",
},
v1alpha1.ManagedByCLIAnnotation,
tt.cluster,
"",
).Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, eksaClusterResourceType, "workload-cluster-1", expectedPauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(tt.ctx, eksaClusterResourceType, "workload-cluster-1", pauseAnnotation, tt.cluster, "").Return(nil)
tt.mocks.client.EXPECT().RemoveAnnotationInNamespace(
tt.ctx,
eksaClusterResourceType,
"workload-cluster-1",
map[string]string{
v1alpha1.ManagedByCLIAnnotation: "true",
},
v1alpha1.ManagedByCLIAnnotation,
tt.cluster,
"",
).Return(nil)

tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).To(Succeed())
tt.Expect(tt.clusterManager.ResumeEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).To(Succeed())
}

func TestPauseEKSAControllerReconcileManagementClusterListObjectsError(t *testing.T) {
func TestResumeEKSAControllerReconcileManagementClusterListObjectsError(t *testing.T) {
tt := newTest(t, clustermanager.WithRetrier(retrier.NewWithMaxRetries(1, 0)))
tt.clusterSpec.Cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -891,9 +911,20 @@ func TestPauseEKSAControllerReconcileManagementClusterListObjectsError(t *testin
},
}

datacenterConfig := &v1alpha1.VSphereDatacenterConfig{
ObjectMeta: metav1.ObjectMeta{
Name: tt.clusterName,
},
Spec: v1alpha1.VSphereDatacenterConfigSpec{
Insecure: true,
},
}

tt.mocks.provider.EXPECT().DatacenterConfig(tt.clusterSpec).Return(datacenterConfig)

tt.mocks.client.EXPECT().ListObjects(tt.ctx, eksaClusterResourceType, "", "", &v1alpha1.ClusterList{}).Return(errors.New("list error"))

tt.Expect(tt.clusterManager.PauseEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
tt.Expect(tt.clusterManager.ResumeEKSAControllerReconcile(tt.ctx, tt.cluster, tt.clusterSpec, tt.mocks.provider)).NotTo(Succeed())
}

func TestPauseEKSAControllerReconcileWorkloadClusterWithMachineConfig(t *testing.T) {
Expand Down Expand Up @@ -1084,3 +1115,31 @@ func TestCreateRegistryCredSecretSuccess(t *testing.T) {
err := tt.clusterManager.CreateRegistryCredSecret(tt.ctx, tt.cluster)
tt.Expect(err).To(BeNil())
}

func TestAllowDeleteWhilePaused(t *testing.T) {
tests := []struct {
name string
err error
}{
{
name: "success allow delete while paused",
err: nil,
},
{
name: "fail allow delete while paused",
err: fmt.Errorf("failure"),
},
}
allowDelete := map[string]string{v1alpha1.AllowDeleteWhenPausedAnnotation: "true"}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
tt := newTest(t)
cluster := tt.clusterSpec.Cluster
tt.mocks.client.EXPECT().UpdateAnnotationInNamespace(tt.ctx, cluster.ResourceType(), cluster.Name, allowDelete, tt.cluster, cluster.Namespace).Return(test.err)
err := tt.clusterManager.AllowDeleteWhilePaused(tt.ctx, tt.cluster, tt.clusterSpec)
expectedErr := fmt.Errorf("updating paused annotation in cluster reconciliation: %v", test.err)
tt.Expect(err).To(Or(BeNil(), MatchError(expectedErr)))
})
}
}
1 change: 1 addition & 0 deletions pkg/task/task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ type CommandContext struct {
OriginalError error
BackupClusterStateDir string
ForceCleanup bool
ClusterMover interfaces.ClusterMover
}

func (c *CommandContext) SetError(err error) {
Expand Down
17 changes: 3 additions & 14 deletions pkg/workflows/create_prep.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,11 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/aws/eks-anywhere/pkg/workflows/interfaces"
"github.com/aws/eks-anywhere/pkg/clients/kubernetes"
)

// CreateNamespaceIfNotPresent creates the namespace on the cluster if it does not already exist.
func CreateNamespaceIfNotPresent(ctx context.Context, namespace, kubeconfig string, clientFactory interfaces.ClientFactory) error {
client, err := clientFactory.BuildClientFromKubeconfig(kubeconfig)
if err != nil {
return err
}

if err := client.Get(ctx, namespace, "", &corev1.Namespace{}); err != nil && !errors.IsNotFound(err) {
return err
} else if err == nil {
return nil
}

func CreateNamespaceIfNotPresent(ctx context.Context, namespace string, client kubernetes.Client) error {
ns := &corev1.Namespace{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Expand All @@ -33,7 +22,7 @@ func CreateNamespaceIfNotPresent(ctx context.Context, namespace, kubeconfig stri
},
}

if err = client.Create(ctx, ns); err != nil {
if err := client.Create(ctx, ns); err != nil && !errors.IsAlreadyExists(err) {
return err
}

Expand Down
Loading

0 comments on commit 4328ced

Please sign in to comment.