From 5dba34914b40d74474f70339091837f430f68656 Mon Sep 17 00:00:00 2001 From: Alexander Dejanovski Date: Mon, 6 May 2024 15:00:04 +0200 Subject: [PATCH] Remove the medusa standalone pod (#1304) * Remove the medusa standalone pod * address review comments --- CHANGELOG/CHANGELOG-1.16.md | 3 +- .../k8ssandracluster_controller_test.go | 7 -- controllers/k8ssandra/medusa_reconciler.go | 59 +---------- .../k8ssandra/medusa_reconciler_test.go | 97 ------------------- controllers/medusa/controllers_test.go | 11 ++- .../medusa/medusabackupjob_controller_test.go | 44 --------- .../medusa/medusarestorejob_controller.go | 45 +++++---- .../medusarestorejob_controller_test.go | 59 ++++++++--- .../medusa/medusatask_controller_test.go | 1 - pkg/medusa/reconcile.go | 82 ---------------- pkg/medusa/reconcile_test.go | 36 ------- test/e2e/medusa_test.go | 28 ------ 12 files changed, 82 insertions(+), 390 deletions(-) diff --git a/CHANGELOG/CHANGELOG-1.16.md b/CHANGELOG/CHANGELOG-1.16.md index 2db975a04..c3acc17bf 100644 --- a/CHANGELOG/CHANGELOG-1.16.md +++ b/CHANGELOG/CHANGELOG-1.16.md @@ -15,4 +15,5 @@ When cutting a new release, update the `unreleased` heading to the tag being gen ## unreleased -* [BUGFIX] [#1272](https://github.com/k8ssandra/k8ssandra-operator/issues/1272) Prevent cass-operator from creating users when an external DC is referenced to allow migration through expansion \ No newline at end of file +* [BUGFIX] [#1272](https://github.com/k8ssandra/k8ssandra-operator/issues/1272) Prevent cass-operator from creating users when an external DC is referenced to allow migration through expansion +* [ENHANCEMENT] [#1066](https://github.com/k8ssandra/k8ssandra-operator/issues/1066) Remove the medusa standalone pod as it is not needed anymore \ No newline at end of file diff --git a/controllers/k8ssandra/k8ssandracluster_controller_test.go b/controllers/k8ssandra/k8ssandracluster_controller_test.go index 2e622a4dd..f382a2083 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller_test.go +++ b/controllers/k8ssandra/k8ssandracluster_controller_test.go @@ -14,7 +14,6 @@ import ( "github.com/k8ssandra/k8ssandra-operator/pkg/encryption" "github.com/k8ssandra/k8ssandra-operator/pkg/images" "github.com/k8ssandra/k8ssandra-operator/pkg/labels" - medusa "github.com/k8ssandra/k8ssandra-operator/pkg/medusa" "github.com/k8ssandra/k8ssandra-operator/pkg/utils" promapi "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -1582,8 +1581,6 @@ func applyClusterWithEncryptionOptions(t *testing.T, ctx context.Context, f *fra verifySystemReplicationAnnotationSet(ctx, t, f, kc) - medusaKey := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), "dc1")}, K8sContext: f.DataPlaneContexts[0]} - require.NoError(f.SetMedusaDeplAvailable(ctx, medusaKey)) t.Log("check that dc1 was created") dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -1956,8 +1953,6 @@ func applyClusterWithEncryptionOptionsExternalSecrets(t *testing.T, ctx context. require.NoError(err, "failed to create K8ssandraCluster") verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) - medusaKey := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), "dc1")}, K8sContext: f.DataPlaneContexts[0]} - require.NoError(f.SetMedusaDeplAvailable(ctx, medusaKey)) t.Log("check that dc1 was created") dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -2487,8 +2482,6 @@ func injectContainersAndVolumes(t *testing.T, ctx context.Context, f *framework. verifySystemReplicationAnnotationSet(ctx, t, f, kc) - // Create a Medusa deployment object and simulate it being available to make the k8c reconcile progress. - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc1", f.DataPlaneContexts[0]) t.Log("check that dc1 was never created") dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) diff --git a/controllers/k8ssandra/medusa_reconciler.go b/controllers/k8ssandra/medusa_reconciler.go index b23bb0cc0..9cc35c1bb 100644 --- a/controllers/k8ssandra/medusa_reconciler.go +++ b/controllers/k8ssandra/medusa_reconciler.go @@ -18,7 +18,6 @@ import ( "github.com/k8ssandra/k8ssandra-operator/pkg/result" "github.com/k8ssandra/k8ssandra-operator/pkg/secret" "github.com/k8ssandra/k8ssandra-operator/pkg/utils" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -85,14 +84,6 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( cassandra.AddOrUpdateVolume(dcConfig, volume.Volume, volume.VolumeIndex, volume.Exists) } - // Create the Medusa standalone pod - desiredMedusaStandalone := medusa.StandaloneMedusaDeployment(*medusaContainer, kc.SanitizedName(), dcConfig.SanitizedName(), dcNamespace, logger, kc.Spec.Medusa.ContainerImage) - - // Add the volumes previously computed to the Medusa standalone pod - for _, volume := range volumes { - cassandra.AddOrUpdateVolumeToSpec(&desiredMedusaStandalone.Spec.Template, volume.Volume, volume.VolumeIndex, volume.Exists) - } - if !kc.Spec.UseExternalSecrets() { cassandraUserSecretName := medusa.CassandraUserSecretName(medusaSpec, kc.SanitizedName()) cassandra.AddCqlUser(medusaSpec.CassandraUserSecretRef, dcConfig, cassandraUserSecretName) @@ -105,38 +96,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( } } - // Reconcile the Medusa standalone deployment kcKey := utils.GetKey(kc) - desiredMedusaStandalone.SetLabels(labels.CleanedUpByLabels(kcKey)) - recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *desiredMedusaStandalone) - switch { - case recRes.IsError(): - return recRes - case recRes.IsRequeue(): - return recRes - } - - // Create and reconcile the Medusa service for the standalone deployment - medusaService := medusa.StandaloneMedusaService(dcConfig, medusaSpec, kc.SanitizedName(), dcNamespace, logger) - medusaService.SetLabels(labels.CleanedUpByLabels(kcKey)) - recRes = reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *medusaService) - switch { - case recRes.IsError(): - return recRes - case recRes.IsRequeue(): - return recRes - } - - // Check if the Medusa Standalone deployment is ready, and requeue if not - ready, err := r.isMedusaStandaloneReady(ctx, remoteClient, desiredMedusaStandalone) - if err != nil { - logger.Info("Failed to check if Medusa standalone deployment is ready", "error", err) - return result.Error(err) - } - if !ready { - logger.Info("Medusa standalone deployment is not ready yet") - return result.RequeueSoon(r.DefaultDelay) - } // Create a cron job to purge Medusa backups operatorNamespace := r.getOperatorNamespace() purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger) @@ -145,7 +105,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( return result.Error(err) } purgeCronJob.SetLabels(labels.CleanedUpByLabels(kcKey)) - recRes = reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob) + recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob) switch { case recRes.IsError(): return recRes @@ -160,23 +120,6 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( return result.Continue() } -// Check if the Medusa standalone deployment is ready -func (r *K8ssandraClusterReconciler) isMedusaStandaloneReady(ctx context.Context, remoteClient client.Client, desiredMedusaStandalone *appsv1.Deployment) (bool, error) { - // Get the medusa standalone deployment and check the rollout status - deplKey := utils.GetKey(desiredMedusaStandalone) - medusaStandalone := &appsv1.Deployment{} - if err := remoteClient.Get(context.Background(), deplKey, medusaStandalone); err != nil { - return false, err - } - // Check the conditions to see if the deployment has successfully rolled out - for _, c := range medusaStandalone.Status.Conditions { - if c.Type == appsv1.DeploymentAvailable { - return c.Status == corev1.ConditionTrue, nil // deployment is available - } - } - return false, nil // deployment condition not found -} - // Generate a secret for Medusa or use the existing one if provided in the spec func (r *K8ssandraClusterReconciler) reconcileMedusaSecrets( ctx context.Context, diff --git a/controllers/k8ssandra/medusa_reconciler_test.go b/controllers/k8ssandra/medusa_reconciler_test.go index 25740587a..cdfc5ade5 100644 --- a/controllers/k8ssandra/medusa_reconciler_test.go +++ b/controllers/k8ssandra/medusa_reconciler_test.go @@ -11,12 +11,10 @@ import ( medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1" cassandra "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "github.com/k8ssandra/k8ssandra-operator/pkg/images" - medusa "github.com/k8ssandra/k8ssandra-operator/pkg/medusa" "github.com/k8ssandra/k8ssandra-operator/pkg/utils" "github.com/k8ssandra/k8ssandra-operator/test/framework" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -167,7 +165,6 @@ func createMultiDcClusterWithMedusa(t *testing.T, ctx context.Context, f *framew require.NoError(err, "failed to create K8ssandraCluster") verifyReplicatedSecretReconciled(ctx, t, f, kc) - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc1", f.DataPlaneContexts[0]) t.Log("check that dc1 was created") dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -176,28 +173,6 @@ func createMultiDcClusterWithMedusa(t *testing.T, ctx context.Context, f *framew defaultPrefix := kc.Spec.Medusa.StorageProperties.Prefix verifyConfigMap(require, ctx, f, namespace, defaultPrefix, defaultConcurrentTransfers) - t.Log("check that the standalone Medusa deployment was created in dc1") - medusaDeploymentKey1 := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusa.MedusaStandaloneDeploymentName(k8ssandraClusterName, "dc1")}, K8sContext: f.DataPlaneContexts[0]} - medusaDeployment1 := &appsv1.Deployment{} - require.Eventually(func() bool { - if err := f.Get(ctx, medusaDeploymentKey1, medusaDeployment1); err != nil { - return false - } - return true - }, timeout, interval) - - require.True(f.ContainerHasEnvVar(medusaDeployment1.Spec.Template.Spec.Containers[0], "MEDUSA_RESOLVE_IP_ADDRESSES", "False")) - - t.Log("check that the standalone Medusa service was created") - medusaServiceKey1 := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusa.MedusaServiceName(k8ssandraClusterName, "dc1")}, K8sContext: f.DataPlaneContexts[0]} - medusaService1 := &corev1.Service{} - require.Eventually(func() bool { - if err := f.Get(ctx, medusaServiceKey1, medusaService1); err != nil { - return false - } - return true - }, timeout, interval) - t.Log("update datacenter status to scaling up") err = f.PatchDatacenterStatus(ctx, dc1Key, func(dc *cassdcapi.CassandraDatacenter) { dc.SetCondition(cassdcapi.DatacenterCondition{ @@ -259,30 +234,9 @@ func createMultiDcClusterWithMedusa(t *testing.T, ctx context.Context, f *framew return f.UpdateDatacenterGeneration(ctx, t, dc1Key) }, timeout, interval, "failed to update dc1 generation") - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc2", f.DataPlaneContexts[1]) t.Log("check that dc2 was created") require.Eventually(f.DatacenterExists(ctx, dc2Key), timeout, interval) - t.Log("check that the standalone Medusa deployment was created in dc2") - medusaDeploymentKey2 := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusa.MedusaStandaloneDeploymentName(k8ssandraClusterName, "dc2")}, K8sContext: f.DataPlaneContexts[1]} - medusaDeployment2 := &appsv1.Deployment{} - require.Eventually(func() bool { - if err := f.Get(ctx, medusaDeploymentKey2, medusaDeployment2); err != nil { - return false - } - return true - }, timeout, interval) - - t.Log("check that the standalone Medusa service was created in dc2") - medusaServiceKey2 := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: medusa.MedusaServiceName(k8ssandraClusterName, "dc2")}, K8sContext: f.DataPlaneContexts[1]} - medusaService2 := &corev1.Service{} - require.Eventually(func() bool { - if err := f.Get(ctx, medusaServiceKey2, medusaService2); err != nil { - return false - } - return true - }, timeout, interval) - t.Log("check that remote seeds are set on dc2") dc2 = &cassdcapi.CassandraDatacenter{} err = f.Get(ctx, dc2Key, dc2) @@ -355,11 +309,6 @@ func createMultiDcClusterWithMedusa(t *testing.T, ctx context.Context, f *framew err = f.DeleteK8ssandraCluster(ctx, client.ObjectKey{Namespace: namespace, Name: kc.Name}, timeout, interval) require.NoError(err, "failed to delete K8ssandraCluster") f.AssertObjectDoesNotExist(ctx, t, dc1Key, &cassdcapi.CassandraDatacenter{}, timeout, interval) - // Check that Medusa Standalone deployment and service were deleted - f.AssertObjectDoesNotExist(ctx, t, medusaDeploymentKey1, &appsv1.Deployment{}, timeout, interval) - f.AssertObjectDoesNotExist(ctx, t, medusaDeploymentKey2, &appsv1.Deployment{}, timeout, interval) - f.AssertObjectDoesNotExist(ctx, t, medusaServiceKey1, &corev1.Service{}, timeout, interval) - f.AssertObjectDoesNotExist(ctx, t, medusaServiceKey2, &corev1.Service{}, timeout, interval) } // Check that all the Medusa related objects have been created and are in the expected state. @@ -397,47 +346,6 @@ func checkMedusaObjectsCompliance(t *testing.T, f *framework.Framework, dc *cass } } -func reconcileMedusaStandaloneDeployment(ctx context.Context, t *testing.T, f *framework.Framework, kc *api.K8ssandraCluster, dcName string, k8sContext string) { - t.Log("create Medusa Standalone deployment") - - medusaDepl := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName), - Namespace: kc.Namespace, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName)}, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName)}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName), - Image: "quay.io/k8ssandra/medusa:0.11.0", - }, - }, - }, - }, - }, - } - medusaKey := framework.ClusterKey{NamespacedName: utils.GetKey(medusaDepl), K8sContext: k8sContext} - require.NoError(t, f.Create(ctx, medusaKey, medusaDepl)) - - actualMedusaDepl := &appsv1.Deployment{} - assert.Eventually(t, func() bool { - err := f.Get(ctx, medusaKey, actualMedusaDepl) - return err == nil - }, timeout, interval, "failed to get Medusa Deployment") - - err := f.SetMedusaDeplAvailable(ctx, medusaKey) - - require.NoError(t, err, "Failed to update Medusa Deployment status") -} - func createSingleDcClusterWithMedusaConfigRef(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) { require := require.New(t) @@ -636,8 +544,6 @@ func createMultiDcClusterWithReplicatedSecrets(t *testing.T, ctx context.Context verifySuperuserSecretCreated(ctx, t, f, kc) verifyReplicatedSecretReconciled(ctx, t, f, kc) - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc1", f.DataPlaneContexts[1]) - // crate the first DC dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[1]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -648,7 +554,6 @@ func createMultiDcClusterWithReplicatedSecrets(t *testing.T, ctx context.Context require.NoError(err, "failed to update dc1 status to ready") // create the second DC - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc2", f.DataPlaneContexts[2]) dc2Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc2"}, K8sContext: f.DataPlaneContexts[2]} require.Eventually(f.DatacenterExists(ctx, dc2Key), timeout, interval) @@ -709,8 +614,6 @@ func createSingleDcClusterWithManagementApiSecured(t *testing.T, ctx context.Con require.NoError(f.Client.Create(ctx, kc)) verifyReplicatedSecretReconciled(ctx, t, f, kc) - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc1", f.DataPlaneContexts[0]) - dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) diff --git a/controllers/medusa/controllers_test.go b/controllers/medusa/controllers_test.go index a5de2436a..4bceca221 100644 --- a/controllers/medusa/controllers_test.go +++ b/controllers/medusa/controllers_test.go @@ -162,7 +162,15 @@ func setupMedusaRestoreJobTestEnv(t *testing.T, ctx context.Context) *testutils. } for _, env := range testEnv.GetDataPlaneEnvTests() { - dataPlaneMgr, err := ctrl.NewManager(env.Config, ctrl.Options{Scheme: scheme.Scheme}) + dataPlaneMgr, err := ctrl.NewManager( + env.Config, + ctrl.Options{ + Scheme: scheme.Scheme, + Host: env.WebhookInstallOptions.LocalServingHost, + Port: env.WebhookInstallOptions.LocalServingPort, + CertDir: env.WebhookInstallOptions.LocalServingCertDir, + }, + ) if err != nil { return err } @@ -173,6 +181,7 @@ func setupMedusaRestoreJobTestEnv(t *testing.T, ctx context.Context) *testutils. Scheme: scheme.Scheme, ClientFactory: medusaRestoreClientFactory, }).SetupWithManager(dataPlaneMgr) + secretswebhook.SetupSecretsInjectorWebhook(dataPlaneMgr) if err != nil { return err } diff --git a/controllers/medusa/medusabackupjob_controller_test.go b/controllers/medusa/medusabackupjob_controller_test.go index 6c1704073..d9b7e339b 100644 --- a/controllers/medusa/medusabackupjob_controller_test.go +++ b/controllers/medusa/medusabackupjob_controller_test.go @@ -15,11 +15,9 @@ import ( "github.com/k8ssandra/k8ssandra-operator/pkg/images" "github.com/k8ssandra/k8ssandra-operator/pkg/medusa" "github.com/k8ssandra/k8ssandra-operator/pkg/shared" - "github.com/k8ssandra/k8ssandra-operator/pkg/utils" "github.com/k8ssandra/k8ssandra-operator/test/framework" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -89,7 +87,6 @@ func testMedusaBackupDatacenter(t *testing.T, ctx context.Context, f *framework. require.NoError(err, "failed to create K8ssandraCluster") reconcileReplicatedSecret(ctx, t, f, kc) - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "dc1", f.DataPlaneContexts[0]) t.Log("check that dc1 was created") dc1Key := framework.NewClusterKey(f.DataPlaneContexts[0], namespace, "dc1") require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -469,47 +466,6 @@ func verifyObjectDoesNotExist(ctx context.Context, t *testing.T, f *framework.Fr }, timeout, interval, "failed to verify object does not exist", key) } -func reconcileMedusaStandaloneDeployment(ctx context.Context, t *testing.T, f *framework.Framework, kc *k8ss.K8ssandraCluster, dcName string, k8sContext string) { - t.Logf("start reconcileMedusaStandaloneDeployment for dc %s", dcName) - - medusaDepl := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName), - Namespace: kc.Namespace, - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app": medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName)}, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName)}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: medusa.MedusaStandaloneDeploymentName(kc.SanitizedName(), dcName), - Image: "quay.io/k8ssandra/medusa:0.11.0", - }, - }, - }, - }, - }, - } - medusaKey := framework.ClusterKey{NamespacedName: utils.GetKey(medusaDepl), K8sContext: k8sContext} - require.NoError(t, f.Create(ctx, medusaKey, medusaDepl)) - - actualMedusaDepl := &appsv1.Deployment{} - assert.Eventually(t, func() bool { - err := f.Get(ctx, medusaKey, actualMedusaDepl) - return err == nil - }, timeout, interval, "failed to get Medusa Deployment") - - err := f.SetMedusaDeplAvailable(ctx, medusaKey) - - require.NoError(t, err, "Failed to update Medusa Deployment status") -} - func TestHumanize(t *testing.T) { t.Run("humanizeTrivialSizes", humanizeTrivialSizes) t.Run("humanizeArbitrarySizes", humanizeArbitrarySizes) diff --git a/controllers/medusa/medusarestorejob_controller.go b/controllers/medusa/medusarestorejob_controller.go index 6c2eae0b7..5d36c32c9 100644 --- a/controllers/medusa/medusarestorejob_controller.go +++ b/controllers/medusa/medusarestorejob_controller.go @@ -20,6 +20,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/go-logr/logr" + "github.com/k8ssandra/k8ssandra-operator/pkg/shared" "net" "time" @@ -121,7 +123,7 @@ func (r *MedusaRestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Req // Prepare the restore by placing a mapping file in the Cassandra data volume. if !request.RestoreJob.Status.RestorePrepared { restorePrepared := false - if restoreMapping, err := r.prepareRestore(ctx, request); err != nil { + if restoreMapping, err := r.prepareRestore(ctx, request, logger); err != nil { logger.Error(err, "Failed to prepare restore") return ctrl.Result{}, err } else { @@ -262,26 +264,33 @@ func (r *MedusaRestoreJobReconciler) podTemplateSpecUpdateComplete(ctx context.C } // prepareRestore prepares the MedusaRestoreMapping for the restore operation. -// It uses the Medusa client to get the host map for the restore operation, using the Medusa standalone service to get the backup topology. -func (r *MedusaRestoreJobReconciler) prepareRestore(ctx context.Context, request *medusa.RestoreRequest) (*medusav1alpha1.MedusaRestoreMapping, error) { - medusaClient, err := r.ClientFactory.NewClient(ctx, medusaServiceUrl( - cassdcapi.CleanupForKubernetes(request.Datacenter.Spec.ClusterName), - request.Datacenter.SanitizedName(), - request.Datacenter.Namespace)) - if err != nil { - return nil, err - } - restoreHostMap, err := medusa.GetHostMap(request.Datacenter, *request.RestoreJob, medusaClient, ctx) +// It uses the Medusa client to get the host map for the restore operation, using the first pod answering on the backup sidecar port. +func (r *MedusaRestoreJobReconciler) prepareRestore(ctx context.Context, request *medusa.RestoreRequest, logger logr.Logger) (*medusav1alpha1.MedusaRestoreMapping, error) { + pods, err := medusa.GetCassandraDatacenterPods(ctx, request.Datacenter, r, request.Log) if err != nil { + logger.Error(err, "Failed to get datacenter pods") return nil, err } - medusaRestoreMapping, err := restoreHostMap.ToMedusaRestoreMapping() - if err != nil { - return nil, err - } + for _, pod := range pods { + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + if medusaClient, err := r.ClientFactory.NewClient(ctx, addr); err != nil { + logger.Error(err, "Failed to create Medusa client", "address", addr) + } else { + restoreHostMap, err := medusa.GetHostMap(request.Datacenter, *request.RestoreJob, medusaClient, ctx) + if err != nil { + return nil, err + } + + medusaRestoreMapping, err := restoreHostMap.ToMedusaRestoreMapping() + if err != nil { + return nil, err + } - return &medusaRestoreMapping, nil + return &medusaRestoreMapping, nil + } + } + return nil, fmt.Errorf("failed to get host map from backup") } func validateBackupForRestore(backup *medusav1alpha1.MedusaBackup, cassdc *cassdcapi.CassandraDatacenter) error { @@ -465,7 +474,3 @@ func getRestoreInitContainerIndex(dc *cassdcapi.CassandraDatacenter) (int, error return 0, fmt.Errorf("restore initContainer (%s) not found", restoreContainerName) } - -func medusaServiceUrl(clusterName, dcName, dcNamespace string) string { - return net.JoinHostPort(fmt.Sprintf("%s.%s.svc", medusa.MedusaServiceName(clusterName, dcName), dcNamespace), fmt.Sprintf("%d", medusa.DefaultMedusaPort)) -} diff --git a/controllers/medusa/medusarestorejob_controller_test.go b/controllers/medusa/medusarestorejob_controller_test.go index 2e8c2a38f..bd602b2e9 100644 --- a/controllers/medusa/medusarestorejob_controller_test.go +++ b/controllers/medusa/medusarestorejob_controller_test.go @@ -2,6 +2,7 @@ package medusa import ( "context" + "k8s.io/apimachinery/pkg/api/errors" "sync" "testing" "time" @@ -77,7 +78,6 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework require.NoError(f.Client.Create(ctx, kc), "failed to create K8ssandraCluster") reconcileReplicatedSecret(ctx, t, f, kc) - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "real-dc1", f.DataPlaneContexts[0]) t.Log("check that dc1 was created") dc1Key := framework.NewClusterKey(f.DataPlaneContexts[0], namespace, "dc1") require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -118,9 +118,6 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework return condition != nil && condition.Status == corev1.ConditionTrue }, timeout, interval, "timed out waiting for K8ssandraCluster status update") - dc1 := &cassdcapi.CassandraDatacenter{} - err = f.Get(ctx, dc1Key, dc1) - t.Log("update dc1 status to ready") err = f.PatchDatacenterStatus(ctx, dc1Key, func(dc *cassdcapi.CassandraDatacenter) { dc.Status.CassandraOperatorProgress = cassdcapi.ProgressReady @@ -132,6 +129,12 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework }) require.NoError(err, "failed to update dc1 status to ready") + dc1 := &cassdcapi.CassandraDatacenter{} + err = f.Get(ctx, dc1Key, dc1) + + err = createCassandraPods(t, f, ctx, dc1Key, dc1) + require.NoError(err) + t.Log("creating MedusaBackup") backup := &api.MedusaBackup{ ObjectMeta: metav1.ObjectMeta{ @@ -189,7 +192,7 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework t.Log("check that the datacenter is set to be stopped") require.Eventually(withDc1(func(dc *cassdcapi.CassandraDatacenter) bool { return dc.Spec.Stopped == true - }), timeout, interval, "timed out waiting for CassandraDatacenter stopped flag to be set") + }), timeout*5, interval, "timed out waiting for CassandraDatacenter stopped flag to be set") t.Log("delete datacenter pods to simulate shutdown") err = f.DeleteAllOf(ctx, dc1Key.K8sContext, &corev1.Pod{}, client.InNamespace(namespace), client.MatchingLabels{cassdcapi.DatacenterLabel: "real-dc1"}) @@ -297,6 +300,42 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework } +func createCassandraPods(t *testing.T, f *framework.Framework, ctx context.Context, dcKey framework.ClusterKey, dc *cassdcapi.CassandraDatacenter) error { + dcServiceKey := framework.NewClusterKey(dcKey.K8sContext, dcKey.Namespace, dc.GetAllPodsServiceName()) + dcService := &corev1.Service{} + if err := f.Get(ctx, dcServiceKey, dcService); err != nil { + if errors.IsNotFound(err) { + dcService = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: dcServiceKey.Namespace, + Name: dcServiceKey.Name, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + cassdcapi.ClusterLabel: cassdcapi.CleanLabelValue(dc.Spec.ClusterName), + }, + Ports: []corev1.ServicePort{ + { + Name: "cql", + Port: 9042, + }, + }, + }, + } + + err := f.Create(ctx, dcServiceKey, dcService) + if err != nil { + return err + } + } else { + t.Errorf("failed to get service %s: %v", dcServiceKey, err) + return err + } + } + createDatacenterPods(t, f, ctx, dcKey, dc) + return nil +} + func testValidationErrorStopsRestore(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) { require := require.New(t) require.NoError(f.Client.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(namespace))) @@ -349,7 +388,6 @@ func testValidationErrorStopsRestore(t *testing.T, ctx context.Context, f *frame require.NoError(err, "failed to create K8ssandraCluster") reconcileReplicatedSecret(ctx, t, f, kc) - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, "real-dc1", f.DataPlaneContexts[0]) t.Log("check that dc1 was created") dc1Key := framework.NewClusterKey(f.DataPlaneContexts[0], namespace, "dc1") require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -480,11 +518,6 @@ func findContainer(containers []corev1.Container, name string) *corev1.Container return nil } -func TestMedusaServiceAddress(t *testing.T) { - serviceUrl := medusaServiceUrl("k8c-cluster", "real-dc1", "dc-namespace") - assert.Equal(t, "k8c-cluster-real-dc1-medusa-service.dc-namespace.svc:50051", serviceUrl) -} - func TestValidateBackupForRestore(t *testing.T) { assert := assert.New(t) @@ -565,10 +598,6 @@ type fakeMedusaRestoreClientFactory struct { clients map[string]*fakeMedusaRestoreClient } -func NewMedusaRestoreClientFactory() *fakeMedusaRestoreClientFactory { - return &fakeMedusaRestoreClientFactory{clients: make(map[string]*fakeMedusaRestoreClient, 0)} -} - func NewMedusaClientRestoreFactory() *fakeMedusaRestoreClientFactory { return &fakeMedusaRestoreClientFactory{clients: make(map[string]*fakeMedusaRestoreClient, 0)} } diff --git a/controllers/medusa/medusatask_controller_test.go b/controllers/medusa/medusatask_controller_test.go index fb592f3f6..55239b6a9 100644 --- a/controllers/medusa/medusatask_controller_test.go +++ b/controllers/medusa/medusatask_controller_test.go @@ -96,7 +96,6 @@ func testMedusaTasks(t *testing.T, ctx context.Context, f *framework.Framework, reconcileReplicatedSecret(ctx, t, f, kc) for _, dcKey := range []framework.ClusterKey{dc1Key, dc2Key} { - reconcileMedusaStandaloneDeployment(ctx, t, f, kc, dcKey.Name, f.DataPlaneContexts[0]) t.Logf("check that %s was created", dcKey.Name) require.Eventually(f.DatacenterExists(ctx, dcKey), timeout, interval) diff --git a/pkg/medusa/reconcile.go b/pkg/medusa/reconcile.go index 48c28b9aa..cc1e2c686 100644 --- a/pkg/medusa/reconcile.go +++ b/pkg/medusa/reconcile.go @@ -12,7 +12,6 @@ import ( k8ss "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/images" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" @@ -21,7 +20,6 @@ import ( "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -492,90 +490,10 @@ func GenerateMedusaVolumes(dcConfig *cassandra.DatacenterConfig, medusaSpec *api return newVolumes } -func MedusaServiceName(clusterName string, dcName string) string { - return fmt.Sprintf("%s-%s-medusa-service", clusterName, dcName) -} - -func MedusaStandaloneDeploymentName(clusterName string, dcName string) string { - return fmt.Sprintf("%s-%s-medusa-standalone", clusterName, dcName) -} - func MedusaPurgeCronJobName(clusterName string, dcName string) string { return fmt.Sprintf("%s-%s-medusa-purge", clusterName, dcName) } -func StandaloneMedusaDeployment(medusaContainer corev1.Container, clusterName, dcName, namespace string, logger logr.Logger, medusaImage *images.Image) *appsv1.Deployment { - // The standalone medusa pod won't be able to resolve its own IP address using DNS entries - medusaContainer.Env = append(medusaContainer.Env, corev1.EnvVar{Name: "MEDUSA_RESOLVE_IP_ADDRESSES", Value: "False"}) - medusaDeployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: MedusaStandaloneDeploymentName(clusterName, dcName), - Namespace: namespace, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: pointer.Int32(1), - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": MedusaStandaloneDeploymentName(clusterName, dcName), - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": MedusaStandaloneDeploymentName(clusterName, dcName), - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - medusaContainer, - }, - ImagePullSecrets: images.CollectPullSecrets(medusaImage.ApplyDefaults(defaultMedusaImage)), - Volumes: []corev1.Volume{}, - Hostname: MedusaStandaloneDeploymentName(clusterName, dcName), - }, - }, - }, - } - - logger.Info("Creating standalone medusa deployment on namespace", "namespace", medusaDeployment.Namespace) - // Create dummy additional volumes - // These volumes won't be used by the Medusa standalone pod, but the mounts will be necessary for Medusa to startup properly - // TODO: Investigate if we can adapt Medusa to remove these volumes - for _, extraVolume := range [](string){"server-config", "server-data", MedusaBackupsVolumeName} { - medusaDeployment.Spec.Template.Spec.Volumes = append(medusaDeployment.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: extraVolume, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }) - } - - return medusaDeployment -} - -func StandaloneMedusaService(dcConfig *cassandra.DatacenterConfig, medusaSpec *api.MedusaClusterTemplate, clusterName, namespace string, logger logr.Logger) *corev1.Service { - medusaService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: MedusaServiceName(clusterName, dcConfig.SanitizedName()), - Namespace: namespace, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "grpc", - Port: DefaultMedusaPort, - TargetPort: intstr.FromInt(DefaultMedusaPort), - }, - }, - Selector: map[string]string{ - "app": MedusaStandaloneDeploymentName(clusterName, dcConfig.SanitizedName()), - }, - }, - } - - return medusaService -} - func PurgeCronJob(dcConfig *cassandra.DatacenterConfig, clusterName, namespace string, logger logr.Logger) (*batchv1.CronJob, error) { cronJobName := MedusaPurgeCronJobName(cassdcapi.CleanupForKubernetes(clusterName), dcConfig.SanitizedName()) logger.Info(fmt.Sprintf("Creating Medusa purge backups cronjob: %s", cronJobName)) diff --git a/pkg/medusa/reconcile_test.go b/pkg/medusa/reconcile_test.go index f9b49bd7a..9d75d31cd 100644 --- a/pkg/medusa/reconcile_test.go +++ b/pkg/medusa/reconcile_test.go @@ -9,7 +9,6 @@ import ( api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" medusaapi "github.com/k8ssandra/k8ssandra-operator/apis/medusa/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" - "github.com/k8ssandra/k8ssandra-operator/pkg/images" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -596,41 +595,6 @@ func TestInitContainerCustomResources(t *testing.T) { } -func TestStandaloneMedusaDeploymentImageSettings(t *testing.T) { - medusaSpec := &medusaapi.MedusaClusterTemplate{ - StorageProperties: medusaapi.Storage{ - StorageProvider: "s3", - StorageSecretRef: corev1.LocalObjectReference{ - Name: "secret", - }, - BucketName: "bucket", - }, - ContainerImage: &images.Image{ - Registry: "reg1", - Name: "img1", - Repository: "repo1", - Tag: "tag1", - PullPolicy: "Always", - PullSecretRef: &corev1.LocalObjectReference{Name: "main-secret"}, - }, - } - - dcConfig := cassandra.DatacenterConfig{} - - logger := logr.New(logr.Discard().GetSink()) - - medusaContainer, err := CreateMedusaMainContainer(&dcConfig, medusaSpec, false, "test", logger) - assert.NoError(t, err) - UpdateMedusaInitContainer(&dcConfig, medusaSpec, false, "test", logger) - UpdateMedusaMainContainer(&dcConfig, medusaContainer) - - desiredMedusaStandalone := StandaloneMedusaDeployment(*medusaContainer, "dc1", dcConfig.SanitizedName(), "test", logger, medusaSpec.ContainerImage) - - assert.Equal(t, "main-secret", desiredMedusaStandalone.Spec.Template.Spec.ImagePullSecrets[0].Name, "expected standalone container image pull secret to be set") - assert.Equal(t, "reg1/repo1/img1:tag1", desiredMedusaStandalone.Spec.Template.Spec.Containers[0].Image, "expected standalone container image to be set to reg1/repo1/img1:tag1") - assert.Equal(t, corev1.PullAlways, desiredMedusaStandalone.Spec.Template.Spec.Containers[0].ImagePullPolicy, "expected standalone pull policy to be set to Always") -} - func TestExternalSecretsFlag(t *testing.T) { medusaSpec := &medusaapi.MedusaClusterTemplate{ StorageProperties: medusaapi.Storage{ diff --git a/test/e2e/medusa_test.go b/test/e2e/medusa_test.go index 14752f432..a43eba784 100644 --- a/test/e2e/medusa_test.go +++ b/test/e2e/medusa_test.go @@ -15,7 +15,6 @@ import ( medusapkg "github.com/k8ssandra/k8ssandra-operator/pkg/medusa" "github.com/k8ssandra/k8ssandra-operator/test/framework" "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -82,8 +81,6 @@ func createMultiMedusaJob(t *testing.T, ctx context.Context, namespace string, f checkDatacenterReady(t, ctx, dcKey, f) checkMedusaContainersExist(t, ctx, namespace, dcKey, f, kc) checkPurgeCronJobExists(t, ctx, namespace, dcKey, f, kc) - checkMedusaStandaloneDeploymentExists(t, ctx, dcKey, f, kc) - checkMedusaStandaloneServiceExists(t, ctx, dcKey, f, kc) checkReplicatedSecretMounted(t, ctx, f, dcKey, dcKey.Namespace, kc) } @@ -311,31 +308,6 @@ func verifyRestoreJobFinished(t *testing.T, ctx context.Context, f *framework.E2 } -func checkMedusaStandaloneDeploymentExists(t *testing.T, ctx context.Context, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) { - t.Log("Checking that the Medusa standalone pod has been created") - require := require.New(t) - // Get the medusa standalone pod and check that it is running - require.Eventually(func() bool { - deployment := &appsv1.Deployment{} - deploymentKey := framework.ClusterKey{K8sContext: dcKey.K8sContext, NamespacedName: types.NamespacedName{Namespace: dcKey.Namespace, Name: medusapkg.MedusaStandaloneDeploymentName(kc.Name, DcName(t, f, dcKey))}} - err := f.Get(ctx, deploymentKey, deployment) - require.NoError(err, "Error getting the medusa standalone pod") - return deployment.Status.ReadyReplicas == 1 - }, polling.medusaReady.timeout, polling.medusaReady.interval, "Medusa standalone pod is not running") -} - -func checkMedusaStandaloneServiceExists(t *testing.T, ctx context.Context, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) { - t.Log("Checking that the Medusa standalone service has been created") - require := require.New(t) - // Get the medusa standalone pod and check that it is running - require.Eventually(func() bool { - service := &corev1.Service{} - serviceKey := framework.ClusterKey{K8sContext: dcKey.K8sContext, NamespacedName: types.NamespacedName{Namespace: dcKey.Namespace, Name: medusapkg.MedusaServiceName(kc.Name, DcName(t, f, dcKey))}} - err := f.Get(ctx, serviceKey, service) - return err == nil - }, polling.medusaReady.timeout, polling.medusaReady.interval, "Medusa standalone service doesn't exist") -} - func createMedusaConfiguration(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework) { require := require.New(t) medusaConfig := &medusa.MedusaConfiguration{}