From 95411ac622ca17483a417aeb25a52e532d50a815 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Wed, 22 May 2024 06:36:05 -0500 Subject: [PATCH 1/8] Add ability to disable medusa-purge cron --- apis/medusa/v1alpha1/medusa_types.go | 6 ++++ .../bases/k8ssandra.io_k8ssandraclusters.yaml | 1 + controllers/k8ssandra/medusa_reconciler.go | 29 ++++++++++--------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/apis/medusa/v1alpha1/medusa_types.go b/apis/medusa/v1alpha1/medusa_types.go index d70994002..4c53ea936 100644 --- a/apis/medusa/v1alpha1/medusa_types.go +++ b/apis/medusa/v1alpha1/medusa_types.go @@ -181,4 +181,10 @@ type MedusaClusterTemplate struct { // Define the liveness probe settings to use for the Medusa containers. // +optional LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` + + // Should medusa purge backups nightly or not + // Defaults to true. + // +kubebuilder:default=true + // +optional + PurgeBackups bool `json:"purgeBackups,omitempty"` } diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml index 9bd79e06f..2a9ddcef7 100644 --- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml +++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml @@ -26365,6 +26365,7 @@ spec: type: string type: object type: object + purgeBackups: false storageProperties: description: Provides all storage backend related properties for backups. diff --git a/controllers/k8ssandra/medusa_reconciler.go b/controllers/k8ssandra/medusa_reconciler.go index 9cc35c1bb..8cc0a15b3 100644 --- a/controllers/k8ssandra/medusa_reconciler.go +++ b/controllers/k8ssandra/medusa_reconciler.go @@ -98,19 +98,22 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( kcKey := utils.GetKey(kc) // Create a cron job to purge Medusa backups - operatorNamespace := r.getOperatorNamespace() - purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger) - if err != nil { - logger.Info("Failed to create Medusa purge backups cronjob", "error", err) - return result.Error(err) - } - purgeCronJob.SetLabels(labels.CleanedUpByLabels(kcKey)) - recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob) - switch { - case recRes.IsError(): - return recRes - case recRes.IsRequeue(): - return recRes + logger.Info("Checking if Medusa backups should be purged, PurgeBackups: " + fmt.Sprintf("%t", medusaSpec.PurgeBackups)) + if medusaSpec.PurgeBackups { + operatorNamespace := r.getOperatorNamespace() + purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger) + if err != nil { + logger.Info("Failed to create Medusa purge backups cronjob", "error", err) + return result.Error(err) + } + purgeCronJob.SetLabels(labels.CleanedUpByLabels(kcKey)) + recRes = reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob) + switch { + case recRes.IsError(): + return recRes + case recRes.IsRequeue(): + return recRes + } } } else { From 295d263c35eda0a821cda443d88597d23178d459 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Fri, 19 Apr 2024 12:25:38 -0500 Subject: [PATCH 2/8] Fixing tests --- apis/medusa/v1alpha1/medusa_types.go | 2 +- .../k8ssandra-operator/crds/k8ssandra-operator-crds.yaml | 6 ++++++ config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/apis/medusa/v1alpha1/medusa_types.go b/apis/medusa/v1alpha1/medusa_types.go index 4c53ea936..1fb46dcf9 100644 --- a/apis/medusa/v1alpha1/medusa_types.go +++ b/apis/medusa/v1alpha1/medusa_types.go @@ -182,7 +182,7 @@ type MedusaClusterTemplate struct { // +optional LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` - // Should medusa purge backups nightly or not + // PurgeBackups toggles if the medusa backups should be purged nightly or not // Defaults to true. // +kubebuilder:default=true // +optional diff --git a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml index 8898869e9..7b2bd7b2e 100644 --- a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml +++ b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml @@ -26109,6 +26109,12 @@ spec: type: string type: object x-kubernetes-map-type: atomic + purgeBackups: + default: true + description: |- + PurgeBackups toggles if the medusa backups should be purged nightly or not + Defaults to true. + type: boolean readinessProbe: description: Define the readiness probe settings to use for the Medusa containers. diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml index 2a9ddcef7..bed283f7c 100644 --- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml +++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml @@ -26047,6 +26047,12 @@ spec: type: string type: object x-kubernetes-map-type: atomic + purgeBackups: + default: true + description: |- + PurgeBackups toggles if the medusa backups should be purged nightly or not + Defaults to true. + type: boolean readinessProbe: description: Define the readiness probe settings to use for the Medusa containers. @@ -26365,7 +26371,6 @@ spec: type: string type: object type: object - purgeBackups: false storageProperties: description: Provides all storage backend related properties for backups. From f95c330b0263a019bb28bd532b5c8616aef42f08 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Wed, 22 May 2024 06:49:30 -0500 Subject: [PATCH 3/8] bool to pointer --- apis/medusa/v1alpha1/medusa_types.go | 2 +- apis/medusa/v1alpha1/zz_generated.deepcopy.go | 5 +++++ controllers/k8ssandra/medusa_reconciler.go | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apis/medusa/v1alpha1/medusa_types.go b/apis/medusa/v1alpha1/medusa_types.go index 1fb46dcf9..db88f7d8c 100644 --- a/apis/medusa/v1alpha1/medusa_types.go +++ b/apis/medusa/v1alpha1/medusa_types.go @@ -186,5 +186,5 @@ type MedusaClusterTemplate struct { // Defaults to true. // +kubebuilder:default=true // +optional - PurgeBackups bool `json:"purgeBackups,omitempty"` + PurgeBackups *bool `json:"purgeBackups,omitempty"` } diff --git a/apis/medusa/v1alpha1/zz_generated.deepcopy.go b/apis/medusa/v1alpha1/zz_generated.deepcopy.go index 316dd4b36..d9a9eaa22 100644 --- a/apis/medusa/v1alpha1/zz_generated.deepcopy.go +++ b/apis/medusa/v1alpha1/zz_generated.deepcopy.go @@ -404,6 +404,11 @@ func (in *MedusaClusterTemplate) DeepCopyInto(out *MedusaClusterTemplate) { *out = new(v1.Probe) (*in).DeepCopyInto(*out) } + if in.PurgeBackups != nil { + in, out := &in.PurgeBackups, &out.PurgeBackups + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MedusaClusterTemplate. diff --git a/controllers/k8ssandra/medusa_reconciler.go b/controllers/k8ssandra/medusa_reconciler.go index 8cc0a15b3..1245475db 100644 --- a/controllers/k8ssandra/medusa_reconciler.go +++ b/controllers/k8ssandra/medusa_reconciler.go @@ -98,8 +98,8 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( kcKey := utils.GetKey(kc) // Create a cron job to purge Medusa backups - logger.Info("Checking if Medusa backups should be purged, PurgeBackups: " + fmt.Sprintf("%t", medusaSpec.PurgeBackups)) - if medusaSpec.PurgeBackups { + logger.Info("Checking if Medusa backups should be purged, PurgeBackups: " + fmt.Sprintf("%t", *medusaSpec.PurgeBackups)) + if *medusaSpec.PurgeBackups { operatorNamespace := r.getOperatorNamespace() purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger) if err != nil { @@ -107,7 +107,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( return result.Error(err) } purgeCronJob.SetLabels(labels.CleanedUpByLabels(kcKey)) - recRes = reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob) + recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob) switch { case recRes.IsError(): return recRes From 374f13a3395ab7fb415e1f21e8655e37ac4c59f2 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Wed, 22 May 2024 08:26:28 -0500 Subject: [PATCH 4/8] wip e2e-tests --- test/e2e/medusa_test.go | 36 ++++++++++++++++++- .../fixtures/multi-dc-medusa/k8ssandra.yaml | 1 + 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/test/e2e/medusa_test.go b/test/e2e/medusa_test.go index a43eba784..9aff1fb72 100644 --- a/test/e2e/medusa_test.go +++ b/test/e2e/medusa_test.go @@ -113,7 +113,7 @@ func createMultiDcSingleMedusaJob(t *testing.T, ctx context.Context, namespace s checkDatacenterReady(t, ctx, dcKey, f) checkMedusaContainersExist(t, ctx, namespace, dcKey, f, kc) - checkPurgeCronJobExists(t, ctx, namespace, dcKey, f, kc) + checkNoPurgeCronJob(t, ctx, namespace, dcKey, f, kc) createBackupJob(t, ctx, namespace, f, dcKey) verifyBackupJobFinished(t, ctx, f, dcKey, backupKey) } @@ -198,6 +198,40 @@ func checkPurgeCronJobExists(t *testing.T, ctx context.Context, namespace string }, polling.medusaBackupDone.timeout, polling.medusaBackupDone.interval, "Medusa purge Job didn't finish within timeout") } +func checkNoPurgeCronJob(t *testing.T, ctx context.Context, namespace string, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) { + require := require.New(t) + // Get the Cassandra pod + dc1 := &cassdcapi.CassandraDatacenter{} + err := f.Get(ctx, dcKey, dc1) + // check medusa containers exist + require.NoError(err, "Error getting the CassandraDatacenter") + t.Log("Checking that all the Medusa related objects have been created and are in the expected state") + // check that the cronjob exists + cronJob := &batchv1.CronJob{} + err = f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, medusapkg.MedusaPurgeCronJobName(kc.SanitizedName(), dc1.SanitizedName())), cronJob) + require.NoErrorf(err, "Error getting the Medusa purge CronJob. ClusterName: %s, DatacenterName: %s", kc.SanitizedName(), dc1.SanitizedName()) + require.Equal("k8ssandra-operator", cronJob.Spec.JobTemplate.Spec.Template.Spec.ServiceAccountName, "Service account name is not correct") + // create a Job from the cronjob spec + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "test-purge-job", + }, + Spec: cronJob.Spec.JobTemplate.Spec, + } + err = f.Create(ctx, dcKey, job) + require.NoErrorf(err, "A Medusa purge Job was created when it should not have been. ClusterName: %s, DataceneterName: %s, Namespace: %s, JobName: test-no-purge-job", kc.SanitizedName(), dc1.SanitizedName(), namespace) + // ensure the job was not created + require.Never(func() bool { + updated := &batchv1.Job{} + err := f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, "test-purge-job"), updated) + if err != nil { + return false + } + return updated.Status.Succeeded == 1 + }, polling.medusaBackupDone.timeout, polling.medusaBackupDone.interval, "Timeout checking for Medusa purge Job") +} + func createBackupJob(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework, dcKey framework.ClusterKey) { require := require.New(t) t.Log("creating MedusaBackupJob") diff --git a/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml b/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml index 8768ea427..edc532cf9 100644 --- a/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml +++ b/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml @@ -33,6 +33,7 @@ spec: k8sContext: kind-k8ssandra-1 mgmtAPIHeap: 64Mi medusa: + purgeBackups: false storageProperties: storageProvider: s3_compatible bucketName: k8ssandra-medusa From bd1314503a0f065b53ce7637cb70d5c14e88c8ee Mon Sep 17 00:00:00 2001 From: Alexander Dejanovski Date: Wed, 22 May 2024 17:31:56 +0200 Subject: [PATCH 5/8] Slightly rework how the check for cron job absence is done --- test/e2e/medusa_test.go | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/test/e2e/medusa_test.go b/test/e2e/medusa_test.go index 9aff1fb72..619893ef5 100644 --- a/test/e2e/medusa_test.go +++ b/test/e2e/medusa_test.go @@ -113,9 +113,9 @@ func createMultiDcSingleMedusaJob(t *testing.T, ctx context.Context, namespace s checkDatacenterReady(t, ctx, dcKey, f) checkMedusaContainersExist(t, ctx, namespace, dcKey, f, kc) - checkNoPurgeCronJob(t, ctx, namespace, dcKey, f, kc) createBackupJob(t, ctx, namespace, f, dcKey) verifyBackupJobFinished(t, ctx, f, dcKey, backupKey) + checkNoPurgeCronJob(t, ctx, namespace, dcKey, f, kc) } func checkBucketKeyPresent(t *testing.T, f *framework.E2eFramework, ctx context.Context, namespace string, k8sContext string, kc *k8ssandraapi.K8ssandraCluster) { @@ -205,31 +205,10 @@ func checkNoPurgeCronJob(t *testing.T, ctx context.Context, namespace string, dc err := f.Get(ctx, dcKey, dc1) // check medusa containers exist require.NoError(err, "Error getting the CassandraDatacenter") - t.Log("Checking that all the Medusa related objects have been created and are in the expected state") - // check that the cronjob exists + // ensure the cronjob was not created cronJob := &batchv1.CronJob{} err = f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, medusapkg.MedusaPurgeCronJobName(kc.SanitizedName(), dc1.SanitizedName())), cronJob) - require.NoErrorf(err, "Error getting the Medusa purge CronJob. ClusterName: %s, DatacenterName: %s", kc.SanitizedName(), dc1.SanitizedName()) - require.Equal("k8ssandra-operator", cronJob.Spec.JobTemplate.Spec.Template.Spec.ServiceAccountName, "Service account name is not correct") - // create a Job from the cronjob spec - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "test-purge-job", - }, - Spec: cronJob.Spec.JobTemplate.Spec, - } - err = f.Create(ctx, dcKey, job) - require.NoErrorf(err, "A Medusa purge Job was created when it should not have been. ClusterName: %s, DataceneterName: %s, Namespace: %s, JobName: test-no-purge-job", kc.SanitizedName(), dc1.SanitizedName(), namespace) - // ensure the job was not created - require.Never(func() bool { - updated := &batchv1.Job{} - err := f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, "test-purge-job"), updated) - if err != nil { - return false - } - return updated.Status.Succeeded == 1 - }, polling.medusaBackupDone.timeout, polling.medusaBackupDone.interval, "Timeout checking for Medusa purge Job") + require.Error(err, "Cronjob should not exist") } func createBackupJob(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework, dcKey framework.ClusterKey) { From fc2a6ef2f102a37820576a14fd6a09ff09b6c366 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Thu, 23 May 2024 12:53:56 -0500 Subject: [PATCH 6/8] Set toggle on correct cr file --- test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml | 1 - .../fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml b/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml index edc532cf9..8768ea427 100644 --- a/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml +++ b/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml @@ -33,7 +33,6 @@ spec: k8sContext: kind-k8ssandra-1 mgmtAPIHeap: 64Mi medusa: - purgeBackups: false storageProperties: storageProvider: s3_compatible bucketName: k8ssandra-medusa diff --git a/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml b/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml index 6b194cb8e..2df832a08 100644 --- a/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml +++ b/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml @@ -69,6 +69,7 @@ spec: jvmOptions: heapSize: 512Mi medusa: + purgeBackups: false cassandraUserSecretRef: name: cluster2-medusa storageProperties: From 838df433c9ad1d9db14aff39810c9cd2d723d2d6 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Thu, 23 May 2024 14:14:01 -0500 Subject: [PATCH 7/8] Add docs and changelog --- CHANGELOG/CHANGELOG-1.17.md | 3 ++- docs/content/en/tasks/backup-restore/_index.md | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG/CHANGELOG-1.17.md b/CHANGELOG/CHANGELOG-1.17.md index 7fb62ce6e..64e174499 100644 --- a/CHANGELOG/CHANGELOG-1.17.md +++ b/CHANGELOG/CHANGELOG-1.17.md @@ -22,4 +22,5 @@ When cutting a new release, update the `unreleased` heading to the tag being gen * [BUGFIX] [#1322](https://github.com/k8ssandra/k8ssandra-operator/issues/1322) Fix bug where server-system-logger customisations from the Containers field would be overwritten when vector was enabled. * [FEATURE] Add support for HCD 1.0 * [ENHANCEMENT] [#1329](https://github.com/k8ssandra/k8ssandra-operator/issues/1329) Add config emptyDir volume mount on Reaper deployment to allow read only root FS -* [BUGFIX] Fix HCD jvm options generation \ No newline at end of file +* [BUGFIX] Fix HCD jvm options generation +* [ENHANCEMENT] [#1278](https://github.com/k8ssandra/k8ssandra-operator/issues/1278) Add toggle to allow the disabling of the Medusa purge CronJob creation diff --git a/docs/content/en/tasks/backup-restore/_index.md b/docs/content/en/tasks/backup-restore/_index.md index d20b29d08..426bc737b 100644 --- a/docs/content/en/tasks/backup-restore/_index.md +++ b/docs/content/en/tasks/backup-restore/_index.md @@ -473,7 +473,18 @@ spec: ``` The above `CronJob` will be scheduled to run every day at midnight, and trigger a `MedusaTask` object creation to purge the backups. +### Disabling the purge CronJob +If you want to disable the automatic creation of the above purge `CronJob`, you can do so by setting the optional `purgeBackups` field to `false` in the `K8ssandraCluster` spec: +```yaml +apiVersion: k8ssandra.io/v1alpha1 +kind: K8ssandraCluster +spec: + medusa: + purgeBackups: false + ``` + +```yaml ## Deprecation Notice The `CassandraBackup` and `CassandraRestore` CRDs are removed in v1.6.0. From f58df085415852fa838851f41256664124d9d039 Mon Sep 17 00:00:00 2001 From: Eric Smalling Date: Fri, 31 May 2024 14:33:15 -0500 Subject: [PATCH 8/8] Add handling for unset purgeBackup and dangling cron job Signed-off-by: Eric Smalling --- controllers/k8ssandra/medusa_reconciler.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/controllers/k8ssandra/medusa_reconciler.go b/controllers/k8ssandra/medusa_reconciler.go index 1245475db..05030605a 100644 --- a/controllers/k8ssandra/medusa_reconciler.go +++ b/controllers/k8ssandra/medusa_reconciler.go @@ -99,7 +99,7 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( kcKey := utils.GetKey(kc) // Create a cron job to purge Medusa backups logger.Info("Checking if Medusa backups should be purged, PurgeBackups: " + fmt.Sprintf("%t", *medusaSpec.PurgeBackups)) - if *medusaSpec.PurgeBackups { + if backupPurgeIsOn(medusaSpec.PurgeBackups) { operatorNamespace := r.getOperatorNamespace() purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger) if err != nil { @@ -114,6 +114,16 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( case recRes.IsRequeue(): return recRes } + } else { + // if an existing purge cron job exists, delete it + purgeCronJob, _ := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), r.getOperatorNamespace(), logger) + if purgeCronJob != nil { + logger.Info("Deleting Medusa purge backups cronjob (may have been created before PurgeBackups was set to false") + if err := remoteClient.Delete(ctx, purgeCronJob); err != nil { + logger.Info("Failed to delete Medusa purge backups cronjob", "error", err) + return result.Error(err) + } + } } } else { @@ -123,6 +133,13 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa( return result.Continue() } +func backupPurgeIsOn(purgeBackups *bool) bool { + if purgeBackups == nil { + return true + } + return *purgeBackups +} + // Generate a secret for Medusa or use the existing one if provided in the spec func (r *K8ssandraClusterReconciler) reconcileMedusaSecrets( ctx context.Context,