Esmalling cronjob improvements

CHANGELOG/CHANGELOG-1.17.md

@@ -22,4 +22,5 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 * [BUGFIX] [#1322](https://github.com/k8ssandra/k8ssandra-operator/issues/1322) Fix bug where server-system-logger customisations from the Containers field would be overwritten when vector was enabled. 
 * [FEATURE] Add support for HCD 1.0
 * [ENHANCEMENT] [#1329](https://github.com/k8ssandra/k8ssandra-operator/issues/1329) Add config emptyDir volume mount on Reaper deployment to allow read only root FS
-* [BUGFIX] Fix HCD jvm options generation
+* [BUGFIX] Fix HCD jvm options generation
+* [ENHANCEMENT] [#1278](https://github.com/k8ssandra/k8ssandra-operator/issues/1278) Add toggle to allow the disabling of the Medusa purge CronJob creation

apis/medusa/v1alpha1/medusa_types.go

@@ -181,4 +181,10 @@ type MedusaClusterTemplate struct {
 	// Define the liveness probe settings to use for the Medusa containers.
 	// +optional
 	LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"`
+
+	// PurgeBackups toggles if the medusa backups should be purged nightly or not
+	// Defaults to true.
+	// +kubebuilder:default=true
+	// +optional
+	PurgeBackups *bool `json:"purgeBackups,omitempty"`
 }

charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml

@@ -26109,6 +26109,12 @@ spec:
                         type: string
                     type: object
                     x-kubernetes-map-type: atomic
+                  purgeBackups:
+                    default: true
+                    description: |-
+                      PurgeBackups toggles if the medusa backups should be purged nightly or not
+                      Defaults to true.
+                    type: boolean
                   readinessProbe:
                     description: Define the readiness probe settings to use for the
                       Medusa containers.

config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml

@@ -26047,6 +26047,12 @@ spec:
                         type: string
                     type: object
                     x-kubernetes-map-type: atomic
+                  purgeBackups:
+                    default: true
+                    description: |-
+                      PurgeBackups toggles if the medusa backups should be purged nightly or not
+                      Defaults to true.
+                    type: boolean
                   readinessProbe:
                     description: Define the readiness probe settings to use for the
                       Medusa containers.

controllers/k8ssandra/medusa_reconciler.go

@@ -3,6 +3,8 @@ package k8ssandra
 import (
 	"context"
 	"fmt"
+	batchv1 "k8s.io/api/batch/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"os"
 
 	"github.com/adutra/goalesce"
@@ -98,19 +100,41 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 
 		kcKey := utils.GetKey(kc)
 		// Create a cron job to purge Medusa backups
+		logger.Info("Checking if Medusa backups should be purged, PurgeBackups: " + fmt.Sprintf("%t", *medusaSpec.PurgeBackups))
 		operatorNamespace := r.getOperatorNamespace()
-		purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger)
-		if err != nil {
-			logger.Info("Failed to create Medusa purge backups cronjob", "error", err)
-			return result.Error(err)
-		}
-		purgeCronJob.SetLabels(labels.CleanedUpByLabels(kcKey))
-		recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob)
-		switch {
-		case recRes.IsError():
-			return recRes
-		case recRes.IsRequeue():
-			return recRes
+		if backupPurgeIsOn(medusaSpec.PurgeBackups) {
+			purgeCronJob, err := medusa.PurgeCronJob(dcConfig, kc.SanitizedName(), operatorNamespace, logger)
+			if err != nil {
+				logger.Info("Failed to generate Medusa purge backups cronjob definition", "error", err)
+				return result.Error(err)
+			}
+			purgeCronJob.SetLabels(labels.CleanedUpByLabels(kcKey))
+			recRes := reconciliation.ReconcileObject(ctx, remoteClient, r.DefaultDelay, *purgeCronJob)
+			switch {
+			case recRes.IsError():
+				return recRes
+			case recRes.IsRequeue():
+				return recRes
+			}
+		} else {
+			// if an existing purge cron job exists, delete it
+			cronJobName := medusa.MedusaPurgeCronJobName(kc.SanitizedName(), dcConfig.SanitizedName())
+			cronJobKey := types.NamespacedName{Namespace: operatorNamespace, Name: cronJobName}
+			cronJob := &batchv1.CronJob{}
+			if err := remoteClient.Get(ctx, cronJobKey, cronJob); err != nil {
+				// If the error is anything else but not found, fail the reconcile
+				if !errors.IsNotFound(err) {
+					logger.Error(err, "Failed to get Medusa purge backups cronjob")
+					return result.Error(err)
+				}
+			} else {
+				// The cron job exists, delete it
+				logger.Info("Deleting Medusa purge backups cronjob (may have been created before PurgeBackups was set to false")
+				if err := remoteClient.Delete(ctx, cronJob); err != nil {
+					logger.Info("Failed to delete Medusa purge backups cronjob", "error", err)
+					return result.Error(err)
+				}
+			}
 		}
 
 	} else {
@@ -120,6 +144,13 @@ func (r *K8ssandraClusterReconciler) reconcileMedusa(
 	return result.Continue()
 }
 
+func backupPurgeIsOn(purgeBackups *bool) bool {
+	if purgeBackups == nil {
+		return true
+	}
+	return *purgeBackups
+}
+
 // Generate a secret for Medusa or use the existing one if provided in the spec
 func (r *K8ssandraClusterReconciler) reconcileMedusaSecrets(
 	ctx context.Context,

docs/content/en/tasks/backup-restore/_index.md

@@ -473,7 +473,18 @@ spec:
 ```
 
 The above `CronJob` will be scheduled to run every day at midnight, and trigger a `MedusaTask` object creation to purge the backups.
+### Disabling the purge CronJob
+If you want to disable the automatic creation of the above purge `CronJob`, you can do so by setting the optional `purgeBackups` field to `false` in the `K8ssandraCluster` spec:
 
+```yaml
+apiVersion: k8ssandra.io/v1alpha1
+kind: K8ssandraCluster
+spec:
+  medusa:
+    purgeBackups: false
+ ```
+
+```yaml
 ## Deprecation Notice
 
 The `CassandraBackup` and `CassandraRestore` CRDs are removed in v1.6.0.

pkg/medusa/reconcile.go

@@ -496,7 +496,13 @@ func MedusaPurgeCronJobName(clusterName string, dcName string) string {
 
 func PurgeCronJob(dcConfig *cassandra.DatacenterConfig, clusterName, namespace string, logger logr.Logger) (*batchv1.CronJob, error) {
 	cronJobName := MedusaPurgeCronJobName(cassdcapi.CleanupForKubernetes(clusterName), dcConfig.SanitizedName())
-	logger.Info(fmt.Sprintf("Creating Medusa purge backups cronjob: %s", cronJobName))
+	// The MedusaTask must be created in the same namespace as the CassandraDatacenter. If the CassandraDatacenter
+	// does not have a namespace specified, use the namespace of the K8ssandraCluster.
+	dcNamespace := dcConfig.Meta.Namespace
+	if dcNamespace == "" {
+		dcNamespace = namespace
+	}
+	logger.Info(fmt.Sprintf("Creating Medusa purge backups cronjob definition: %s", cronJobName))
 	if len(cronJobName) > 253 {
 		return nil, fmt.Errorf("Medusa purge backups cronjob name too long (must be less than 253 characters). Length: %d, Job name: %s", len(cronJobName), cronJobName)
 	}
@@ -526,7 +532,7 @@ func PurgeCronJob(dcConfig *cassandra.DatacenterConfig, clusterName, namespace s
 									Command: []string{
 										"/bin/bash",
 										"-c",
-										createPurgeTaskStr(dcConfig.SanitizedName(), namespace),
+										createPurgeTaskStr(dcConfig.SanitizedName(), dcNamespace),
 									},
 								},
 							},

test/e2e/medusa_test.go

@@ -3,6 +3,8 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/utils/ptr"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -41,6 +43,16 @@ func createSingleMedusaJob(t *testing.T, ctx context.Context, namespace string,
 	checkDatacenterReady(t, ctx, dcKey, f)
 	checkMedusaContainersExist(t, ctx, namespace, dcKey, f, kc)
 	checkPurgeCronJobExists(t, ctx, namespace, dcKey, f, kc)
+
+	// Disable purges
+	err = f.Get(ctx, kcKey, kc)
+	require.NoError(err, "Error getting the K8ssandraCluster")
+	medusaPurgePatch := client.MergeFromWithOptions(kc.DeepCopy(), client.MergeFromWithOptimisticLock{})
+	kc.Spec.Medusa.PurgeBackups = ptr.To(false)
+	err = f.Client.Patch(ctx, kc, medusaPurgePatch)
+	require.NoError(err, "failed to patch K8ssandraCluster with purge modification in namespace %s", namespace)
+	checkPurgeCronJobDeleted(t, ctx, namespace, dcKey, f, kc)
+
 	createBackupJob(t, ctx, namespace, f, dcKey)
 	verifyBackupJobFinished(t, ctx, f, dcKey, backupKey)
 	restoreBackupJob(t, ctx, namespace, f, dcKey)
@@ -113,9 +125,9 @@ func createMultiDcSingleMedusaJob(t *testing.T, ctx context.Context, namespace s
 
 	checkDatacenterReady(t, ctx, dcKey, f)
 	checkMedusaContainersExist(t, ctx, namespace, dcKey, f, kc)
-	checkPurgeCronJobExists(t, ctx, namespace, dcKey, f, kc)
 	createBackupJob(t, ctx, namespace, f, dcKey)
 	verifyBackupJobFinished(t, ctx, f, dcKey, backupKey)
+	checkNoPurgeCronJob(t, ctx, namespace, dcKey, f, kc)
 }
 
 func checkBucketKeyPresent(t *testing.T, f *framework.E2eFramework, ctx context.Context, namespace string, k8sContext string, kc *k8ssandraapi.K8ssandraCluster) {
@@ -171,7 +183,7 @@ func checkPurgeCronJobExists(t *testing.T, ctx context.Context, namespace string
 	err := f.Get(ctx, dcKey, dc1)
 	// check medusa containers exist
 	require.NoError(err, "Error getting the CassandraDatacenter")
-	t.Log("Checking that all the Medusa related objects have been created and are in the expected state")
+	t.Log("Checking that the purge Cron Job exists")
 	// check that the cronjob exists
 	cronJob := &batchv1.CronJob{}
 	err = f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, medusapkg.MedusaPurgeCronJobName(kc.SanitizedName(), dc1.SanitizedName())), cronJob)
@@ -198,6 +210,36 @@ func checkPurgeCronJobExists(t *testing.T, ctx context.Context, namespace string
 	}, polling.medusaBackupDone.timeout, polling.medusaBackupDone.interval, "Medusa purge Job didn't finish within timeout")
 }
 
+func checkNoPurgeCronJob(t *testing.T, ctx context.Context, namespace string, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) {
+	require := require.New(t)
+	t.Log("Checking that the purge Cron Job doesn't exist")
+	// Get the Cassandra pod
+	dc1 := &cassdcapi.CassandraDatacenter{}
+	err := f.Get(ctx, dcKey, dc1)
+	// check medusa containers exist
+	require.NoError(err, "Error getting the CassandraDatacenter")
+	// ensure the cronjob was not created
+	cronJob := &batchv1.CronJob{}
+	err = f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, medusapkg.MedusaPurgeCronJobName(kc.SanitizedName(), dc1.SanitizedName())), cronJob)
+	require.Error(err, "Cronjob should not exist")
+}
+
+func checkPurgeCronJobDeleted(t *testing.T, ctx context.Context, namespace string, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) {
+	require := require.New(t)
+	// Get the Cassandra pod
+	dc1 := &cassdcapi.CassandraDatacenter{}
+	err := f.Get(ctx, dcKey, dc1)
+	t.Log("Checking that the purge Cron Job was deleted")
+	require.NoError(err, "Error getting the CassandraDatacenter")
+
+	require.Eventually(func() bool {
+		// ensure the cronjob was deleted
+		cronJob := &batchv1.CronJob{}
+		err = f.Get(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, medusapkg.MedusaPurgeCronJobName(kc.SanitizedName(), dc1.SanitizedName())), cronJob)
+		return errors.IsNotFound(err)
+	}, polling.medusaBackupDone.timeout, polling.medusaBackupDone.interval, "Medusa purge CronJob wasn't deleted within timeout")
+}
+
 func createBackupJob(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework, dcKey framework.ClusterKey) {
 	require := require.New(t)
 	t.Log("creating MedusaBackupJob")

test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml

@@ -69,6 +69,7 @@ spec:
           jvmOptions:
             heapSize: 512Mi
   medusa:
+    purgeBackups: false
     cassandraUserSecretRef:
       name: cluster2-medusa
     storageProperties: