diff --git a/CHANGELOG/CHANGELOG-1.18.md b/CHANGELOG/CHANGELOG-1.18.md index 58ab84827..f3c094353 100644 --- a/CHANGELOG/CHANGELOG-1.18.md +++ b/CHANGELOG/CHANGELOG-1.18.md @@ -14,3 +14,5 @@ Changelog for the K8ssandra Operator, new PRs should update the `unreleased` sec When cutting a new release, update the `unreleased` heading to the tag being generated and date, like `## vX.Y.Z - YYYY-MM-DD` and create a new placeholder section for `unreleased` entries. ## unreleased + +* [FEATURE] [#1310](https://github.com/k8ssandra/k8ssandra-operator/issues/1310) Enhance the MedusaBackupSchedule API to allow scheduling purge tasks \ No newline at end of file diff --git a/apis/medusa/v1alpha1/medusaschedule_types.go b/apis/medusa/v1alpha1/medusaschedule_types.go index 0bd6c94ca..a7af544b8 100644 --- a/apis/medusa/v1alpha1/medusaschedule_types.go +++ b/apis/medusa/v1alpha1/medusaschedule_types.go @@ -39,6 +39,12 @@ type MedusaBackupScheduleSpec struct { // The "Allow" property is only valid if all the other active Tasks have "Allow" as well. // +optional ConcurrencyPolicy batchv1.ConcurrencyPolicy `json:"concurrencyPolicy,omitempty"` + + // Specifies the type of operation to be performed + // +kubebuilder:validation:Enum=backup;purge + // +kubebuilder:default=backup + // +optional + OperationType string `json:"operationType,omitempty"` } // MedusaBackupScheduleStatus defines the observed state of MedusaBackupSchedule @@ -56,7 +62,7 @@ type MedusaBackupScheduleStatus struct { // +kubebuilder:printcolumn:name="Datacenter",type=string,JSONPath=".spec.backupSpec.cassandraDatacenter",description="Datacenter which the task targets" // +kubebuilder:printcolumn:name="ScheduledExecution",type="date",JSONPath=".status.nextSchedule",description="Next scheduled execution time" // +kubebuilder:printcolumn:name="LastExecution",type="date",JSONPath=".status.lastExecution",description="Previous execution time" -// +kubebuilder:printcolumn:name="BackupType",type="string",JSONPath=".spec.backupSpec.backupType",description="Type of backup" +// +kubebuilder:printcolumn:name="OperationType",type="string",JSONPath=".spec.operationType",description="Type of scheduled operation" // MedusaBackupSchedule is the Schema for the medusabackupschedules API type MedusaBackupSchedule struct { metav1.TypeMeta `json:",inline"` diff --git a/apis/medusa/v1alpha1/medusatask_types.go b/apis/medusa/v1alpha1/medusatask_types.go index 0dc62469f..784703e1d 100644 --- a/apis/medusa/v1alpha1/medusatask_types.go +++ b/apis/medusa/v1alpha1/medusatask_types.go @@ -90,6 +90,10 @@ const ( //+kubebuilder:subresource:status // MedusaTask is the Schema for the MedusaTasks API +// +kubebuilder:printcolumn:name="Datacenter",type=string,JSONPath=".spec.cassandraDatacenter",description="Datacenter which the task targets" +// +kubebuilder:printcolumn:name="Operation",type="string",JSONPath=".spec.operation",description="Type of operation" +// +kubebuilder:printcolumn:name="Start",type="date",JSONPath=".status.startTime",description="Start time" +// +kubebuilder:printcolumn:name="Finish",type="date",JSONPath=".status.finishTime",description="Finish time" type MedusaTask struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml index 587b2125d..7a404f056 100644 --- a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml +++ b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml @@ -31708,9 +31708,9 @@ spec: jsonPath: .status.lastExecution name: LastExecution type: date - - description: Type of backup - jsonPath: .spec.backupSpec.backupType - name: BackupType + - description: Type of scheduled operation + jsonPath: .spec.operationType + name: OperationType type: string name: v1alpha1 schema: @@ -31771,6 +31771,13 @@ spec: disabled: description: Disabled if set ensures this job is not scheduling anything type: boolean + operationType: + default: backup + description: Specifies the type of operation to be performed + enum: + - backup + - purge + type: string required: - backupSpec - cronSchedule @@ -32184,7 +32191,24 @@ spec: singular: medusatask scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: Datacenter which the task targets + jsonPath: .spec.cassandraDatacenter + name: Datacenter + type: string + - description: Type of operation + jsonPath: .spec.operation + name: Operation + type: string + - description: Start time + jsonPath: .status.startTime + name: Start + type: date + - description: Finish time + jsonPath: .status.finishTime + name: Finish + type: date + name: v1alpha1 schema: openAPIV3Schema: description: MedusaTask is the Schema for the MedusaTasks API diff --git a/config/crd/bases/medusa.k8ssandra.io_medusabackupschedules.yaml b/config/crd/bases/medusa.k8ssandra.io_medusabackupschedules.yaml index 978422ef1..88c441a24 100644 --- a/config/crd/bases/medusa.k8ssandra.io_medusabackupschedules.yaml +++ b/config/crd/bases/medusa.k8ssandra.io_medusabackupschedules.yaml @@ -27,9 +27,9 @@ spec: jsonPath: .status.lastExecution name: LastExecution type: date - - description: Type of backup - jsonPath: .spec.backupSpec.backupType - name: BackupType + - description: Type of scheduled operation + jsonPath: .spec.operationType + name: OperationType type: string name: v1alpha1 schema: @@ -90,6 +90,13 @@ spec: disabled: description: Disabled if set ensures this job is not scheduling anything type: boolean + operationType: + default: backup + description: Specifies the type of operation to be performed + enum: + - backup + - purge + type: string required: - backupSpec - cronSchedule diff --git a/config/crd/bases/medusa.k8ssandra.io_medusatasks.yaml b/config/crd/bases/medusa.k8ssandra.io_medusatasks.yaml index 6b0483767..a2a0eb2d3 100644 --- a/config/crd/bases/medusa.k8ssandra.io_medusatasks.yaml +++ b/config/crd/bases/medusa.k8ssandra.io_medusatasks.yaml @@ -14,7 +14,24 @@ spec: singular: medusatask scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: Datacenter which the task targets + jsonPath: .spec.cassandraDatacenter + name: Datacenter + type: string + - description: Type of operation + jsonPath: .spec.operation + name: Operation + type: string + - description: Start time + jsonPath: .status.startTime + name: Start + type: date + - description: Finish time + jsonPath: .status.finishTime + name: Finish + type: date + name: v1alpha1 schema: openAPIV3Schema: description: MedusaTask is the Schema for the MedusaTasks API diff --git a/controllers/medusa/medusabackupschedule_controller.go b/controllers/medusa/medusabackupschedule_controller.go index 5a4967cb6..298fbf1ca 100644 --- a/controllers/medusa/medusabackupschedule_controller.go +++ b/controllers/medusa/medusabackupschedule_controller.go @@ -19,6 +19,7 @@ package medusa import ( "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "time" batchv1 "k8s.io/api/batch/v1" @@ -80,6 +81,20 @@ func (r *MedusaBackupScheduleReconciler) Reconcile(ctx context.Context, req ctrl return ctrl.Result{}, err } + // Set an owner reference on the task so that it can be cleaned up when the cassandra datacenter is deleted + if backupSchedule.OwnerReferences == nil { + if err = controllerutil.SetControllerReference(dc, backupSchedule, r.Scheme); err != nil { + logger.Error(err, "failed to set controller reference", "CassandraDatacenter", dcKey) + return ctrl.Result{}, err + } + if err = r.Update(ctx, backupSchedule); err != nil { + logger.Error(err, "failed to update task with owner reference", "CassandraDatacenter", dcKey) + return ctrl.Result{}, err + } else { + logger.Info("updated task with owner reference", "CassandraDatacenter", dcKey) + } + } + defaults(backupSchedule) previousExecution, err := getPreviousExecutionTime(ctx, backupSchedule) @@ -93,22 +108,24 @@ func (r *MedusaBackupScheduleReconciler) Reconcile(ctx context.Context, req ctrl nextExecution := sched.Next(previousExecution).UTC() createBackup := false + createPurge := false if nextExecution.Before(now) { if backupSchedule.Spec.ConcurrencyPolicy == batchv1.ForbidConcurrent { - if activeTasks, err := r.activeTasks(backupSchedule, dc); err != nil { + if activeTasks, err := r.activeTasks(backupSchedule, dc, backupSchedule.Spec.OperationType); err != nil { logger.V(1).Info("failed to get activeTasks", "error", err) return ctrl.Result{}, err } else { - if len(activeTasks) > 0 { - logger.V(1).Info("Postponing backup schedule due to existing active backups", "MedusaBackupSchedule", req.NamespacedName) + if activeTasks > 0 { + logger.V(1).Info("Postponing backup schedule due to an unfinished existing job", "MedusaBackupSchedule", req.NamespacedName) return ctrl.Result{RequeueAfter: 1 * time.Minute}, nil } } } nextExecution = sched.Next(now) previousExecution = now - createBackup = true && !backupSchedule.Spec.Disabled + createBackup = true && !backupSchedule.Spec.Disabled && (backupSchedule.Spec.OperationType == "backup" || backupSchedule.Spec.OperationType == "") + createPurge = true && !backupSchedule.Spec.Disabled && (backupSchedule.Spec.OperationType == string(medusav1alpha1.OperationTypePurge)) } // Update the status if there are modifications @@ -140,6 +157,27 @@ func (r *MedusaBackupScheduleReconciler) Reconcile(ctx context.Context, req ctrl } } + if createPurge { + logger.V(1).Info("Scheduled time has been reached, creating a backup purge job", "MedusaBackupSchedule", req.NamespacedName) + generatedName := fmt.Sprintf("%s-%d", backupSchedule.Name, now.Unix()) + purgeJob := &medusav1alpha1.MedusaTask{ + ObjectMeta: metav1.ObjectMeta{ + Name: generatedName, + Namespace: backupSchedule.Namespace, + Labels: dc.GetDatacenterLabels(), + }, + Spec: medusav1alpha1.MedusaTaskSpec{ + CassandraDatacenter: backupSchedule.Spec.BackupSpec.CassandraDatacenter, + Operation: medusav1alpha1.OperationTypePurge, + }, + } + + if err := r.Client.Create(ctx, purgeJob); err != nil { + // We've already updated the Status times.. we'll miss this job now? + return ctrl.Result{}, err + } + } + nextRunTime := nextExecution.Sub(now) logger.V(1).Info("Requeing for next scheduled event", "nextRuntime", nextRunTime.String()) return ctrl.Result{RequeueAfter: nextRunTime}, nil @@ -162,19 +200,35 @@ func defaults(backupSchedule *medusav1alpha1.MedusaBackupSchedule) { } } -func (r *MedusaBackupScheduleReconciler) activeTasks(backupSchedule *medusav1alpha1.MedusaBackupSchedule, dc *cassdcapi.CassandraDatacenter) ([]medusav1alpha1.MedusaBackupJob, error) { - backupJobs := &medusav1alpha1.MedusaBackupJobList{} - if err := r.Client.List(context.Background(), backupJobs, client.InNamespace(backupSchedule.Namespace), client.MatchingLabels(dc.GetDatacenterLabels())); err != nil { - return nil, err - } - activeJobs := make([]medusav1alpha1.MedusaBackupJob, 0) - for _, job := range backupJobs.Items { - if job.Status.FinishTime.IsZero() { - activeJobs = append(activeJobs, job) +func (r *MedusaBackupScheduleReconciler) activeTasks(backupSchedule *medusav1alpha1.MedusaBackupSchedule, dc *cassdcapi.CassandraDatacenter, operationType string) (int, error) { + if operationType == "" || operationType == "backup" { + backupJobs := &medusav1alpha1.MedusaBackupJobList{} + if err := r.Client.List(context.Background(), backupJobs, client.InNamespace(backupSchedule.Namespace), client.MatchingLabels(dc.GetDatacenterLabels())); err != nil { + return 0, err + } + activeJobs := make([]medusav1alpha1.MedusaBackupJob, 0) + for _, job := range backupJobs.Items { + if job.Status.FinishTime.IsZero() { + activeJobs = append(activeJobs, job) + } } - } - return activeJobs, nil + return len(activeJobs), nil + } else if operationType == string(medusav1alpha1.OperationTypePurge) { + medusaTasks := &medusav1alpha1.MedusaTaskList{} + if err := r.Client.List(context.Background(), medusaTasks, client.InNamespace(backupSchedule.Namespace), client.MatchingLabels(dc.GetDatacenterLabels())); err != nil { + return 0, err + } + activeJobs := make([]medusav1alpha1.MedusaTask, 0) + for _, job := range medusaTasks.Items { + if job.Spec.Operation == medusav1alpha1.OperationTypePurge && job.Status.FinishTime.IsZero() { + activeJobs = append(activeJobs, job) + } + } + + return len(activeJobs), nil + } + return 0, fmt.Errorf("unknown operation type %s", operationType) } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/medusa/medusabackupschedule_controller_test.go b/controllers/medusa/medusabackupschedule_controller_test.go index e8fe547b1..2e3df1318 100644 --- a/controllers/medusa/medusabackupschedule_controller_test.go +++ b/controllers/medusa/medusabackupschedule_controller_test.go @@ -2,6 +2,7 @@ package medusa import ( "context" + cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" "testing" "time" @@ -24,160 +25,316 @@ func (f *FakeClock) Now() time.Time { var _ Clock = &FakeClock{} -// func TestScheduler(t *testing.T) { -// require := require.New(t) -// require.NoError(medusav1alpha1.AddToScheme(scheme.Scheme)) -// require.NoError(cassdcapi.AddToScheme(scheme.Scheme)) - -// fClock := &FakeClock{} - -// dc := cassdcapi.CassandraDatacenter{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: "dc1", -// Namespace: "test-ns", -// }, -// Spec: cassdcapi.CassandraDatacenterSpec{}, -// } - -// // To manipulate time and requeue, we use fakeclient here instead of envtest -// backupSchedule := &medusav1alpha1.MedusaBackupSchedule{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: "test-schedule", -// Namespace: "test-ns", -// }, -// Spec: medusav1alpha1.MedusaBackupScheduleSpec{ -// CronSchedule: "* * * * *", -// BackupSpec: medusav1alpha1.MedusaBackupJobSpec{ -// CassandraDatacenter: "dc1", -// Type: "differential", -// }, -// }, -// } - -// fakeClient := fake.NewClientBuilder(). -// WithRuntimeObjects(backupSchedule, &dc). -// WithScheme(scheme.Scheme). -// Build() - -// nsName := types.NamespacedName{ -// Name: backupSchedule.Name, -// Namespace: backupSchedule.Namespace, -// } - -// r := &MedusaBackupScheduleReconciler{ -// Client: fakeClient, -// Scheme: scheme.Scheme, -// Clock: fClock, -// } - -// res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) -// require.NoError(err) -// require.True(res.RequeueAfter > 0) - -// fClock.currentTime = fClock.currentTime.Add(1 * time.Minute).Add(1 * time.Second) - -// _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) -// require.NoError(err) -// require.True(res.RequeueAfter > 0) - -// // We should have a backup now.. -// backupRequests := medusav1alpha1.MedusaBackupJobList{} -// err = fakeClient.List(context.TODO(), &backupRequests) -// require.NoError(err) -// require.Equal(1, len(backupRequests.Items)) - -// // Ensure the backup object is created correctly -// backup := backupRequests.Items[0] -// require.Equal(backupSchedule.Spec.BackupSpec.CassandraDatacenter, backup.Spec.CassandraDatacenter) -// require.Equal(backupSchedule.Spec.BackupSpec.Type, backup.Spec.Type) - -// // Verify the Status of the BackupSchedule is modified and the object is requeued -// backupScheduleLive := &medusav1alpha1.MedusaBackupSchedule{} -// err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) -// require.NoError(err) - -// require.Equal(fClock.currentTime, backupScheduleLive.Status.LastExecution.Time.UTC()) -// require.Equal(time.Time{}.Add(2*time.Minute), backupScheduleLive.Status.NextSchedule.Time.UTC()) - -// // Test that next invocation also works -// fClock.currentTime = fClock.currentTime.Add(1 * time.Minute) - -// _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) -// require.NoError(err) -// require.True(res.RequeueAfter > 0) - -// // We should not have more than 1, since we never set the previous one as finished -// backupRequests = medusav1alpha1.MedusaBackupJobList{} -// err = fakeClient.List(context.TODO(), &backupRequests) -// require.NoError(err) -// require.Equal(1, len(backupRequests.Items)) - -// // Mark the first one as finished and try again -// backup.Status.FinishTime = metav1.NewTime(fClock.currentTime) -// require.NoError(fakeClient.Update(context.TODO(), &backup)) - -// backupRequests = medusav1alpha1.MedusaBackupJobList{} -// err = fakeClient.List(context.TODO(), &backupRequests) -// require.NoError(err) -// require.Equal(1, len(backupRequests.Items)) - -// _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) -// require.NoError(err) -// require.True(res.RequeueAfter > 0) - -// backupRequests = medusav1alpha1.MedusaBackupJobList{} -// err = fakeClient.List(context.TODO(), &backupRequests) -// require.NoError(err) -// require.Equal(2, len(backupRequests.Items)) - -// for _, backup := range backupRequests.Items { -// backup.Status.FinishTime = metav1.NewTime(fClock.currentTime) -// require.NoError(fakeClient.Update(context.TODO(), &backup)) -// } - -// // Verify that invocating again without reaching the next time does not generate another backup -// // or modify the Status -// backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} -// err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) -// require.NoError(err) - -// previousExecutionTime := backupScheduleLive.Status.LastExecution -// fClock.currentTime = fClock.currentTime.Add(30 * time.Second) -// _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) -// require.NoError(err) -// require.True(res.RequeueAfter > 0) - -// backupRequests = medusav1alpha1.MedusaBackupJobList{} -// err = fakeClient.List(context.TODO(), &backupRequests) -// require.NoError(err) -// require.Equal(2, len(backupRequests.Items)) - -// backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} -// err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) -// require.NoError(err) -// require.Equal(previousExecutionTime, backupScheduleLive.Status.LastExecution) - -// // Set to disabled and verify that the backups aren't scheduled anymore - but the status is updated -// backupScheduleLive.Spec.Disabled = true -// err = fakeClient.Update(context.TODO(), backupScheduleLive) -// require.NoError(err) - -// fClock.currentTime = fClock.currentTime.Add(1 * time.Minute) - -// _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) -// require.NoError(err) -// require.True(res.RequeueAfter > 0) - -// backupRequests = medusav1alpha1.MedusaBackupJobList{} -// err = fakeClient.List(context.TODO(), &backupRequests) -// require.NoError(err) -// require.Equal(2, len(backupRequests.Items)) // No new items were created - -// backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} -// err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) -// require.NoError(err) -// require.True(previousExecutionTime.Before(&backupScheduleLive.Status.LastExecution)) // Status time is still updated -// } +func TestScheduler(t *testing.T) { + require := require.New(t) + require.NoError(medusav1alpha1.AddToScheme(scheme.Scheme)) + require.NoError(cassdcapi.AddToScheme(scheme.Scheme)) + + fClock := &FakeClock{} + + dc := cassdcapi.CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dc1", + Namespace: "test-ns", + }, + Spec: cassdcapi.CassandraDatacenterSpec{}, + } + + // To manipulate time and requeue, we use fakeclient here instead of envtest + backupSchedule := &medusav1alpha1.MedusaBackupSchedule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-schedule", + Namespace: "test-ns", + }, + Spec: medusav1alpha1.MedusaBackupScheduleSpec{ + CronSchedule: "* * * * *", + BackupSpec: medusav1alpha1.MedusaBackupJobSpec{ + CassandraDatacenter: "dc1", + Type: "differential", + }, + }, + } + + fakeClient := fake.NewClientBuilder(). + WithObjects(backupSchedule, &dc). + WithStatusSubresource(backupSchedule). + WithScheme(scheme.Scheme). + Build() + + nsName := types.NamespacedName{ + Name: backupSchedule.Name, + Namespace: backupSchedule.Namespace, + } + + r := &MedusaBackupScheduleReconciler{ + Client: fakeClient, + Scheme: scheme.Scheme, + Clock: fClock, + } + + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + fClock.currentTime = fClock.currentTime.Add(1 * time.Minute).Add(1 * time.Second) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + // We should have a backup now.. + backupRequests := medusav1alpha1.MedusaBackupJobList{} + err = fakeClient.List(context.TODO(), &backupRequests) + require.NoError(err) + require.Equal(1, len(backupRequests.Items)) + + // Ensure the backup object is created correctly + backup := backupRequests.Items[0] + require.Equal(backupSchedule.Spec.BackupSpec.CassandraDatacenter, backup.Spec.CassandraDatacenter) + require.Equal(backupSchedule.Spec.BackupSpec.Type, backup.Spec.Type) + + // Verify the Status of the BackupSchedule is modified and the object is requeued + backupScheduleLive := &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + + require.Equal(fClock.currentTime, backupScheduleLive.Status.LastExecution.Time.UTC()) + require.Equal(time.Time{}.Add(2*time.Minute), backupScheduleLive.Status.NextSchedule.Time.UTC()) + + // Test that next invocation also works + fClock.currentTime = fClock.currentTime.Add(1 * time.Minute) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + // We should not have more than 1, since we never set the previous one as finished + backupRequests = medusav1alpha1.MedusaBackupJobList{} + err = fakeClient.List(context.TODO(), &backupRequests) + require.NoError(err) + require.Equal(1, len(backupRequests.Items)) + + // Mark the first one as finished and try again + backup.Status.FinishTime = metav1.NewTime(fClock.currentTime) + require.NoError(fakeClient.Update(context.TODO(), &backup)) + + backupRequests = medusav1alpha1.MedusaBackupJobList{} + err = fakeClient.List(context.TODO(), &backupRequests) + require.NoError(err) + require.Equal(1, len(backupRequests.Items)) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + backupRequests = medusav1alpha1.MedusaBackupJobList{} + err = fakeClient.List(context.TODO(), &backupRequests) + require.NoError(err) + require.Equal(2, len(backupRequests.Items)) + + for _, backup := range backupRequests.Items { + backup.Status.FinishTime = metav1.NewTime(fClock.currentTime) + require.NoError(fakeClient.Update(context.TODO(), &backup)) + } + + // Verify that invocating again without reaching the next time does not generate another backup + // or modify the Status + backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + + previousExecutionTime := backupScheduleLive.Status.LastExecution + fClock.currentTime = fClock.currentTime.Add(30 * time.Second) + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + backupRequests = medusav1alpha1.MedusaBackupJobList{} + err = fakeClient.List(context.TODO(), &backupRequests) + require.NoError(err) + require.Equal(2, len(backupRequests.Items)) + + backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + require.Equal(previousExecutionTime, backupScheduleLive.Status.LastExecution) + + // Set to disabled and verify that the backups aren't scheduled anymore - but the status is updated + backupScheduleLive.Spec.Disabled = true + err = fakeClient.Update(context.TODO(), backupScheduleLive) + require.NoError(err) + + fClock.currentTime = fClock.currentTime.Add(1 * time.Minute) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + backupRequests = medusav1alpha1.MedusaBackupJobList{} + err = fakeClient.List(context.TODO(), &backupRequests) + require.NoError(err) + require.Equal(2, len(backupRequests.Items)) // No new items were created + + backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + require.True(previousExecutionTime.Before(&backupScheduleLive.Status.LastExecution)) // Status time is still updated +} + +func TestPurgeScheduler(t *testing.T) { + require := require.New(t) + require.NoError(medusav1alpha1.AddToScheme(scheme.Scheme)) + require.NoError(cassdcapi.AddToScheme(scheme.Scheme)) + + fClock := &FakeClock{} + + dc := cassdcapi.CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dc1", + Namespace: "test-ns", + }, + Spec: cassdcapi.CassandraDatacenterSpec{}, + } + + // To manipulate time and requeue, we use fakeclient here instead of envtest + purgeSchedule := &medusav1alpha1.MedusaBackupSchedule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-purge-schedule", + Namespace: "test-ns", + }, + Spec: medusav1alpha1.MedusaBackupScheduleSpec{ + CronSchedule: "* * * * *", + BackupSpec: medusav1alpha1.MedusaBackupJobSpec{ + CassandraDatacenter: "dc1", + }, + OperationType: "purge", + }, + } + + fakeClient := fake.NewClientBuilder(). + WithObjects(purgeSchedule, &dc). + WithStatusSubresource(purgeSchedule). + WithScheme(scheme.Scheme). + Build() + + nsName := types.NamespacedName{ + Name: purgeSchedule.Name, + Namespace: purgeSchedule.Namespace, + } + + r := &MedusaBackupScheduleReconciler{ + Client: fakeClient, + Scheme: scheme.Scheme, + Clock: fClock, + } + + res, err := r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + fClock.currentTime = fClock.currentTime.Add(1 * time.Minute).Add(1 * time.Second) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + // We should have a backup now.. + purgeTasks := medusav1alpha1.MedusaTaskList{} + err = fakeClient.List(context.TODO(), &purgeTasks) + require.NoError(err) + require.Equal(1, len(purgeTasks.Items)) + + // Ensure the backup object is created correctly + backup := purgeTasks.Items[0] + require.Equal(purgeSchedule.Spec.BackupSpec.CassandraDatacenter, backup.Spec.CassandraDatacenter) + + // Verify the Status of the BackupSchedule is modified and the object is requeued + backupScheduleLive := &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + + require.Equal(fClock.currentTime, backupScheduleLive.Status.LastExecution.Time.UTC()) + require.Equal(time.Time{}.Add(2*time.Minute), backupScheduleLive.Status.NextSchedule.Time.UTC()) + + // Test that next invocation also works + fClock.currentTime = fClock.currentTime.Add(1 * time.Minute) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + // We should not have more than 1, since we never set the previous one as finished + purgeTasks = medusav1alpha1.MedusaTaskList{} + err = fakeClient.List(context.TODO(), &purgeTasks) + require.NoError(err) + require.Equal(1, len(purgeTasks.Items)) + + // Mark the first one as finished and try again + backup.Status.FinishTime = metav1.NewTime(fClock.currentTime) + require.NoError(fakeClient.Update(context.TODO(), &backup)) + + purgeTasks = medusav1alpha1.MedusaTaskList{} + err = fakeClient.List(context.TODO(), &purgeTasks) + require.NoError(err) + require.Equal(1, len(purgeTasks.Items)) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + purgeTasks = medusav1alpha1.MedusaTaskList{} + err = fakeClient.List(context.TODO(), &purgeTasks) + require.NoError(err) + require.Equal(2, len(purgeTasks.Items)) + + for _, backup := range purgeTasks.Items { + backup.Status.FinishTime = metav1.NewTime(fClock.currentTime) + require.NoError(fakeClient.Update(context.TODO(), &backup)) + } + + // Verify that invocating again without reaching the next time does not generate another backup + // or modify the Status + backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + + previousExecutionTime := backupScheduleLive.Status.LastExecution + fClock.currentTime = fClock.currentTime.Add(30 * time.Second) + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + purgeTasks = medusav1alpha1.MedusaTaskList{} + err = fakeClient.List(context.TODO(), &purgeTasks) + require.NoError(err) + require.Equal(2, len(purgeTasks.Items)) + + backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + require.Equal(previousExecutionTime, backupScheduleLive.Status.LastExecution) + + // Set to disabled and verify that the backups aren't scheduled anymore - but the status is updated + backupScheduleLive.Spec.Disabled = true + err = fakeClient.Update(context.TODO(), backupScheduleLive) + require.NoError(err) + + fClock.currentTime = fClock.currentTime.Add(1 * time.Minute) + + _, err = r.Reconcile(context.TODO(), reconcile.Request{NamespacedName: nsName}) + require.NoError(err) + require.True(res.RequeueAfter > 0) + + purgeTasks = medusav1alpha1.MedusaTaskList{} + err = fakeClient.List(context.TODO(), &purgeTasks) + require.NoError(err) + require.Equal(2, len(purgeTasks.Items)) // No new items were created + + backupScheduleLive = &medusav1alpha1.MedusaBackupSchedule{} + err = fakeClient.Get(context.TODO(), nsName, backupScheduleLive) + require.NoError(err) + require.True(previousExecutionTime.Before(&backupScheduleLive.Status.LastExecution)) // Status time is still updated +} func TestSchedulerParseError(t *testing.T) { require := require.New(t) diff --git a/test/e2e/medusa_test.go b/test/e2e/medusa_test.go index 32d05e5a8..14cca68a6 100644 --- a/test/e2e/medusa_test.go +++ b/test/e2e/medusa_test.go @@ -6,6 +6,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/utils/ptr" "testing" + "time" "github.com/stretchr/testify/assert" @@ -55,6 +56,7 @@ func createSingleMedusaJob(t *testing.T, ctx context.Context, namespace string, createBackupJob(t, ctx, namespace, f, dcKey) verifyBackupJobFinished(t, ctx, f, dcKey, backupKey) + checkPurgeTaskWasCreated(t, ctx, namespace, dcKey, f, kc) restoreBackupJob(t, ctx, namespace, f, dcKey) verifyRestoreJobFinished(t, ctx, f, dcKey, backupKey) @@ -240,6 +242,30 @@ func checkPurgeCronJobDeleted(t *testing.T, ctx context.Context, namespace strin }, polling.medusaBackupDone.timeout, polling.medusaBackupDone.interval, "Medusa purge CronJob wasn't deleted within timeout") } +func checkPurgeTaskWasCreated(t *testing.T, ctx context.Context, namespace string, dcKey framework.ClusterKey, f *framework.E2eFramework, kc *api.K8ssandraCluster) { + require := require.New(t) + // list MedusaTask objects + t.Log("Checking that the purge task was created") + require.Eventually(func() bool { + medusaTasks := &medusa.MedusaTaskList{} + err := f.List(ctx, framework.NewClusterKey(dcKey.K8sContext, namespace, ""), medusaTasks) + if err != nil { + t.Logf("failed to list MedusaTasks: %v", err) + return false + } + // check that the task is a purge task + found := false + for _, task := range medusaTasks.Items { + if task.Spec.Operation == medusa.OperationTypePurge && task.Spec.CassandraDatacenter == dcKey.Name { + found = true + break + } + } + return found + }, 2*time.Minute, 5*time.Second, "Medusa purge task wasn't created within timeout") + +} + func createBackupJob(t *testing.T, ctx context.Context, namespace string, f *framework.E2eFramework, dcKey framework.ClusterKey) { require := require.New(t) t.Log("creating MedusaBackupJob") diff --git a/test/testdata/fixtures/single-dc-encryption-medusa/kustomization.yaml b/test/testdata/fixtures/single-dc-encryption-medusa/kustomization.yaml index 2ac1a3634..291d36e80 100644 --- a/test/testdata/fixtures/single-dc-encryption-medusa/kustomization.yaml +++ b/test/testdata/fixtures/single-dc-encryption-medusa/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - k8ssandra.yaml + - medusa-purge.yaml diff --git a/test/testdata/fixtures/single-dc-encryption-medusa/medusa-purge.yaml b/test/testdata/fixtures/single-dc-encryption-medusa/medusa-purge.yaml new file mode 100644 index 000000000..abd97267e --- /dev/null +++ b/test/testdata/fixtures/single-dc-encryption-medusa/medusa-purge.yaml @@ -0,0 +1,10 @@ +apiVersion: medusa.k8ssandra.io/v1alpha1 +kind: MedusaBackupSchedule +metadata: + name: purge-schedule + namespace: k8ssandra-operator +spec: + backupSpec: + cassandraDatacenter: dc1 + cronSchedule: '* * * * *' + operationType: purge \ No newline at end of file