Run a cleanup Job when ManilaShare is scaled down

When a backend config is removed, the "service" database entry in manila needs
to be adjusted to account for the removal. This patch introduces a Job that
runs a manila-manage command every time a share is scaled down. The main job
definition is now more flexible and we can pass a TTL in case we want to
customize the time the job is kept.

Signed-off-by: Francesco Pantano <[email protected]>

controllers/manila_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 
 	"github.com/go-logr/logr"
 	memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1"
@@ -370,6 +371,7 @@ func (r *ManilaReconciler) reconcileInit(
 		instance,
 		serviceLabels,
 		serviceAnnotations,
+		nil,
 		manila.DBSyncJobName,
 		manila.DBSyncCommand,
 	)
@@ -821,7 +823,7 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila
 	instance.Status.Conditions.MarkTrue(condition.CronJobReadyCondition, condition.CronJobReadyMessage)
 	// create CronJob - end
 
-	cleanJob, err := r.shareCleanup(ctx, instance)
+	cleanJob, hash, err := r.shareCleanup(ctx, instance)
 	if err != nil {
 		return ctrl.Result{}, err
 	}
@@ -831,26 +833,23 @@ func (r *ManilaReconciler) reconcileNormal(ctx context.Context, instance *manila
 			instance,
 			serviceLabels,
 			serviceAnnotations,
-			manila.SvcCleanupJobName,
+			ptr.To(manila.TTL),
+			fmt.Sprintf("%s-%s", manila.SvcCleanupJobName, hash[:manila.TruncateHash]),
 			manila.SvcCleanupCommand,
 		)
-		svcCleanupHash := instance.Status.Hash[manilav1beta1.SvcCleanupHash]
 		shareCleanupJob := job.NewJob(
 			jobDef,
 			manilav1beta1.SvcCleanupHash,
-			instance.Spec.PreserveJobs,
+			false,
 			manila.ShortDuration,
-			svcCleanupHash,
+			"",
 		)
 		ctrlResult, err := shareCleanupJob.DoJob(
 			ctx,
 			helper,
 		)
 		if err != nil {
-			return ctrl.Result{}, err
-		}
-		if (ctrlResult != ctrl.Result{}) {
-			return ctrlResult, nil
+			return ctrlResult, err
 		}
 	}
 	r.Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name))
@@ -1286,34 +1285,40 @@ func (r *ManilaReconciler) checkManilaShareGeneration(
 func (r *ManilaReconciler) shareCleanup(
 	ctx context.Context,
 	instance *manilav1beta1.Manila,
-) (bool, error) {
+) (bool, string, error) {
+	cleanJob := false
+	var deletedShares = []string{}
 	// Generate a list of share CRs
 	shares := &manilav1beta1.ManilaShareList{}
-	cleanJob := false
+
 	listOpts := []client.ListOption{
 		client.InNamespace(instance.Namespace),
 	}
 	if err := r.Client.List(ctx, shares, listOpts...); err != nil {
 		r.Log.Error(err, "Unable to retrieve Manila Share CRs %v")
-		return cleanJob, nil
+		return cleanJob, "", nil
 	}
 	for _, share := range shares.Items {
 		// Skip shares CRs that we don't own
 		if manila.GetOwningManilaName(&share) != instance.Name {
 			continue
 		}
-
 		// Delete the manilaShare if it's no longer in the spec
 		_, exists := instance.Spec.ManilaShares[share.ShareName()]
 		if !exists && share.DeletionTimestamp.IsZero() {
 			err := r.Client.Delete(ctx, &share)
 			if err != nil && !k8s_errors.IsNotFound(err) {
 				err = fmt.Errorf("Error cleaning up %s: %w", share.Name, err)
-				return cleanJob, err
+				return cleanJob, "", err
 			}
 			delete(instance.Status.ManilaSharesReadyCounts, share.ShareName())
+			deletedShares = append(deletedShares, share.ShareName())
 			cleanJob = true
 		}
 	}
-	return cleanJob, nil
+	hash, err := manila.SharesListHash(deletedShares)
+	if err != nil {
+		return false, hash, err
+	}
+	return cleanJob, hash, nil
 }

pkg/manila/const.go

@@ -87,6 +87,10 @@ const (
 	SvcCleanupJobName = "service-cleanup"
 	// SvcCleanupCommand -
 	SvcCleanupCommand = "/usr/bin/manila-manage --config-dir /etc/manila/manila.conf.d service cleanup"
+	// TruncateHash -
+	TruncateHash int = 8
+	// TTL -
+	TTL int32 = 5 * 60 // 5 minutes
 )
 
 // DbsyncPropagation keeps track of the DBSync Service Propagation Type

pkg/manila/funcs.go

@@ -3,6 +3,7 @@ package manila
 import (
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -55,3 +56,13 @@ func GetPodAffinity(componentName string) *corev1.Affinity {
 		corev1.LabelHostname,
 	)
 }
+
+// SharesListHash - Given a list of share names passed as an array of strings,
+// it returns an hash that is currently used to build the job name
+func SharesListHash(shareNames []string) (string, error) {
+	hash, err := util.ObjectHash(shareNames)
+	if err != nil {
+		return hash, err
+	}
+	return hash, err
+}

pkg/manila/job.go

@@ -9,11 +9,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// ManilaJob func
+// Job func
 func Job(
 	instance *manilav1.Manila,
 	labels map[string]string,
 	annotations map[string]string,
+	ttl *int32,
 	jobName string,
 	jobCommand string,
 ) *batchv1.Job {
@@ -82,7 +83,6 @@ func Job(
 
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
-			//Name:      instance.Name + "-db-sync",
 			Name:      fmt.Sprintf("%s-%s", instance.Name, jobName),
 			Namespace: instance.Namespace,
 			Labels:    labels,
@@ -113,6 +113,9 @@ func Job(
 			},
 		},
 	}
-
+	if ttl != nil {
+		// Setting TTL to delete the job after it has completed
+		job.Spec.TTLSecondsAfterFinished = ttl
+	}
 	return job
 }