ReplicatedSecrets can have a prefix per target (#1243)

* Allow a ReplicatedSecret replication target to define a prefix to de-conflict instances where the same source may target the same target multiple times due to presence of several clusters in the target namespace.

CHANGELOG/CHANGELOG-1.14.md

@@ -14,6 +14,7 @@ Changelog for the K8ssandra Operator, new PRs should update the `unreleased` sec
 When cutting a new release, update the `unreleased` heading to the tag being generated and date, like `## vX.Y.Z - YYYY-MM-DD` and create a new placeholder section for  `unreleased` entries.
 
 ## unreleased
+* [FEATURE] [#1242](https://github.com/k8ssandra/k8ssandra-operator/issues/1242) Allow for creation of replicated secrets with a prefix, so that we can distinguish between multiple secrets with the same origin but targeting different clusters.
 * [BUGFIX] [#1226](https://github.com/k8ssandra/k8ssandra-operator/issues/1226) Medusa purge cronjob should be created in the operator namespace
 * [BUGFIX] [#1141](https://github.com/k8ssandra/k8ssandra-operator/issues/1141) Use DC name override when naming secondary resources
 * [BUGFIX] [#1138](https://github.com/k8ssandra/k8ssandra-operator/issues/1138) Use cluster name override for metrics agent ConfigMap 

apis/replication/v1alpha1/replicatedsecret_types.go

@@ -51,6 +51,11 @@ type ReplicationTarget struct {
 	// Selector defines which clusters are targeted.
 	// +optional
 	// Selector *metav1.LabelSelector `json:"selector,omitempty"`
+
+	// TargetPrefix is the prefix to be used for the replicated secret in the target cluster. If left empty, the same name is used
+	// as the original secret.
+	// +optional
+	TargetPrefix string `json:"targetPrefix,omitempty"`
 }
 
 // ReplicatedSecretStatus defines the observed state of ReplicatedSecret

charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml

@@ -35641,6 +35641,11 @@ spec:
                         the data to in the target cluster. If left empty, current
                         namespace is used.
                       type: string
+                    targetPrefix:
+                      description: TargetPrefix is the prefix to be used for the replicated
+                        secret in the target cluster. If left empty, the same name
+                        is used as the original secret.
+                      type: string
                   type: object
                 type: array
               selector:

config/crd/bases/replication.k8ssandra.io_replicatedsecrets.yaml

@@ -49,6 +49,11 @@ spec:
                         the data to in the target cluster. If left empty, current
                         namespace is used.
                       type: string
+                    targetPrefix:
+                      description: TargetPrefix is the prefix to be used for the replicated
+                        secret in the target cluster. If left empty, the same name
+                        is used as the original secret.
+                      type: string
                   type: object
                 type: array
               selector:

controllers/replication/secret_controller.go

@@ -3,10 +3,11 @@ package replication
 import (
 	"context"
 	"fmt"
-	"github.com/k8ssandra/k8ssandra-operator/pkg/secret"
 	"strings"
 	"sync"
 
+	"github.com/k8ssandra/k8ssandra-operator/pkg/secret"
+
 	coreapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/clientcache"
@@ -225,14 +226,15 @@ func (s *SecretSyncController) Reconcile(ctx context.Context, req ctrl.Request)
 				namespace = target.Namespace
 			}
 			fetchedSecret := &corev1.Secret{}
-			if err = remoteClient.Get(ctx, types.NamespacedName{Name: sec.Name, Namespace: namespace}, fetchedSecret); err != nil {
+			if err = remoteClient.Get(ctx, types.NamespacedName{Name: getPrefixedSecretName(target.TargetPrefix, sec.Name), Namespace: namespace}, fetchedSecret); err != nil {
 				if errors.IsNotFound(err) {
 					logger.Info("Copying secret to target cluster", "Secret", sec.Name, "TargetContext", target)
 					// Create it
 					copiedSecret := sec.DeepCopy()
 					copiedSecret.Namespace = namespace
 					copiedSecret.ResourceVersion = ""
 					copiedSecret.OwnerReferences = []metav1.OwnerReference{}
+					copiedSecret.Name = getPrefixedSecretName(target.TargetPrefix, sec.Name)
 					if err = remoteClient.Create(ctx, copiedSecret); err != nil {
 						logger.Error(err, "Failed to sync secret to target cluster", "Secret", copiedSecret.Name, "TargetContext", target)
 						break TargetSecrets
@@ -252,7 +254,9 @@ func (s *SecretSyncController) Reconcile(ctx context.Context, req ctrl.Request)
 			if requiresUpdate(sec, fetchedSecret) {
 				logger.Info("Modifying secret in target cluster", "Secret", sec.Name, "TargetContext", target)
 				syncSecrets(sec, fetchedSecret)
-				if err = remoteClient.Update(ctx, fetchedSecret); err != nil {
+				copiedSecret := fetchedSecret.DeepCopy()
+				copiedSecret.Name = getPrefixedSecretName(target.TargetPrefix, sec.Name)
+				if err = remoteClient.Update(ctx, copiedSecret); err != nil {
 					logger.Error(err, "Failed to sync target secret for matching payloads", "Secret", fetchedSecret.Name, "TargetContext", target)
 					break TargetSecrets
 				}
@@ -433,3 +437,7 @@ func (s *SecretSyncController) initializeCache() error {
 	}
 	return nil
 }
+
+func getPrefixedSecretName(prefix string, secretName string) string {
+	return fmt.Sprintf("%s%s", prefix, secretName)
+}

controllers/replication/secret_controller_test.go

@@ -3,15 +3,16 @@ package replication
 import (
 	"context"
 	"fmt"
+	"testing"
+	"time"
+
 	"github.com/go-logr/logr"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/clientcache"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/config"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/secret"
 	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"testing"
-	"time"
 
 	coreapi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1"
 	api "github.com/k8ssandra/k8ssandra-operator/apis/replication/v1alpha1"
@@ -63,6 +64,7 @@ func TestSecretController(t *testing.T) {
 	t.Run("SingleClusterDoNothingToSecretsTest", testEnv.ControllerTest(ctx, wrongClusterIgnoreCopy))
 	t.Run("MultiClusterSyncSecretsTest", testEnv.ControllerTest(ctx, copySecretsFromClusterToCluster))
 	t.Run("VerifyFinalizerInMultiCluster", testEnv.ControllerTest(ctx, verifySecretIsDeleted))
+	t.Run("TargetSecretsPrefixTest", testEnv.ControllerTest(ctx, prefixedSecret))
 }
 
 // copySecretsFromClusterToCluster Tests:
@@ -94,7 +96,7 @@ func copySecretsFromClusterToCluster(t *testing.T, ctx context.Context, f *frame
 		return verifySecretsMatch(t, ctx, f.Client, []string{f.DataPlaneContexts[targetCopyToCluster]}, map[string]struct{}{
 			generatedSecrets[0].Name: empty,
 		}, generatedSecrets[0].Namespace)
-	}, timeout, interval)
+	}, timeout*3, interval)
 
 	t.Log("check that secret not match by replicated secret was not copied")
 	require.Never(func() bool {
@@ -472,3 +474,54 @@ func TestRequiresUpdate(t *testing.T) {
 	syncSecrets(orig, dest)
 	assert.False(requiresUpdate(orig, dest))
 }
+
+func prefixedSecret(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) {
+	require := require.New(t)
+	rsec := generateReplicatedSecret(f.DataPlaneContexts[0], namespace)
+	rsec.Spec.ReplicationTargets[0].TargetPrefix = "prefix-"
+	rsec.Name = "broke"
+	err := f.Client.Create(ctx, rsec)
+	require.NoError(err, "failed to create replicated secret to main cluster")
+
+	generatedSecrets := generateSecrets(namespace)
+	for i, s := range generatedSecrets {
+		s.Name = fmt.Sprintf("broken-secret-%d", i)
+		err := f.Client.Create(ctx, s)
+		require.NoError(err, "failed to create secret to main cluster")
+	}
+
+	t.Log("check that the secrets were copied to other cluster(s)")
+	localContext := f.Client
+	remoteContext := testEnv.Clients[f.DataPlaneContexts[0]]
+	require.Eventually(func() bool {
+		localSecret := &corev1.Secret{}
+		remoteSecret := &corev1.Secret{}
+		if err := localContext.Get(ctx, types.NamespacedName{Name: "broken-secret-0", Namespace: namespace}, localSecret); err != nil {
+			return false
+		}
+		nsList := &corev1.NamespaceList{}
+		require.NoError(remoteContext.List(ctx, nsList))
+		tempList := &corev1.SecretList{}
+		require.NoError(remoteContext.List(ctx, tempList, &client.ListOptions{
+			Namespace: namespace,
+		}))
+		if err := remoteContext.Get(ctx, types.NamespacedName{Name: "prefix-broken-secret-0", Namespace: namespace}, remoteSecret); err != nil {
+			return false
+		}
+		return string(localSecret.Data["key"]) == string(remoteSecret.Data["key"])
+	}, timeout*3, interval)
+
+	t.Log("update secret content")
+	localSecret := &corev1.Secret{}
+	require.NoError(localContext.Get(ctx, types.NamespacedName{Name: "broken-secret-0", Namespace: namespace}, localSecret))
+	localSecret.Data = map[string][]byte{"modifiedKey": []byte("newValue")}
+	require.NoError(localContext.Update(ctx, localSecret))
+	require.Eventually(func() bool {
+		remoteSecret := &corev1.Secret{}
+		if err := remoteContext.Get(ctx, types.NamespacedName{Name: "prefix-broken-secret-0", Namespace: namespace}, remoteSecret); err != nil {
+			return false
+		}
+		return string(remoteSecret.Data["modifiedKey"]) == string(localSecret.Data["modifiedKey"])
+	}, timeout*3, interval)
+
+}