From 3aacb8e74d69c9491f0b07f7248712d9cdcd0ebd Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Tue, 19 Dec 2023 15:24:32 +1100 Subject: [PATCH 1/8] Add AdditionalAnnotations field and logic to reconcile it. --- .../v1beta1/cassandradatacenter_types.go | 3 + .../control/cassandratask_controller.go | 2 +- pkg/oplabels/labels.go | 8 ++- .../construct_podtemplatespec.go | 2 +- .../construct_podtemplatespec_test.go | 30 +++++--- pkg/reconciliation/construct_service.go | 8 +-- pkg/reconciliation/construct_service_test.go | 69 ++++++++++++++++++- pkg/reconciliation/construct_statefulset.go | 4 +- .../construct_statefulset_test.go | 14 ++++ pkg/reconciliation/constructor.go | 2 +- pkg/reconciliation/reconcile_racks.go | 6 +- pkg/reconciliation/secrets.go | 2 +- pkg/reconciliation/secrets_test.go | 5 ++ 13 files changed, 130 insertions(+), 25 deletions(-) diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index 28b5d1e2..23c0af57 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -238,6 +238,9 @@ type CassandraDatacenterSpec struct { // Additional Labels allows to define additional labels that will be included in all objects created by the operator. Note, user can override values set by default from the cass-operator and doing so could break cass-operator functionality. AdditionalLabels map[string]string `json:"additionalLabels,omitempty"` + // Additional Annotations allows to define additional labels that will be included in all objects created by the operator. Note, user can override values set by default from the cass-operator and doing so could break cass-operator functionality. + AdditionalAnnotations map[string]string `json:"additionalAnnotations,omitempty"` + // CDC allows configuration of the change data capture agent which can run within the Management API container. Use it to send data to Pulsar. CDC *CDCConfiguration `json:"cdc,omitempty"` diff --git a/internal/controllers/control/cassandratask_controller.go b/internal/controllers/control/cassandratask_controller.go index c4b0125c..b4332c8e 100644 --- a/internal/controllers/control/cassandratask_controller.go +++ b/internal/controllers/control/cassandratask_controller.go @@ -237,7 +237,7 @@ func (r *CassandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques } utils.MergeMap(cassTask.Labels, dc.GetDatacenterLabels()) - oplabels.AddOperatorLabels(cassTask.GetLabels(), dc) + oplabels.AddOperatorTags(cassTask.GetLabels(), dc) // Starting the run, set the Active label so we can quickly fetch the active ones cassTask.GetLabels()[taskStatusLabel] = activeTaskLabelValue diff --git a/pkg/oplabels/labels.go b/pkg/oplabels/labels.go index 3deaed87..6308f0cd 100644 --- a/pkg/oplabels/labels.go +++ b/pkg/oplabels/labels.go @@ -20,7 +20,7 @@ const ( CreatedByLabelValue = ManagedByLabelValue ) -func AddOperatorLabels(m map[string]string, dc *api.CassandraDatacenter) { +func AddOperatorTags(m map[string]string, dc *api.CassandraDatacenter) { m[ManagedByLabel] = ManagedByLabelValue m[NameLabel] = NameLabelValue m[VersionLabel] = dc.Spec.ServerVersion @@ -32,6 +32,12 @@ func AddOperatorLabels(m map[string]string, dc *api.CassandraDatacenter) { m[key] = api.CleanLabelValue(value) } } + + if len(dc.Spec.AdditionalAnnotations) != 0 { + for key, value := range dc.Spec.AdditionalAnnotations { + m[key] = api.CleanLabelValue(value) + } + } } func HasManagedByCassandraOperatorLabel(m map[string]string) bool { diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 0ab215bc..6b6e6a91 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -799,7 +799,7 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI // Labels podLabels := dc.GetRackLabels(rack.Name) - oplabels.AddOperatorLabels(podLabels, dc) + oplabels.AddOperatorTags(podLabels, dc) podLabels[api.CassNodeState] = stateReadyToStart if baseTemplate.Labels == nil { diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index a09f3e4e..672e0677 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -977,7 +977,7 @@ func TestCassandraDatacenter_buildPodTemplateSpec_labels_merge(t *testing.T) { got := spec.Labels expected := dc.GetRackLabels("testrack") - oplabels.AddOperatorLabels(expected, dc) + oplabels.AddOperatorTags(expected, dc) expected[api.CassNodeState] = stateReadyToStart expected["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue expected["abc"] = "123" @@ -1003,24 +1003,34 @@ func TestCassandraDatacenter_buildContainers_additional_labels(t *testing.T) { AdditionalLabels: map[string]string{ "Add": "Label", }, + AdditionalAnnotations: map[string]string{ + "Add": "Annotation", + }, }, } dc.Spec.PodTemplateSpec.Labels = map[string]string{"abc": "123"} spec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) - got := spec.Labels + gotLabels := spec.Labels - expected := dc.GetRackLabels("testrack") - oplabels.AddOperatorLabels(expected, dc) - expected[api.CassNodeState] = stateReadyToStart - expected["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue - expected["abc"] = "123" - expected["Add"] = "Label" + expectedLabels := dc.GetRackLabels("testrack") + oplabels.AddOperatorTags(expectedLabels, dc) + expectedLabels[api.CassNodeState] = stateReadyToStart + expectedLabels["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue + expectedLabels["abc"] = "123" + expectedLabels["Add"] = "Label" assert.NoError(t, err, "should not have gotten error when building podTemplateSpec") - if !reflect.DeepEqual(expected, got) { - t.Errorf("labels = %v, want %v", got, expected) + if !reflect.DeepEqual(expectedLabels, gotLabels) { + t.Errorf("labels = %v, want %v", gotLabels, expectedLabels) } + + gotAnns := spec.Annotations + expectedAnnotations := map[string]string{"Add": "Annotation"} + if !reflect.DeepEqual(expectedAnnotations, gotAnns) { + t.Errorf("labels = %v, want %v", gotAnns, expectedAnnotations) + } + } func TestCassandraDatacenter_buildPodTemplateSpec_overrideSecurityContext(t *testing.T) { diff --git a/pkg/reconciliation/construct_service.go b/pkg/reconciliation/construct_service.go index 3d15a963..5e0b5b21 100644 --- a/pkg/reconciliation/construct_service.go +++ b/pkg/reconciliation/construct_service.go @@ -118,7 +118,7 @@ func newSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.S service.ObjectMeta.Name = dc.GetSeedServiceName() labels := dc.GetClusterLabels() - oplabels.AddOperatorLabels(labels, dc) + oplabels.AddOperatorTags(labels, dc) service.ObjectMeta.Labels = labels service.Spec.Selector = buildLabelSelectorForSeedService(dc) @@ -135,7 +135,7 @@ func newSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.S // whether the additional seed pods are ready or not func newAdditionalSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Service { labels := dc.GetDatacenterLabels() - oplabels.AddOperatorLabels(labels, dc) + oplabels.AddOperatorTags(labels, dc) var service corev1.Service service.ObjectMeta.Name = dc.GetAdditionalSeedsServiceName() service.ObjectMeta.Namespace = dc.Namespace @@ -154,7 +154,7 @@ func newAdditionalSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) func newEndpointsForAdditionalSeeds(dc *api.CassandraDatacenter) (*corev1.Endpoints, error) { labels := dc.GetDatacenterLabels() - oplabels.AddOperatorLabels(labels, dc) + oplabels.AddOperatorTags(labels, dc) endpoints := corev1.Endpoints{} endpoints.ObjectMeta.Name = dc.GetAdditionalSeedsServiceName() endpoints.ObjectMeta.Namespace = dc.Namespace @@ -282,7 +282,7 @@ func newAllPodsServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev // inside the k8s cluster. func makeGenericHeadlessService(dc *api.CassandraDatacenter) *corev1.Service { labels := dc.GetDatacenterLabels() - oplabels.AddOperatorLabels(labels, dc) + oplabels.AddOperatorTags(labels, dc) selector := dc.GetDatacenterLabels() var service corev1.Service service.ObjectMeta.Namespace = dc.Namespace diff --git a/pkg/reconciliation/construct_service_test.go b/pkg/reconciliation/construct_service_test.go index a6860dac..d3bbdaec 100644 --- a/pkg/reconciliation/construct_service_test.go +++ b/pkg/reconciliation/construct_service_test.go @@ -5,10 +5,11 @@ package reconciliation import ( "fmt" - "github.com/k8ssandra/cass-operator/pkg/oplabels" "reflect" "testing" + "github.com/k8ssandra/cass-operator/pkg/oplabels" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -115,6 +116,9 @@ func TestLabelsWithNewSeedServiceForCassandraDatacenter(t *testing.T) { AdditionalLabels: map[string]string{ "Add": "label", }, + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, }, } @@ -134,6 +138,12 @@ func TestLabelsWithNewSeedServiceForCassandraDatacenter(t *testing.T) { if !reflect.DeepEqual(expected, service.Labels) { t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected) } + if !reflect.DeepEqual(expected, service.Labels) { + t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{ + "Add": "annotation", + }) + } + } func TestLabelsWithNewNodePortServiceForCassandraDatacenter(t *testing.T) { @@ -174,6 +184,9 @@ func TestLabelsWithNewNodePortServiceForCassandraDatacenter(t *testing.T) { AdditionalLabels: map[string]string{ "Add": "label", }, + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, }, } @@ -194,6 +207,11 @@ func TestLabelsWithNewNodePortServiceForCassandraDatacenter(t *testing.T) { if !reflect.DeepEqual(expected, service.Labels) { t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected) } + if !reflect.DeepEqual(expected, service.Labels) { + t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{ + "Add": "annotation", + }) + } } func TestLabelsWithNewAllPodsServiceForCassandraDatacenter(t *testing.T) { @@ -234,6 +252,9 @@ func TestLabelsWithNewAllPodsServiceForCassandraDatacenter(t *testing.T) { AdditionalLabels: map[string]string{ "Add": "label", }, + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, }, } @@ -255,6 +276,11 @@ func TestLabelsWithNewAllPodsServiceForCassandraDatacenter(t *testing.T) { if !reflect.DeepEqual(expected, service.Labels) { t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected) } + if !reflect.DeepEqual(expected, service.Labels) { + t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{ + "Add": "annotation", + }) + } } func TestLabelsWithNewServiceForCassandraDatacenter(t *testing.T) { @@ -295,6 +321,9 @@ func TestLabelsWithNewServiceForCassandraDatacenter(t *testing.T) { AdditionalLabels: map[string]string{ "Add": "label", }, + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, }, } @@ -315,6 +344,11 @@ func TestLabelsWithNewServiceForCassandraDatacenter(t *testing.T) { if !reflect.DeepEqual(expected, service.Labels) { t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected) } + if !reflect.DeepEqual(expected, service.Labels) { + t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{ + "Add": "annotation", + }) + } } func TestLabelsWithNewAdditionalSeedServiceForCassandraDatacenter(t *testing.T) { @@ -355,6 +389,9 @@ func TestLabelsWithNewAdditionalSeedServiceForCassandraDatacenter(t *testing.T) AdditionalLabels: map[string]string{ "Add": "label", }, + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, }, } @@ -375,6 +412,11 @@ func TestLabelsWithNewAdditionalSeedServiceForCassandraDatacenter(t *testing.T) if !reflect.DeepEqual(expected, service.Labels) { t.Errorf("service labels = \n %v \n, want \n %v", service.Labels, expected) } + if !reflect.DeepEqual(expected, service.Labels) { + t.Errorf("service labels = \n %v \n, want \n %v", service.Annotations, map[string]string{ + "Add": "annotation", + }) + } } func TestAddingAdditionalLabels(t *testing.T) { @@ -409,6 +451,31 @@ func TestAddingAdditionalLabels(t *testing.T) { } } +func TestAddingAdditionalAnnotations(t *testing.T) { + dc := &api.CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dc1", + }, + Spec: api.CassandraDatacenterSpec{ + ClusterName: "piclem", + ServerVersion: "4.0.1", + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, + }, + } + + expected := map[string]string{ + "Add": "annotation", + } + + service := newServiceForCassandraDatacenter(dc) + + if !reflect.DeepEqual(expected, service.Annotations) { + t.Errorf("service Annotations = %v, want %v", service.Annotations, expected) + } +} + func TestServicePorts(t *testing.T) { tests := []struct { name string diff --git a/pkg/reconciliation/construct_statefulset.go b/pkg/reconciliation/construct_statefulset.go index 6df42f97..a3a090ca 100644 --- a/pkg/reconciliation/construct_statefulset.go +++ b/pkg/reconciliation/construct_statefulset.go @@ -71,10 +71,10 @@ func newStatefulSetForCassandraDatacenter( // see https://github.com/kubernetes/kubernetes/pull/74941 // pvc labels are ignored before k8s 1.15.0 pvcLabels := dc.GetRackLabels(rackName) - oplabels.AddOperatorLabels(pvcLabels, dc) + oplabels.AddOperatorTags(pvcLabels, dc) statefulSetLabels := dc.GetRackLabels(rackName) - oplabels.AddOperatorLabels(statefulSetLabels, dc) + oplabels.AddOperatorTags(statefulSetLabels, dc) statefulSetSelectorLabels := dc.GetRackLabels(rackName) diff --git a/pkg/reconciliation/construct_statefulset_test.go b/pkg/reconciliation/construct_statefulset_test.go index 48728aeb..7ffba7d4 100644 --- a/pkg/reconciliation/construct_statefulset_test.go +++ b/pkg/reconciliation/construct_statefulset_test.go @@ -79,6 +79,9 @@ func Test_newStatefulSetForCassandraDatacenter_additionalLabels(t *testing.T) { AdditionalLabels: map[string]string{ "Add": "label", }, + AdditionalAnnotations: map[string]string{ + "Add": "annotation", + }, }, } @@ -120,6 +123,17 @@ func Test_newStatefulSetForCassandraDatacenter_additionalLabels(t *testing.T) { assert.Equal(t, expectedStatefulsetLabels, volumeClaim.Labels) } + assert.Equal(t, + map[string]string{ + "Add": "annotation"}, + statefulset.Annotations) + + for _, volumeClaim := range statefulset.Spec.VolumeClaimTemplates { + assert.Equal(t, map[string]string{ + "Add": "annotation"}, + volumeClaim.Annotations) + } + } func Test_newStatefulSetForCassandraDatacenter_rackNodeAffinitylabels(t *testing.T) { diff --git a/pkg/reconciliation/constructor.go b/pkg/reconciliation/constructor.go index 3f03ae00..f173397d 100644 --- a/pkg/reconciliation/constructor.go +++ b/pkg/reconciliation/constructor.go @@ -20,7 +20,7 @@ import ( func newPodDisruptionBudgetForDatacenter(dc *api.CassandraDatacenter) *policyv1.PodDisruptionBudget { minAvailable := intstr.FromInt(int(dc.Spec.Size - 1)) labels := dc.GetDatacenterLabels() - oplabels.AddOperatorLabels(labels, dc) + oplabels.AddOperatorTags(labels, dc) selectorLabels := dc.GetDatacenterLabels() pdb := &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index 04361737..c19e63ef 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -1634,7 +1634,7 @@ func mergeInLabelsIfDifferent(existingLabels, newLabels map[string]string) (bool // resource. It will return the updated map and a boolean denoting whether the resource needs to be updated with the new labels. func shouldUpdateLabelsForClusterResource(resourceLabels map[string]string, dc *api.CassandraDatacenter) (bool, map[string]string) { desired := dc.GetClusterLabels() - oplabels.AddOperatorLabels(desired, dc) + oplabels.AddOperatorTags(desired, dc) return mergeInLabelsIfDifferent(resourceLabels, desired) } @@ -1642,7 +1642,7 @@ func shouldUpdateLabelsForClusterResource(resourceLabels map[string]string, dc * // resource. It will return the updated map and a boolean denoting whether the resource needs to be updated with the new labels. func shouldUpdateLabelsForRackResource(resourceLabels map[string]string, dc *api.CassandraDatacenter, rackName string) (bool, map[string]string) { desired := dc.GetRackLabels(rackName) - oplabels.AddOperatorLabels(desired, dc) + oplabels.AddOperatorTags(desired, dc) return mergeInLabelsIfDifferent(resourceLabels, desired) } @@ -2149,7 +2149,7 @@ func (rc *ReconciliationContext) createTask(command taskapi.CassandraCommand) er Status: taskapi.CassandraTaskStatus{}, } - oplabels.AddOperatorLabels(task.GetLabels(), dc) + oplabels.AddOperatorTags(task.GetLabels(), dc) if err := rc.Client.Create(rc.Ctx, task); err != nil { return err diff --git a/pkg/reconciliation/secrets.go b/pkg/reconciliation/secrets.go index 89c3663f..fa6537d3 100644 --- a/pkg/reconciliation/secrets.go +++ b/pkg/reconciliation/secrets.go @@ -56,7 +56,7 @@ func buildDefaultSuperuserSecret(dc *api.CassandraDatacenter) (*corev1.Secret, e if dc.ShouldGenerateSuperuserSecret() { labels := make(map[string]string) - oplabels.AddOperatorLabels(labels, dc) + oplabels.AddOperatorTags(labels, dc) secretNamespacedName := dc.GetSuperuserSecretNamespacedName() secret = &corev1.Secret{ diff --git a/pkg/reconciliation/secrets_test.go b/pkg/reconciliation/secrets_test.go index 39e813d6..729db032 100644 --- a/pkg/reconciliation/secrets_test.go +++ b/pkg/reconciliation/secrets_test.go @@ -28,6 +28,7 @@ func Test_buildDefaultSuperuserSecret(t *testing.T) { AdditionalLabels: map[string]string{ "piclem": "add", }, + AdditionalAnnotations: map[string]string{"add": "annotation"}, }, } secret, err := buildDefaultSuperuserSecret(dc) @@ -59,6 +60,10 @@ func Test_buildDefaultSuperuserSecret(t *testing.T) { "piclem": "add", } + if !reflect.DeepEqual(map[string]string{"add": "annotation"}, secret.Annotations) { + t.Errorf("labels = \n %v \n, want \n %v", secret.Labels, expectedSecretLabels) + } + if !reflect.DeepEqual(expectedSecretLabels, secret.Labels) { t.Errorf("labels = \n %v \n, want \n %v", secret.Labels, expectedSecretLabels) } From 1c7fbbb3da6b84302772e99dea86d2a2522a7db5 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Tue, 19 Dec 2023 16:51:21 +1100 Subject: [PATCH 2/8] Ensure rack reconciliation adds annotations when they are missing. --- .../v1beta1/zz_generated.deepcopy.go | 7 ++ ...dra.datastax.com_cassandradatacenters.yaml | 8 ++ .../control/cassandratask_controller.go | 3 +- pkg/oplabels/labels.go | 8 +- .../construct_podtemplatespec.go | 6 +- .../construct_podtemplatespec_test.go | 4 +- pkg/reconciliation/construct_service.go | 23 +++++- pkg/reconciliation/construct_statefulset.go | 25 +++--- pkg/reconciliation/constructor.go | 7 +- pkg/reconciliation/reconcile_racks.go | 79 ++++++++++++++++--- pkg/reconciliation/secrets.go | 11 ++- pkg/reconciliation/secrets_test.go | 2 +- 12 files changed, 145 insertions(+), 38 deletions(-) diff --git a/apis/cassandra/v1beta1/zz_generated.deepcopy.go b/apis/cassandra/v1beta1/zz_generated.deepcopy.go index da773243..e296e4df 100644 --- a/apis/cassandra/v1beta1/zz_generated.deepcopy.go +++ b/apis/cassandra/v1beta1/zz_generated.deepcopy.go @@ -346,6 +346,13 @@ func (in *CassandraDatacenterSpec) DeepCopyInto(out *CassandraDatacenterSpec) { (*out)[key] = val } } + if in.AdditionalAnnotations != nil { + in, out := &in.AdditionalAnnotations, &out.AdditionalAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.CDC != nil { in, out := &in.CDC, &out.CDC *out = new(CDCConfiguration) diff --git a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml index 723d8f4d..48ebcb53 100644 --- a/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml +++ b/config/crd/bases/cassandra.datastax.com_cassandradatacenters.yaml @@ -38,6 +38,14 @@ spec: spec: description: CassandraDatacenterSpec defines the desired state of a CassandraDatacenter properties: + additionalAnnotations: + additionalProperties: + type: string + description: Additional Annotations allows to define additional labels + that will be included in all objects created by the operator. Note, + user can override values set by default from the cass-operator and + doing so could break cass-operator functionality. + type: object additionalLabels: additionalProperties: type: string diff --git a/internal/controllers/control/cassandratask_controller.go b/internal/controllers/control/cassandratask_controller.go index b4332c8e..78b1803f 100644 --- a/internal/controllers/control/cassandratask_controller.go +++ b/internal/controllers/control/cassandratask_controller.go @@ -237,7 +237,8 @@ func (r *CassandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques } utils.MergeMap(cassTask.Labels, dc.GetDatacenterLabels()) - oplabels.AddOperatorTags(cassTask.GetLabels(), dc) + oplabels.AddOperatorLabels(cassTask.GetLabels(), dc) + oplabels.AddOperatorAnnotations(cassTask.GetAnnotations(), dc) // Starting the run, set the Active label so we can quickly fetch the active ones cassTask.GetLabels()[taskStatusLabel] = activeTaskLabelValue diff --git a/pkg/oplabels/labels.go b/pkg/oplabels/labels.go index 6308f0cd..9e5e8d17 100644 --- a/pkg/oplabels/labels.go +++ b/pkg/oplabels/labels.go @@ -20,7 +20,7 @@ const ( CreatedByLabelValue = ManagedByLabelValue ) -func AddOperatorTags(m map[string]string, dc *api.CassandraDatacenter) { +func AddOperatorLabels(m map[string]string, dc *api.CassandraDatacenter) { m[ManagedByLabel] = ManagedByLabelValue m[NameLabel] = NameLabelValue m[VersionLabel] = dc.Spec.ServerVersion @@ -33,6 +33,12 @@ func AddOperatorTags(m map[string]string, dc *api.CassandraDatacenter) { } } +} + +func AddOperatorAnnotations(m map[string]string, dc *api.CassandraDatacenter) { + if m == nil { + m = make(map[string]string) + } if len(dc.Spec.AdditionalAnnotations) != 0 { for key, value := range dc.Spec.AdditionalAnnotations { m[key] = api.CleanLabelValue(value) diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index 6b6e6a91..b4cbd50d 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -799,7 +799,7 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI // Labels podLabels := dc.GetRackLabels(rack.Name) - oplabels.AddOperatorTags(podLabels, dc) + oplabels.AddOperatorLabels(podLabels, dc) podLabels[api.CassNodeState] = stateReadyToStart if baseTemplate.Labels == nil { @@ -809,12 +809,10 @@ func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyI // Annotations - podAnnotations := map[string]string{} - if baseTemplate.Annotations == nil { baseTemplate.Annotations = make(map[string]string) } - baseTemplate.Annotations = utils.MergeMap(baseTemplate.Annotations, podAnnotations) + oplabels.AddOperatorAnnotations(baseTemplate.Annotations, dc) // Affinity nodeAffinityLabels, nodeAffinityLabelsConfigurationError := rackNodeAffinitylabels(dc, rack.Name) diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index 672e0677..70db0e6a 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -977,7 +977,7 @@ func TestCassandraDatacenter_buildPodTemplateSpec_labels_merge(t *testing.T) { got := spec.Labels expected := dc.GetRackLabels("testrack") - oplabels.AddOperatorTags(expected, dc) + oplabels.AddOperatorLabels(expected, dc) expected[api.CassNodeState] = stateReadyToStart expected["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue expected["abc"] = "123" @@ -1014,7 +1014,7 @@ func TestCassandraDatacenter_buildContainers_additional_labels(t *testing.T) { gotLabels := spec.Labels expectedLabels := dc.GetRackLabels("testrack") - oplabels.AddOperatorTags(expectedLabels, dc) + oplabels.AddOperatorLabels(expectedLabels, dc) expectedLabels[api.CassNodeState] = stateReadyToStart expectedLabels["app.kubernetes.io/managed-by"] = oplabels.ManagedByLabelValue expectedLabels["abc"] = "123" diff --git a/pkg/reconciliation/construct_service.go b/pkg/reconciliation/construct_service.go index 5e0b5b21..79d5b628 100644 --- a/pkg/reconciliation/construct_service.go +++ b/pkg/reconciliation/construct_service.go @@ -118,9 +118,13 @@ func newSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.S service.ObjectMeta.Name = dc.GetSeedServiceName() labels := dc.GetClusterLabels() - oplabels.AddOperatorTags(labels, dc) + oplabels.AddOperatorLabels(labels, dc) service.ObjectMeta.Labels = labels + anns := dc.GetAnnotations() + oplabels.AddOperatorAnnotations(anns, dc) + service.ObjectMeta.Annotations = anns + service.Spec.Selector = buildLabelSelectorForSeedService(dc) service.Spec.PublishNotReadyAddresses = true @@ -135,11 +139,14 @@ func newSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.S // whether the additional seed pods are ready or not func newAdditionalSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Service { labels := dc.GetDatacenterLabels() - oplabels.AddOperatorTags(labels, dc) + oplabels.AddOperatorLabels(labels, dc) + anns := dc.GetAnnotations() + oplabels.AddOperatorAnnotations(anns, dc) var service corev1.Service service.ObjectMeta.Name = dc.GetAdditionalSeedsServiceName() service.ObjectMeta.Namespace = dc.Namespace service.ObjectMeta.Labels = labels + service.ObjectMeta.Annotations = anns // We omit the label selector because we will create the endpoints manually service.Spec.Type = "ClusterIP" service.Spec.ClusterIP = "None" @@ -154,11 +161,14 @@ func newAdditionalSeedServiceForCassandraDatacenter(dc *api.CassandraDatacenter) func newEndpointsForAdditionalSeeds(dc *api.CassandraDatacenter) (*corev1.Endpoints, error) { labels := dc.GetDatacenterLabels() - oplabels.AddOperatorTags(labels, dc) + oplabels.AddOperatorLabels(labels, dc) endpoints := corev1.Endpoints{} endpoints.ObjectMeta.Name = dc.GetAdditionalSeedsServiceName() endpoints.ObjectMeta.Namespace = dc.Namespace endpoints.ObjectMeta.Labels = labels + anns := dc.GetAnnotations() + oplabels.AddOperatorAnnotations(anns, dc) + endpoints.ObjectMeta.Annotations = anns addresses := make([]corev1.EndpointAddress, 0, len(dc.Spec.AdditionalSeeds)) for _, additionalSeed := range dc.Spec.AdditionalSeeds { @@ -282,7 +292,7 @@ func newAllPodsServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev // inside the k8s cluster. func makeGenericHeadlessService(dc *api.CassandraDatacenter) *corev1.Service { labels := dc.GetDatacenterLabels() - oplabels.AddOperatorTags(labels, dc) + oplabels.AddOperatorLabels(labels, dc) selector := dc.GetDatacenterLabels() var service corev1.Service service.ObjectMeta.Namespace = dc.Namespace @@ -290,5 +300,10 @@ func makeGenericHeadlessService(dc *api.CassandraDatacenter) *corev1.Service { service.Spec.Selector = selector service.Spec.Type = "ClusterIP" service.Spec.ClusterIP = "None" + + anns := dc.GetAnnotations() + oplabels.AddOperatorAnnotations(anns, dc) + service.ObjectMeta.Annotations = anns + return &service } diff --git a/pkg/reconciliation/construct_statefulset.go b/pkg/reconciliation/construct_statefulset.go index a3a090ca..07c7b53a 100644 --- a/pkg/reconciliation/construct_statefulset.go +++ b/pkg/reconciliation/construct_statefulset.go @@ -71,13 +71,16 @@ func newStatefulSetForCassandraDatacenter( // see https://github.com/kubernetes/kubernetes/pull/74941 // pvc labels are ignored before k8s 1.15.0 pvcLabels := dc.GetRackLabels(rackName) - oplabels.AddOperatorTags(pvcLabels, dc) + oplabels.AddOperatorLabels(pvcLabels, dc) statefulSetLabels := dc.GetRackLabels(rackName) - oplabels.AddOperatorTags(statefulSetLabels, dc) + oplabels.AddOperatorLabels(statefulSetLabels, dc) statefulSetSelectorLabels := dc.GetRackLabels(rackName) + anns := dc.GetAnnotations() + oplabels.AddOperatorAnnotations(anns, dc) + var volumeClaimTemplates []corev1.PersistentVolumeClaim rack := dc.GetRack(rackName) @@ -90,8 +93,9 @@ func newStatefulSetForCassandraDatacenter( volumeClaimTemplates = []corev1.PersistentVolumeClaim{{ ObjectMeta: metav1.ObjectMeta{ - Labels: pvcLabels, - Name: PvcName, + Labels: pvcLabels, + Name: PvcName, + Annotations: anns, }, Spec: *dc.Spec.StorageConfig.CassandraDataVolumeClaimSpec, }} @@ -100,8 +104,9 @@ func newStatefulSetForCassandraDatacenter( if storage.PVCSpec != nil { pvc := corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: storage.Name, - Labels: pvcLabels, + Name: storage.Name, + Labels: pvcLabels, + Annotations: anns, }, Spec: *storage.PVCSpec, } @@ -126,9 +131,10 @@ func newStatefulSetForCassandraDatacenter( result := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: nsName.Name, - Namespace: nsName.Namespace, - Labels: statefulSetLabels, + Name: nsName.Name, + Namespace: nsName.Namespace, + Labels: statefulSetLabels, + Annotations: anns, }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ @@ -141,7 +147,6 @@ func newStatefulSetForCassandraDatacenter( VolumeClaimTemplates: volumeClaimTemplates, }, } - result.Annotations = map[string]string{} if sts != nil && sts.Spec.ServiceName != "" && sts.Spec.ServiceName != result.Spec.ServiceName { result.Spec.ServiceName = sts.Spec.ServiceName diff --git a/pkg/reconciliation/constructor.go b/pkg/reconciliation/constructor.go index f173397d..d8053671 100644 --- a/pkg/reconciliation/constructor.go +++ b/pkg/reconciliation/constructor.go @@ -20,14 +20,17 @@ import ( func newPodDisruptionBudgetForDatacenter(dc *api.CassandraDatacenter) *policyv1.PodDisruptionBudget { minAvailable := intstr.FromInt(int(dc.Spec.Size - 1)) labels := dc.GetDatacenterLabels() - oplabels.AddOperatorTags(labels, dc) + oplabels.AddOperatorLabels(labels, dc) selectorLabels := dc.GetDatacenterLabels() + anns := map[string]string{} + oplabels.AddOperatorAnnotations(anns, dc) + pdb := &policyv1.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{ Name: dc.SanitizedName() + "-pdb", Namespace: dc.Namespace, Labels: labels, - Annotations: map[string]string{}, + Annotations: anns, }, Spec: policyv1.PodDisruptionBudgetSpec{ Selector: &metav1.LabelSelector{ diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index c19e63ef..54aad3c9 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -440,6 +440,23 @@ func (rc *ReconciliationContext) CheckRackLabels() result.ReconcileResult { rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, "Update rack labels for StatefulSet %s", statefulSet.Name) } + + stsAnns := statefulSet.GetAnnotations() + oplabels.AddOperatorAnnotations(stsAnns, rc.Datacenter) + if !reflect.DeepEqual(stsAnns, statefulSet.GetAnnotations()) { + rc.ReqLogger.Info("Updating annotations", + "statefulSet", statefulSet, + "current", stsAnns, + "desired", updatedLabels) + statefulSet.SetAnnotations(stsAnns) + + if err := rc.Client.Patch(rc.Ctx, statefulSet, patch); err != nil { + return result.Error(err) + } + + rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, + "Update rack annotations for StatefulSet %s", statefulSet.Name) + } } return result.Continue() @@ -1562,6 +1579,27 @@ func (rc *ReconciliationContext) ReconcilePods(statefulSet *appsv1.StatefulSet) "Update rack labels for Pod %s", podName) } + podAnns := pod.GetAnnotations() + oplabels.AddOperatorAnnotations(podAnns, rc.Datacenter) + if !reflect.DeepEqual(podAnns, pod.GetAnnotations()) { + rc.ReqLogger.Info("Updating annotations", + "Pod", podName, + "current", pod.GetAnnotations(), + "desired", podAnns) + pod.SetAnnotations(podAnns) + + if err := rc.Client.Patch(rc.Ctx, pod, podPatch); err != nil { + rc.ReqLogger.Error( + err, + "Unable to update pod with annotation", + "Pod", podName, + ) + } + + rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, + "Update rack annotations for pod %s", pod.Name) + } + if pod.Spec.Volumes == nil || len(pod.Spec.Volumes) == 0 || pod.Spec.Volumes[0].PersistentVolumeClaim == nil { continue } @@ -1616,12 +1654,32 @@ func (rc *ReconciliationContext) ReconcilePods(statefulSet *appsv1.StatefulSet) rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, "Update rack labels for PersistentVolumeClaim %s", pvc.Name) } + pvcAnns := pvc.GetAnnotations() + oplabels.AddOperatorAnnotations(pvcAnns, rc.Datacenter) + if !reflect.DeepEqual(pvcAnns, pvc.GetAnnotations()) { + rc.ReqLogger.Info("Updating annotations", + "PVC", pvc, + "current", pvc.GetAnnotations(), + "desired", pvcAnns) + pvc.SetAnnotations(pvcAnns) + pvcPatch := client.MergeFrom(pvc.DeepCopy()) + if err := rc.Client.Patch(rc.Ctx, pvc, pvcPatch); err != nil { + rc.ReqLogger.Error( + err, + "Unable to update pvc with annotation", + "PVC", pvc, + ) + } + + rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, + "Update rack annotations for pvc %s", pvc.Name) + } } return nil } -func mergeInLabelsIfDifferent(existingLabels, newLabels map[string]string) (bool, map[string]string) { +func mergeInTagsIfDifferent(existingLabels, newLabels map[string]string) (bool, map[string]string) { updatedLabels := utils.MergeMap(map[string]string{}, existingLabels, newLabels) if reflect.DeepEqual(existingLabels, updatedLabels) { return false, existingLabels @@ -1634,16 +1692,16 @@ func mergeInLabelsIfDifferent(existingLabels, newLabels map[string]string) (bool // resource. It will return the updated map and a boolean denoting whether the resource needs to be updated with the new labels. func shouldUpdateLabelsForClusterResource(resourceLabels map[string]string, dc *api.CassandraDatacenter) (bool, map[string]string) { desired := dc.GetClusterLabels() - oplabels.AddOperatorTags(desired, dc) - return mergeInLabelsIfDifferent(resourceLabels, desired) + oplabels.AddOperatorLabels(desired, dc) + return mergeInTagsIfDifferent(resourceLabels, desired) } // shouldUpdateLabelsForRackResource will compare the labels passed in with what the labels should be for a rack level // resource. It will return the updated map and a boolean denoting whether the resource needs to be updated with the new labels. func shouldUpdateLabelsForRackResource(resourceLabels map[string]string, dc *api.CassandraDatacenter, rackName string) (bool, map[string]string) { desired := dc.GetRackLabels(rackName) - oplabels.AddOperatorTags(desired, dc) - return mergeInLabelsIfDifferent(resourceLabels, desired) + oplabels.AddOperatorLabels(desired, dc) + return mergeInTagsIfDifferent(resourceLabels, desired) } func (rc *ReconciliationContext) labelServerPodStarting(pod *corev1.Pod) error { @@ -2125,12 +2183,15 @@ func (rc *ReconciliationContext) cleanupAfterScaling() result.ReconcileResult { func (rc *ReconciliationContext) createTask(command taskapi.CassandraCommand) error { generatedName := fmt.Sprintf("%s-%d", command, time.Now().Unix()) dc := rc.Datacenter + anns := make(map[string]string) + oplabels.AddOperatorAnnotations(anns, dc) task := &taskapi.CassandraTask{ ObjectMeta: metav1.ObjectMeta{ - Name: generatedName, - Namespace: rc.Datacenter.Namespace, - Labels: dc.GetDatacenterLabels(), + Name: generatedName, + Namespace: rc.Datacenter.Namespace, + Labels: dc.GetDatacenterLabels(), + Annotations: anns, }, Spec: taskapi.CassandraTaskSpec{ Datacenter: corev1.ObjectReference{ @@ -2149,7 +2210,7 @@ func (rc *ReconciliationContext) createTask(command taskapi.CassandraCommand) er Status: taskapi.CassandraTaskStatus{}, } - oplabels.AddOperatorTags(task.GetLabels(), dc) + oplabels.AddOperatorLabels(task.GetLabels(), dc) if err := rc.Client.Create(rc.Ctx, task); err != nil { return err diff --git a/pkg/reconciliation/secrets.go b/pkg/reconciliation/secrets.go index fa6537d3..1a0090e3 100644 --- a/pkg/reconciliation/secrets.go +++ b/pkg/reconciliation/secrets.go @@ -56,7 +56,9 @@ func buildDefaultSuperuserSecret(dc *api.CassandraDatacenter) (*corev1.Secret, e if dc.ShouldGenerateSuperuserSecret() { labels := make(map[string]string) - oplabels.AddOperatorTags(labels, dc) + oplabels.AddOperatorLabels(labels, dc) + anns := make(map[string]string) + oplabels.AddOperatorAnnotations(anns, dc) secretNamespacedName := dc.GetSuperuserSecretNamespacedName() secret = &corev1.Secret{ @@ -65,9 +67,10 @@ func buildDefaultSuperuserSecret(dc *api.CassandraDatacenter) (*corev1.Secret, e APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ - Name: secretNamespacedName.Name, - Namespace: secretNamespacedName.Namespace, - Labels: labels, + Name: secretNamespacedName.Name, + Namespace: secretNamespacedName.Namespace, + Labels: labels, + Annotations: anns, }, } username := api.CleanupForKubernetes(dc.Spec.ClusterName) + "-superuser" diff --git a/pkg/reconciliation/secrets_test.go b/pkg/reconciliation/secrets_test.go index 729db032..192ddaaf 100644 --- a/pkg/reconciliation/secrets_test.go +++ b/pkg/reconciliation/secrets_test.go @@ -61,7 +61,7 @@ func Test_buildDefaultSuperuserSecret(t *testing.T) { } if !reflect.DeepEqual(map[string]string{"add": "annotation"}, secret.Annotations) { - t.Errorf("labels = \n %v \n, want \n %v", secret.Labels, expectedSecretLabels) + t.Errorf("annotations = \n %v \n, want \n %v", secret.Annotations, map[string]string{"add": "annotation"}) } if !reflect.DeepEqual(expectedSecretLabels, secret.Labels) { From e17c5c0e630a5fab2c94d07a35ae7a62210b4a5a Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Tue, 19 Dec 2023 17:03:14 +1100 Subject: [PATCH 3/8] Fix tests. --- pkg/reconciliation/construct_service_test.go | 8 +------- pkg/reconciliation/construct_statefulset.go | 2 +- pkg/reconciliation/construct_statefulset_test.go | 12 +++++------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/pkg/reconciliation/construct_service_test.go b/pkg/reconciliation/construct_service_test.go index d3bbdaec..d1e2bf5b 100644 --- a/pkg/reconciliation/construct_service_test.go +++ b/pkg/reconciliation/construct_service_test.go @@ -465,15 +465,9 @@ func TestAddingAdditionalAnnotations(t *testing.T) { }, } - expected := map[string]string{ - "Add": "annotation", - } - service := newServiceForCassandraDatacenter(dc) - if !reflect.DeepEqual(expected, service.Annotations) { - t.Errorf("service Annotations = %v, want %v", service.Annotations, expected) - } + assert.Contains(t, service.Annotations, "Add") } func TestServicePorts(t *testing.T) { diff --git a/pkg/reconciliation/construct_statefulset.go b/pkg/reconciliation/construct_statefulset.go index 07c7b53a..e48f3ce6 100644 --- a/pkg/reconciliation/construct_statefulset.go +++ b/pkg/reconciliation/construct_statefulset.go @@ -78,7 +78,7 @@ func newStatefulSetForCassandraDatacenter( statefulSetSelectorLabels := dc.GetRackLabels(rackName) - anns := dc.GetAnnotations() + anns := make(map[string]string) oplabels.AddOperatorAnnotations(anns, dc) var volumeClaimTemplates []corev1.PersistentVolumeClaim diff --git a/pkg/reconciliation/construct_statefulset_test.go b/pkg/reconciliation/construct_statefulset_test.go index 7ffba7d4..0d6d1760 100644 --- a/pkg/reconciliation/construct_statefulset_test.go +++ b/pkg/reconciliation/construct_statefulset_test.go @@ -123,15 +123,13 @@ func Test_newStatefulSetForCassandraDatacenter_additionalLabels(t *testing.T) { assert.Equal(t, expectedStatefulsetLabels, volumeClaim.Labels) } - assert.Equal(t, - map[string]string{ - "Add": "annotation"}, - statefulset.Annotations) + assert.Contains(t, + statefulset.Annotations, "Add") for _, volumeClaim := range statefulset.Spec.VolumeClaimTemplates { - assert.Equal(t, map[string]string{ - "Add": "annotation"}, - volumeClaim.Annotations) + assert.Contains(t, + volumeClaim.Annotations, + "Add") } } From e16c57543c3d1b96aada11ca43eeab2703fd0d4b Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Tue, 19 Dec 2023 17:31:38 +1100 Subject: [PATCH 4/8] Final test fixes. --- pkg/reconciliation/construct_service.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/reconciliation/construct_service.go b/pkg/reconciliation/construct_service.go index 79d5b628..9319f9cd 100644 --- a/pkg/reconciliation/construct_service.go +++ b/pkg/reconciliation/construct_service.go @@ -70,6 +70,9 @@ func newServiceForCassandraDatacenter(dc *api.CassandraDatacenter) *corev1.Servi } service.Spec.Ports = ports + anns := make(map[string]string) + oplabels.AddOperatorAnnotations(anns, dc) + service.ObjectMeta.Annotations = anns addAdditionalOptions(service, &dc.Spec.AdditionalServiceConfig.DatacenterService) @@ -301,7 +304,7 @@ func makeGenericHeadlessService(dc *api.CassandraDatacenter) *corev1.Service { service.Spec.Type = "ClusterIP" service.Spec.ClusterIP = "None" - anns := dc.GetAnnotations() + anns := make(map[string]string) oplabels.AddOperatorAnnotations(anns, dc) service.ObjectMeta.Annotations = anns From 38b96f7610fadfa616a267f0d7173960059c3a84 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Wed, 20 Dec 2023 10:06:20 +1100 Subject: [PATCH 5/8] Update pod template spec annotations, not pod annotations. --- pkg/reconciliation/reconcile_racks.go | 38 ++++++++++++--------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index 54aad3c9..bccb2565 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -457,6 +457,23 @@ func (rc *ReconciliationContext) CheckRackLabels() result.ReconcileResult { rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, "Update rack annotations for StatefulSet %s", statefulSet.Name) } + + ptsAnns := statefulSet.Spec.Template.GetAnnotations() + oplabels.AddOperatorAnnotations(ptsAnns, rc.Datacenter) + if !reflect.DeepEqual(ptsAnns, statefulSet.GetAnnotations()) { + rc.ReqLogger.Info("Updating annotations", + "statefulSet", statefulSet, + "current", ptsAnns, + "desired", updatedLabels) + statefulSet.Spec.Template.SetAnnotations(ptsAnns) + + if err := rc.Client.Patch(rc.Ctx, statefulSet, patch); err != nil { + return result.Error(err) + } + + rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, + "Update pod template spec rack annotations for StatefulSet %s", statefulSet.Name) + } } return result.Continue() @@ -1579,27 +1596,6 @@ func (rc *ReconciliationContext) ReconcilePods(statefulSet *appsv1.StatefulSet) "Update rack labels for Pod %s", podName) } - podAnns := pod.GetAnnotations() - oplabels.AddOperatorAnnotations(podAnns, rc.Datacenter) - if !reflect.DeepEqual(podAnns, pod.GetAnnotations()) { - rc.ReqLogger.Info("Updating annotations", - "Pod", podName, - "current", pod.GetAnnotations(), - "desired", podAnns) - pod.SetAnnotations(podAnns) - - if err := rc.Client.Patch(rc.Ctx, pod, podPatch); err != nil { - rc.ReqLogger.Error( - err, - "Unable to update pod with annotation", - "Pod", podName, - ) - } - - rc.Recorder.Eventf(rc.Datacenter, corev1.EventTypeNormal, events.LabeledRackResource, - "Update rack annotations for pod %s", pod.Name) - } - if pod.Spec.Volumes == nil || len(pod.Spec.Volumes) == 0 || pod.Spec.Volumes[0].PersistentVolumeClaim == nil { continue } From 383068af5b8bf2c4ff1c2cdd2b83f144af18ef2e Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Fri, 5 Jan 2024 13:34:13 +1100 Subject: [PATCH 6/8] Michael's feedback. --- pkg/oplabels/labels.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/oplabels/labels.go b/pkg/oplabels/labels.go index 9e5e8d17..9f3d77e2 100644 --- a/pkg/oplabels/labels.go +++ b/pkg/oplabels/labels.go @@ -41,7 +41,7 @@ func AddOperatorAnnotations(m map[string]string, dc *api.CassandraDatacenter) { } if len(dc.Spec.AdditionalAnnotations) != 0 { for key, value := range dc.Spec.AdditionalAnnotations { - m[key] = api.CleanLabelValue(value) + m[key] = value } } } From 023bc9e8e11ebc98ca8e7b2f5720dd8fbfe96aa5 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Mon, 8 Jan 2024 13:38:08 +1100 Subject: [PATCH 7/8] Revert changes to mergeInLabelsIfDifferent. --- pkg/reconciliation/reconcile_racks.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/reconciliation/reconcile_racks.go b/pkg/reconciliation/reconcile_racks.go index bccb2565..ce519f94 100644 --- a/pkg/reconciliation/reconcile_racks.go +++ b/pkg/reconciliation/reconcile_racks.go @@ -1675,7 +1675,7 @@ func (rc *ReconciliationContext) ReconcilePods(statefulSet *appsv1.StatefulSet) return nil } -func mergeInTagsIfDifferent(existingLabels, newLabels map[string]string) (bool, map[string]string) { +func mergeInLabelsIfDifferent(existingLabels, newLabels map[string]string) (bool, map[string]string) { updatedLabels := utils.MergeMap(map[string]string{}, existingLabels, newLabels) if reflect.DeepEqual(existingLabels, updatedLabels) { return false, existingLabels @@ -1689,7 +1689,7 @@ func mergeInTagsIfDifferent(existingLabels, newLabels map[string]string) (bool, func shouldUpdateLabelsForClusterResource(resourceLabels map[string]string, dc *api.CassandraDatacenter) (bool, map[string]string) { desired := dc.GetClusterLabels() oplabels.AddOperatorLabels(desired, dc) - return mergeInTagsIfDifferent(resourceLabels, desired) + return mergeInLabelsIfDifferent(resourceLabels, desired) } // shouldUpdateLabelsForRackResource will compare the labels passed in with what the labels should be for a rack level @@ -1697,7 +1697,7 @@ func shouldUpdateLabelsForClusterResource(resourceLabels map[string]string, dc * func shouldUpdateLabelsForRackResource(resourceLabels map[string]string, dc *api.CassandraDatacenter, rackName string) (bool, map[string]string) { desired := dc.GetRackLabels(rackName) oplabels.AddOperatorLabels(desired, dc) - return mergeInTagsIfDifferent(resourceLabels, desired) + return mergeInLabelsIfDifferent(resourceLabels, desired) } func (rc *ReconciliationContext) labelServerPodStarting(pod *corev1.Pod) error { From a6b76301592659f50b7506e0b581b3a9d3891366 Mon Sep 17 00:00:00 2001 From: Miles-Garnsey Date: Tue, 9 Jan 2024 19:01:21 +1100 Subject: [PATCH 8/8] Changelog. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7517f38..6149ef45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti ## unreleased +* [FEATURE] [#601](https://github.com/k8ssandra/cass-operator/pull/601) Add additionalAnnotations field to CR so that all resources created by the operator can be annotated. + ## v1.18.2 * [BUGFIX] [#593](https://github.com/k8ssandra/cass-operator/issues/593) Update k8ssandra-client to 0.2.2 to fix the issue with clusterName config generation when using 4.1