From 850c69c777382ed571bf37a074212f760e0d488a Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Apr 2024 20:40:44 +0300 Subject: [PATCH 01/26] Add new status field: ObservedGeneration and ensure it is updated at the end of the reconciliation --- apis/k8ssandra/v1alpha1/constants.go | 3 + .../v1alpha1/k8ssandracluster_types.go | 11 ++ .../crds/k8ssandra-operator-crds.yaml | 5 + .../bases/k8ssandra.io_k8ssandraclusters.yaml | 5 + controllers/k8ssandra/auth_test.go | 12 +- .../k8ssandra/cassandra_metrics_agent_test.go | 2 +- controllers/k8ssandra/datacenters.go | 18 ++- .../k8ssandra/k8ssandracluster_controller.go | 23 ++++ .../k8ssandracluster_controller_test.go | 106 +++++++++++++++--- controllers/k8ssandra/per_node_config_test.go | 7 +- controllers/k8ssandra/vector_test.go | 2 +- 11 files changed, 165 insertions(+), 29 deletions(-) diff --git a/apis/k8ssandra/v1alpha1/constants.go b/apis/k8ssandra/v1alpha1/constants.go index 128e714a5..33ca833d5 100644 --- a/apis/k8ssandra/v1alpha1/constants.go +++ b/apis/k8ssandra/v1alpha1/constants.go @@ -64,6 +64,9 @@ const ( // Annotation to indicate the purpose of a given resource. PurposeAnnotation = "k8ssandra.io/purpose" + + // AutomatedUpdateAnnotation is an annotation that allows the Datacenters to be updated even if no changes were done to the K8ssandraCluster spec + AutomatedUpdateAnnotation = "k8ssandra.io/autoupdate-spec" ) var ( diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go index 3fd6670ab..16b684521 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go @@ -106,10 +106,17 @@ type K8ssandraClusterStatus struct { // +kubebuilder:default=None Error string `json:"error,omitempty"` + + // ObservedGeneration is the last observed generation of the K8ssandraCluster. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` } type K8ssandraClusterConditionType string +const ( + ClusterRequiresUpdate K8ssandraClusterConditionType = "RequiresUpdate" +) + type DecommissionProgress string const ( @@ -548,3 +555,7 @@ func (sd *ServerDistribution) IsDse() bool { func (kc *K8ssandraCluster) GetClusterIdHash() string { return utils.HashNameNamespace(kc.Name, kc.Namespace) } + +func (k *K8ssandraCluster) GenerationChanged() bool { + return k.Status.ObservedGeneration < k.Generation +} diff --git a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml index 7a404f056..9f52314df 100644 --- a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml +++ b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml @@ -31025,6 +31025,11 @@ spec: error: default: None type: string + observedGeneration: + description: ObservedGeneration is the last observed generation of + the K8ssandraCluster. + format: int64 + type: integer type: object type: object served: true diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml index 83ec43242..c3977e758 100644 --- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml +++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml @@ -30963,6 +30963,11 @@ spec: error: default: None type: string + observedGeneration: + description: ObservedGeneration is the last observed generation of + the K8ssandraCluster. + format: int64 + type: integer type: object type: object served: true diff --git a/controllers/k8ssandra/auth_test.go b/controllers/k8ssandra/auth_test.go index f3a299934..3d33ce79d 100644 --- a/controllers/k8ssandra/auth_test.go +++ b/controllers/k8ssandra/auth_test.go @@ -55,12 +55,11 @@ func createSingleDcClusterNoAuth(t *testing.T, ctx context.Context, f *framework err := f.Client.Create(ctx, kc) require.NoError(t, err, "failed to create K8ssandraCluster") - kcKey := framework.ClusterKey{K8sContext: f.ControlPlaneContext, NamespacedName: types.NamespacedName{Namespace: namespace, Name: kc.Name}} dcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}} reaperKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "cluster1-dc1-reaper"}} stargateKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "cluster1-dc1-stargate"}} - verifyFinalizerAdded(ctx, t, f, kcKey.NamespacedName) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) verifySecretNotCreated(ctx, t, f, kc.Namespace, reaper.DefaultUserSecretName(kc.SanitizedName())) verifyReplicatedSecretReconciled(ctx, t, f, kc) @@ -165,12 +164,11 @@ func createSingleDcClusterAuth(t *testing.T, ctx context.Context, f *framework.F err := f.Client.Create(ctx, kc) require.NoError(t, err, "failed to create K8ssandraCluster") - kcKey := framework.ClusterKey{K8sContext: f.ControlPlaneContext, NamespacedName: types.NamespacedName{Namespace: namespace, Name: kc.Name}} dcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}} reaperKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "cluster1-dc1-reaper"}} stargateKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "cluster1-dc1-stargate"}} - verifyFinalizerAdded(ctx, t, f, kcKey.NamespacedName) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) verifySecretCreated(ctx, t, f, kc.Namespace, reaper.DefaultUserSecretName(kc.Name)) verifyReplicatedSecretReconciled(ctx, t, f, kc) @@ -285,12 +283,11 @@ func createSingleDcClusterAuthExternalSecrets(t *testing.T, ctx context.Context, err := f.Client.Create(ctx, kc) require.NoError(t, err, "failed to create K8ssandraCluster") - kcKey := framework.ClusterKey{K8sContext: f.ControlPlaneContext, NamespacedName: types.NamespacedName{Namespace: namespace, Name: kc.Name}} dcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}} reaperKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "cluster1-dc1-reaper"}} stargateKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "cluster1-dc1-stargate"}} - verifyFinalizerAdded(ctx, t, f, kcKey.NamespacedName) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretNotCreated(ctx, t, f, kc) // verify not created @@ -416,10 +413,9 @@ func createSingleDcClusterExternalInternode(t *testing.T, ctx context.Context, f err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - kcKey := framework.ClusterKey{K8sContext: f.ControlPlaneContext, NamespacedName: types.NamespacedName{Namespace: namespace, Name: kc.Name}} dcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[1], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}} - verifyFinalizerAdded(ctx, t, f, kcKey.NamespacedName) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) verifyReplicatedSecretReconciled(ctx, t, f, kc) diff --git a/controllers/k8ssandra/cassandra_metrics_agent_test.go b/controllers/k8ssandra/cassandra_metrics_agent_test.go index c56d017d9..e5defbfdc 100644 --- a/controllers/k8ssandra/cassandra_metrics_agent_test.go +++ b/controllers/k8ssandra/cassandra_metrics_agent_test.go @@ -67,7 +67,7 @@ func createSingleDcClusterWithMetricsAgent(t *testing.T, ctx context.Context, f err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 066679b4f..591e357bc 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -30,6 +30,10 @@ const ( rebuildNodesLabel = "k8ssandra.io/rebuild-nodes" ) +func AllowUpdate(kc *api.K8ssandraCluster) bool { + return kc.GenerationChanged() || metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) +} + func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, kc *api.K8ssandraCluster, logger logr.Logger) (result.ReconcileResult, []*cassdcapi.CassandraDatacenter) { kcKey := utils.GetKey(kc) @@ -143,7 +147,19 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k r.setStatusForDatacenter(kc, actualDc) - if !annotations.CompareHashAnnotations(actualDc, desiredDc) { + if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc) { + // We're not allowed to update, but need to + patch := client.MergeFrom(kc.DeepCopy()) + now := metav1.Now() + kc.Status.SetCondition(api.K8ssandraClusterCondition{ + Type: api.ClusterRequiresUpdate, + Status: corev1.ConditionTrue, + LastTransitionTime: &now, // Replace with ptr.To() once we have updated Kubernetes dependencies + }) + if err := r.Client.Status().Patch(ctx, kc, patch); err != nil { + return result.Error(fmt.Errorf("failed to set %s annotation: %v", api.AutomatedUpdateAnnotation, err)), actualDcs + } + } else if !annotations.CompareHashAnnotations(actualDc, desiredDc) { dcLogger.Info("Updating datacenter") if actualDc.Spec.SuperuserSecretName != desiredDc.Spec.SuperuserSecretName { diff --git a/controllers/k8ssandra/k8ssandracluster_controller.go b/controllers/k8ssandra/k8ssandracluster_controller.go index e7d2d77d5..8dc91399b 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller.go +++ b/controllers/k8ssandra/k8ssandracluster_controller.go @@ -33,6 +33,7 @@ import ( "github.com/k8ssandra/k8ssandra-operator/pkg/result" "github.com/k8ssandra/k8ssandra-operator/pkg/utils" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" @@ -154,6 +155,10 @@ func (r *K8ssandraClusterReconciler) reconcile(ctx context.Context, kc *api.K8ss return recResult.Output() } + if res := updateStatus(ctx, r.Client, kc); res.Completed() { + return res.Output() + } + kcLogger.Info("Finished reconciling the k8ssandracluster") return result.Done().Output() @@ -179,6 +184,24 @@ func (r *K8ssandraClusterReconciler) afterCassandraReconciled(ctx context.Contex return result.Continue() } +func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster) result.ReconcileResult { + // Remove the annotation + + if metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) { + delete(kc.ObjectMeta.Annotations, api.AutomatedUpdateAnnotation) + if err := r.Update(ctx, kc); err != nil { + return result.Error(err) + } + } + + kc.Status.ObservedGeneration = kc.Generation + if err := r.Status().Update(ctx, kc); err != nil { + return result.Error(err) + } + + return result.Continue() +} + // SetupWithManager sets up the controller with the Manager. func (r *K8ssandraClusterReconciler) SetupWithManager(mgr ctrl.Manager, clusters []cluster.Cluster) error { cb := ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/k8ssandra/k8ssandracluster_controller_test.go b/controllers/k8ssandra/k8ssandracluster_controller_test.go index 6a30e9b77..2b24765c2 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller_test.go +++ b/controllers/k8ssandra/k8ssandracluster_controller_test.go @@ -121,6 +121,7 @@ func TestK8ssandraCluster(t *testing.T) { t.Run("PerNodeConfiguration", testEnv.ControllerTest(ctx, perNodeConfiguration)) t.Run("CreateSingleDcClusterWithVector", testEnv.ControllerTest(ctx, createSingleDcClusterWithVector)) t.Run("createSingleDcClusterWithMetricsAgent", testEnv.ControllerTest(ctx, createSingleDcClusterWithMetricsAgent)) + t.Run("GenerationCheck", testEnv.ControllerTest(ctx, testGenerationCheck)) } // createSingleDcCluster verifies that the CassandraDatacenter is created and that the @@ -166,7 +167,7 @@ func createSingleDcCluster(t *testing.T, ctx context.Context, f *framework.Frame err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -393,7 +394,7 @@ func applyClusterTemplateConfigs(t *testing.T, ctx context.Context, f *framework err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8sandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -558,7 +559,7 @@ func applyDatacenterTemplateConfigs(t *testing.T, ctx context.Context, f *framew err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8sandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -708,7 +709,7 @@ func applyClusterTemplateAndDatacenterTemplateConfigs(t *testing.T, ctx context. err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8sandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -810,7 +811,7 @@ func createMultiDcCluster(t *testing.T, ctx context.Context, f *framework.Framew err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -1049,7 +1050,7 @@ func createSingleDcCassandra4ClusterWithStargate(t *testing.T, ctx context.Conte err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) verifyReplicatedSecretReconciled(ctx, t, f, kc) verifySystemReplicationAnnotationSet(ctx, t, f, kc) @@ -1208,7 +1209,7 @@ func createMultiDcClusterWithStargate(t *testing.T, ctx context.Context, f *fram err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -1573,7 +1574,7 @@ func applyClusterWithEncryptionOptions(t *testing.T, ctx context.Context, f *fra err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -1798,7 +1799,7 @@ func applyClusterWithEncryptionOptionsFail(t *testing.T, ctx context.Context, f err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -1952,7 +1953,7 @@ func applyClusterWithEncryptionOptionsExternalSecrets(t *testing.T, ctx context. err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) t.Log("check that dc1 was created") dc1Key := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[0]} require.Eventually(f.DatacenterExists(ctx, dc1Key), timeout, interval) @@ -2143,8 +2144,9 @@ func systemReplicationAnnotationIsSet(t *testing.T, f *framework.Framework, ctx } } -func verifyFinalizerAdded(ctx context.Context, t *testing.T, f *framework.Framework, key client.ObjectKey) { +func verifyFinalizerAdded(ctx context.Context, t *testing.T, f *framework.Framework, kc *api.K8ssandraCluster) { t.Log("check finalizer added to K8ssandraCluster") + key := client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name} assert.Eventually(t, func() bool { kc := &api.K8ssandraCluster{} @@ -2156,6 +2158,20 @@ func verifyFinalizerAdded(ctx context.Context, t *testing.T, f *framework.Framew }, timeout, interval, "failed to verify that finalizer was added") } +func verifyClusterReconcileFinished(ctx context.Context, t *testing.T, f *framework.Framework, kc *api.K8ssandraCluster) { + t.Log("check K8ssandraCluster reconciliation finished") + key := client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name} + + assert.Eventually(t, func() bool { + kc := &api.K8ssandraCluster{} + if err := f.Client.Get(ctx, key, kc); err != nil { + t.Logf("failed to get K8ssandraCluster: %v", err) + return false + } + return kc.ObjectMeta.Generation == kc.Status.ObservedGeneration + }, timeout, interval, "cluster hasn't finished reconciliation") +} + func verifyReplicatedSecretReconciled(ctx context.Context, t *testing.T, f *framework.Framework, kc *api.K8ssandraCluster) { t.Log("check ReplicatedSecret reconciled") @@ -2257,7 +2273,7 @@ func convertSystemReplicationAnnotation(t *testing.T, ctx context.Context, f *fr err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -2370,7 +2386,7 @@ func changeClusterNameFails(t *testing.T, ctx context.Context, f *framework.Fram err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -2487,7 +2503,7 @@ func injectContainersAndVolumes(t *testing.T, ctx context.Context, f *framework. err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -2598,7 +2614,7 @@ func createMultiDcDseCluster(t *testing.T, ctx context.Context, f *framework.Fra err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) @@ -2633,3 +2649,63 @@ func createMultiDcDseCluster(t *testing.T, ctx context.Context, f *framework.Fra f.AssertObjectDoesNotExist(ctx, t, dc1Key, &cassdcapi.CassandraDatacenter{}, timeout, interval) f.AssertObjectDoesNotExist(ctx, t, dc2Key, &cassdcapi.CassandraDatacenter{}, timeout, interval) } + +func testGenerationCheck(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) { + require := require.New(t) + + kc := &api.K8ssandraCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "test", + }, + Spec: api.K8ssandraClusterSpec{ + Cassandra: &api.CassandraClusterTemplate{ + ClusterName: "Not K8s_Compliant", + Datacenters: []api.CassandraDatacenterTemplate{ + { + Meta: api.EmbeddedObjectMeta{ + Name: "dc1", + }, + K8sContext: f.DataPlaneContexts[1], + Size: 1, + DatacenterOptions: api.DatacenterOptions{ + ServerVersion: "3.11.14", + StorageConfig: &cassdcapi.StorageConfig{ + CassandraDataVolumeClaimSpec: &corev1.PersistentVolumeClaimSpec{ + StorageClassName: &defaultStorageClass, + }, + }, + PodSecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64(999), + }, + ManagementApiAuth: &cassdcapi.ManagementApiAuthConfig{ + Insecure: &cassdcapi.ManagementApiAuthInsecureConfig{}, + }, + }, + }, + }, + }, + }, + } + + err := f.Client.Create(ctx, kc) + require.NoError(err, "failed to create K8ssandraCluster") + + verifyFinalizerAdded(ctx, t, f, kc) + + verifySuperuserSecretCreated(ctx, t, f, kc) + + verifyReplicatedSecretReconciled(ctx, t, f, kc) + + verifySystemReplicationAnnotationSet(ctx, t, f, kc) + + t.Log("check that the datacenter was created") + dcKey := framework.ClusterKey{NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}, K8sContext: f.DataPlaneContexts[1]} + require.Eventually(f.DatacenterExists(ctx, dcKey), timeout, interval) + + t.Log("update datacenter status to ready") + err = f.SetDatacenterStatusReady(ctx, dcKey) + require.NoError(err, "failed to set datacenter status ready") + + verifyClusterReconcileFinished(ctx, t, f, kc) +} diff --git a/controllers/k8ssandra/per_node_config_test.go b/controllers/k8ssandra/per_node_config_test.go index 68fd5d8c4..c737acb2c 100644 --- a/controllers/k8ssandra/per_node_config_test.go +++ b/controllers/k8ssandra/per_node_config_test.go @@ -2,6 +2,8 @@ package k8ssandra import ( "context" + "testing" + "github.com/go-logr/logr/testr" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" @@ -19,7 +21,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" - "testing" ) func TestK8ssandraClusterReconciler_reconcilePerNodeConfiguration(t *testing.T) { @@ -217,7 +218,7 @@ func defaultPerNodeConfiguration(t *testing.T, ctx context.Context, f *framework } }() - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) verifyReplicatedSecretReconciled(ctx, t, f, kc) verifySystemReplicationAnnotationSet(ctx, t, f, kc) @@ -332,7 +333,7 @@ func userDefinedPerNodeConfiguration(t *testing.T, ctx context.Context, f *frame } }() - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) verifyReplicatedSecretReconciled(ctx, t, f, kc) verifySystemReplicationAnnotationSet(ctx, t, f, kc) diff --git a/controllers/k8ssandra/vector_test.go b/controllers/k8ssandra/vector_test.go index 9f2a77874..343c68d1e 100644 --- a/controllers/k8ssandra/vector_test.go +++ b/controllers/k8ssandra/vector_test.go @@ -67,7 +67,7 @@ func createSingleDcClusterWithVector(t *testing.T, ctx context.Context, f *frame err := f.Client.Create(ctx, kc) require.NoError(err, "failed to create K8ssandraCluster") - verifyFinalizerAdded(ctx, t, f, client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name}) + verifyFinalizerAdded(ctx, t, f, kc) verifySuperuserSecretCreated(ctx, t, f, kc) From 61362443e8dc2e880e817ed77c091264bf1ef7dc Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 25 Apr 2024 15:17:48 +0300 Subject: [PATCH 02/26] Upgrade operator test should verify the state of CassandraDatacenter also, not just Stargate --- test/e2e/suite_test.go | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 7ead381c4..0ad85657c 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -44,6 +44,7 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" ) @@ -942,19 +943,37 @@ func createSingleDatacenterClusterWithUpgrade(t *testing.T, ctx context.Context, require := require.New(t) t.Log("check that the K8ssandraCluster was created") + kcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "test"}} k8ssandra := &api.K8ssandraCluster{} - kcKey := types.NamespacedName{Namespace: namespace, Name: "test"} - err := f.Client.Get(ctx, kcKey, k8ssandra) + err := f.Get(ctx, kcKey, k8ssandra) require.NoError(err, "failed to get K8ssandraCluster in namespace %s", namespace) dcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}} checkDatacenterReady(t, ctx, dcKey, f) - assertCassandraDatacenterK8cStatusReady(ctx, t, f, kcKey, dcKey.Name) + assertCassandraDatacenterK8cStatusReady(ctx, t, f, kcKey.NamespacedName, dcKey.Name) + cassdc := &cassdcapi.CassandraDatacenter{} + require.NoError(f.Get(ctx, dcKey, cassdc)) + dcHash := cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] dcPrefix := DcPrefix(t, f, dcKey) // Perform the upgrade err = upgradeToLatest(t, ctx, f, namespace, dcPrefix) require.NoError(err, "failed to upgrade to latest version") + + verifyClusterReconcileFinished(ctx, t, f, k8ssandra) + require.NoError(f.Get(ctx, dcKey, cassdc)) + newDcHash := cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] + + require.Equal(dcHash, newDcHash, "CassandraDatacenter resource hash changed after upgrade") + require.NoError(f.Get(ctx, kcKey, k8ssandra)) + + require.Equal(corev1.ConditionTrue, k8ssandra.Status.GetConditionStatus(api.ClusterRequiresUpdate)) + metav1.SetMetaDataAnnotation(&k8ssandra.ObjectMeta, api.AutomatedUpdateAnnotation, "always") + require.NoError(f.Update(ctx, kcKey, k8ssandra)) + verifyClusterReconcileFinished(ctx, t, f, k8ssandra) + require.NoError(f.Get(ctx, dcKey, cassdc)) + newDcHash = cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] + require.NotEqual(dcHash, newDcHash, "CassandraDatacenter resource hash hasn't changed after upgrade") } // createSingleDatacenterCluster creates a K8ssandraCluster with one CassandraDatacenter @@ -2251,3 +2270,17 @@ func CheckLabelsAnnotationsCreated(dcKey framework.ClusterKey, t *testing.T, ctx assert.True(t, cassDC.Spec.AdditionalAnnotations["anAnnotationKeyClusterLevel"] == "anAnnotationValueClusterLevel") return nil } + +func verifyClusterReconcileFinished(ctx context.Context, t *testing.T, f *framework.E2eFramework, kc *api.K8ssandraCluster) { + t.Log("check K8ssandraCluster reconciliation finished") + key := client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name} + + assert.Eventually(t, func() bool { + kc := &api.K8ssandraCluster{} + if err := f.Client.Get(ctx, key, kc); err != nil { + t.Logf("failed to get K8ssandraCluster: %v", err) + return false + } + return kc.ObjectMeta.Generation == kc.Status.ObservedGeneration + }, polling.k8ssandraClusterStatus.timeout, polling.k8ssandraClusterStatus.interval, "cluster hasn't finished reconciliation") +} From 27379345a6e854d0e8fa00f7b8a84fb16221b34d Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 25 Apr 2024 18:51:26 +0300 Subject: [PATCH 03/26] Add new SetConditionStatus to reduce duplication in the code, add restricted values for AutomatedUpdate, "always" and "once" and verify them in the webhook. Remove tee outputting to stdout in the helm prepare script. UpdateStatus should only delete and reset the ClusterRequiresUpdate if it was allowed to update in the first place. Also, we should listen to changes and reconcile if annotations change. Add additional checks to the GenerationCheck test to ensure we do not touch the CassandraDatacenter unless given permission. --- apis/k8ssandra/v1alpha1/constants.go | 6 +++ .../v1alpha1/k8ssandracluster_types.go | 9 +++++ .../v1alpha1/k8ssandracluster_webhook.go | 9 +++++ .../v1alpha1/k8ssandracluster_webhook_test.go | 18 +++++++++ controllers/k8ssandra/datacenters.go | 17 ++------- .../k8ssandra/k8ssandracluster_controller.go | 22 +++++++---- .../k8ssandracluster_controller_test.go | 37 +++++++++++++++++++ scripts/prepare-helm-release.sh | 12 +++--- 8 files changed, 102 insertions(+), 28 deletions(-) diff --git a/apis/k8ssandra/v1alpha1/constants.go b/apis/k8ssandra/v1alpha1/constants.go index 33ca833d5..606fe0554 100644 --- a/apis/k8ssandra/v1alpha1/constants.go +++ b/apis/k8ssandra/v1alpha1/constants.go @@ -67,8 +67,14 @@ const ( // AutomatedUpdateAnnotation is an annotation that allows the Datacenters to be updated even if no changes were done to the K8ssandraCluster spec AutomatedUpdateAnnotation = "k8ssandra.io/autoupdate-spec" + + AllowUpdateAlways AllowUpdateType = "always" + AllowUpdateOnce AllowUpdateType = "once" ) +// TODO Use the accepted values from cass-operator's api instead to prevent drift, once Kubernetes dependencies are updated in k8ssandra-operator +type AllowUpdateType string + var ( SystemKeyspaces = []string{"system_traces", "system_distributed", "system_auth"} DseKeyspaces = []string{"dse_leases", "dse_perf", "dse_security"} diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go index 16b684521..7e44c5cdf 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go @@ -518,6 +518,15 @@ func (s *K8ssandraClusterStatus) GetConditionStatus(conditionType K8ssandraClust return corev1.ConditionUnknown } +func (s *K8ssandraClusterStatus) SetConditionStatus(conditionType K8ssandraClusterConditionType, status corev1.ConditionStatus) { + now := metav1.Now() + s.SetCondition(K8ssandraClusterCondition{ + Type: conditionType, + Status: status, + LastTransitionTime: &now, + }) +} + func (s *K8ssandraClusterStatus) SetCondition(condition K8ssandraClusterCondition) { for i, c := range s.Conditions { if c.Type == condition.Type { diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go index 4b30f3bf0..53e996ead 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go @@ -21,6 +21,7 @@ import ( "github.com/Masterminds/semver/v3" "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" @@ -100,6 +101,14 @@ func (r *K8ssandraCluster) validateK8ssandraCluster() error { } } + if metav1.HasAnnotation(r.ObjectMeta, AutomatedUpdateAnnotation) { + // Allow only always and once in the annotation + annotationValue := r.ObjectMeta.GetAnnotations()[AutomatedUpdateAnnotation] + if annotationValue != string(AllowUpdateAlways) && annotationValue != string(AllowUpdateOnce) { + return fmt.Errorf("invalid value for %s annotation: %s", AutomatedUpdateAnnotation, annotationValue) + } + } + if err := r.ValidateMedusa(); err != nil { return err } diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go index d7e62821f..be713bd10 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go @@ -159,6 +159,7 @@ func TestWebhook(t *testing.T) { t.Run("MedusaPrefixMissing", testMedusaPrefixMissing) t.Run("InvalidDcName", testInvalidDcName) t.Run("MedusaConfigNonLocalNamespace", testMedusaNonLocalNamespace) + t.Run("AutomatedUpdateAnnotation", testAutomatedUpdateAnnotation) } func testContextValidation(t *testing.T) { @@ -577,3 +578,20 @@ func TestValidateUpdateNumTokens(t *testing.T) { } } } + +func testAutomatedUpdateAnnotation(t *testing.T) { + require := require.New(t) + createNamespace(require, "automated-update-namespace") + cluster := createMinimalClusterObj("automated-update-test", "automated-update-namespace") + require.NoError(cluster.validateK8ssandraCluster()) + + // Test should accept values once and always + metav1.SetMetaDataAnnotation(&cluster.ObjectMeta, AutomatedUpdateAnnotation, string(AllowUpdateOnce)) + require.NoError(cluster.validateK8ssandraCluster()) + + metav1.SetMetaDataAnnotation(&cluster.ObjectMeta, AutomatedUpdateAnnotation, string(AllowUpdateAlways)) + require.NoError(cluster.validateK8ssandraCluster()) + + cluster.Annotations[AutomatedUpdateAnnotation] = string("true") + require.Error(cluster.validateK8ssandraCluster()) +} diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 591e357bc..378e69be4 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -148,20 +148,14 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k r.setStatusForDatacenter(kc, actualDc) if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc) { + logger.Info("Datacenter requires an update, but we're not allowed to do it", "CassandraDatacenter", dcKey) // We're not allowed to update, but need to patch := client.MergeFrom(kc.DeepCopy()) - now := metav1.Now() - kc.Status.SetCondition(api.K8ssandraClusterCondition{ - Type: api.ClusterRequiresUpdate, - Status: corev1.ConditionTrue, - LastTransitionTime: &now, // Replace with ptr.To() once we have updated Kubernetes dependencies - }) + kc.Status.SetConditionStatus(api.ClusterRequiresUpdate, corev1.ConditionTrue) if err := r.Client.Status().Patch(ctx, kc, patch); err != nil { return result.Error(fmt.Errorf("failed to set %s annotation: %v", api.AutomatedUpdateAnnotation, err)), actualDcs } } else if !annotations.CompareHashAnnotations(actualDc, desiredDc) { - dcLogger.Info("Updating datacenter") - if actualDc.Spec.SuperuserSecretName != desiredDc.Spec.SuperuserSecretName { // If actualDc is created with SuperuserSecretName, it can't be changed anymore. We should reject all changes coming from K8ssandraCluster desiredDc.Spec.SuperuserSecretName = actualDc.Spec.SuperuserSecretName @@ -254,12 +248,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k // distinguish whether we are deploying a CassandraDatacenter as part of a new cluster // or as part of an existing cluster. if kc.Status.GetConditionStatus(api.CassandraInitialized) == corev1.ConditionUnknown { - now := metav1.Now() - kc.Status.SetCondition(api.K8ssandraClusterCondition{ - Type: api.CassandraInitialized, - Status: corev1.ConditionTrue, - LastTransitionTime: &now, - }) + kc.Status.SetConditionStatus(api.CassandraInitialized, corev1.ConditionTrue) } return result.Continue(), actualDcs diff --git a/controllers/k8ssandra/k8ssandracluster_controller.go b/controllers/k8ssandra/k8ssandracluster_controller.go index 8dc91399b..d9162ab55 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller.go +++ b/controllers/k8ssandra/k8ssandracluster_controller.go @@ -19,6 +19,7 @@ package k8ssandra import ( "context" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "github.com/go-logr/logr" @@ -95,7 +96,7 @@ func (r *K8ssandraClusterReconciler) Reconcile(ctx context.Context, req ctrl.Req if kc.GetDeletionTimestamp() == nil { if err != nil { kc.Status.Error = err.Error() - r.Recorder.Event(kc, v1.EventTypeWarning, "Reconcile Error", err.Error()) + r.Recorder.Event(kc, corev1.EventTypeWarning, "Reconcile Error", err.Error()) } else { kc.Status.Error = "None" } @@ -185,13 +186,16 @@ func (r *K8ssandraClusterReconciler) afterCassandraReconciled(ctx context.Contex } func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster) result.ReconcileResult { - // Remove the annotation - - if metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) { - delete(kc.ObjectMeta.Annotations, api.AutomatedUpdateAnnotation) - if err := r.Update(ctx, kc); err != nil { - return result.Error(err) + if AllowUpdate(kc) { + if metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) { + if kc.Annotations[api.AutomatedUpdateAnnotation] == string(api.AllowUpdateOnce) { + delete(kc.ObjectMeta.Annotations, api.AutomatedUpdateAnnotation) + if err := r.Update(ctx, kc); err != nil { + return result.Error(err) + } + } } + kc.Status.SetConditionStatus(api.ClusterRequiresUpdate, corev1.ConditionFalse) } kc.Status.ObservedGeneration = kc.Generation @@ -205,7 +209,9 @@ func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster // SetupWithManager sets up the controller with the Manager. func (r *K8ssandraClusterReconciler) SetupWithManager(mgr ctrl.Manager, clusters []cluster.Cluster) error { cb := ctrl.NewControllerManagedBy(mgr). - For(&api.K8ssandraCluster{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})) // No generation changed predicate here? + For(&api.K8ssandraCluster{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{}))) + + // We need also annotation changes due to autoupdate-spec clusterLabelFilter := func(ctx context.Context, mapObj client.Object) []reconcile.Request { requests := make([]reconcile.Request, 0) diff --git a/controllers/k8ssandra/k8ssandracluster_controller_test.go b/controllers/k8ssandra/k8ssandracluster_controller_test.go index 2b24765c2..db576754c 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller_test.go +++ b/controllers/k8ssandra/k8ssandracluster_controller_test.go @@ -2172,6 +2172,19 @@ func verifyClusterReconcileFinished(ctx context.Context, t *testing.T, f *framew }, timeout, interval, "cluster hasn't finished reconciliation") } +func waitForConditionStatus(ctx context.Context, t *testing.T, f *framework.Framework, conditionType api.K8ssandraClusterConditionType, status corev1.ConditionStatus, kc *api.K8ssandraCluster) { + key := client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name} + assert.Eventually(t, func() bool { + kc := &api.K8ssandraCluster{} + if err := f.Client.Get(ctx, key, kc); err != nil { + t.Logf("failed to get K8ssandraCluster: %v", err) + return false + } + kcCondition := kc.Status.GetConditionStatus(conditionType) + return kcCondition == status + }, timeout, interval, "cluster didn't reach the expected condition status") +} + func verifyReplicatedSecretReconciled(ctx context.Context, t *testing.T, f *framework.Framework, kc *api.K8ssandraCluster) { t.Log("check ReplicatedSecret reconciled") @@ -2708,4 +2721,28 @@ func testGenerationCheck(t *testing.T, ctx context.Context, f *framework.Framewo require.NoError(err, "failed to set datacenter status ready") verifyClusterReconcileFinished(ctx, t, f, kc) + + // Modify the CassandraDatacenter hash to be some gibberish + dc := &cassdcapi.CassandraDatacenter{} + require.NoError(f.Get(ctx, dcKey, dc), "failed to get CassandraDatacenter dc1") + metav1.SetMetaDataAnnotation(&dc.ObjectMeta, api.ResourceHashAnnotation, "gibberish") + require.NoError(f.Update(ctx, dcKey, dc), "failed to update CassandraDatacenter dc1") + + waitForConditionStatus(ctx, t, f, api.ClusterRequiresUpdate, corev1.ConditionTrue, kc) + verifyClusterReconcileFinished(ctx, t, f, kc) + + require.NoError(f.Get(ctx, dcKey, dc), "failed to get CassandraDatacenter dc1") + require.Equal("gibberish", dc.Annotations[api.ResourceHashAnnotation]) + + t.Log("Modifying K8ssandraCluster to allow upgrade") + // Modify K8ssandraCluster to allow upgrade + kcKey := client.ObjectKey{Namespace: namespace, Name: kc.Name} + require.NoError(f.Client.Get(ctx, kcKey, kc), "failed to get K8ssandraCluster") + metav1.SetMetaDataAnnotation(&kc.ObjectMeta, api.AutomatedUpdateAnnotation, "once") + require.NoError(f.Client.Update(ctx, kc), "failed to update K8ssandraCluster") + // Wait for process to start.. + waitForConditionStatus(ctx, t, f, api.ClusterRequiresUpdate, corev1.ConditionFalse, kc) + + require.NoError(f.Get(ctx, dcKey, dc), "failed to get CassandraDatacenter dc1") + require.NotEqual("gibberish", dc.Annotations[api.ResourceHashAnnotation]) } diff --git a/scripts/prepare-helm-release.sh b/scripts/prepare-helm-release.sh index efc3e04cf..17d94ce34 100755 --- a/scripts/prepare-helm-release.sh +++ b/scripts/prepare-helm-release.sh @@ -13,15 +13,15 @@ mkdir -p build/helm kustomize build config/crd > charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml # Generate the role.yaml and clusterrole.yaml files using the RBAC generated manifests kustomize build config/rbac > build/helm/k8ssandra-operator-rbac.yaml -cat charts/templates/role.tmpl.yaml | tee build/helm/role.yaml -cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 1).rules' | tee -a build/helm/role.yaml +cat charts/templates/role.tmpl.yaml | tee build/helm/role.yaml > /dev/null +cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 1).rules' | tee -a build/helm/role.yaml > /dev/null echo "{{- end }}" >> build/helm/role.yaml -cat charts/templates/clusterrole.tmpl.yaml | tee build/helm/clusterrole.yaml -cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 1).rules' | tee -a build/helm/clusterrole.yaml +cat charts/templates/clusterrole.tmpl.yaml | tee build/helm/clusterrole.yaml > /dev/null +cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 1).rules' | tee -a build/helm/clusterrole.yaml > /dev/null echo "{{- end }}" >> build/helm/clusterrole.yaml cp build/helm/role.yaml charts/k8ssandra-operator/templates/role.yaml cp build/helm/clusterrole.yaml charts/k8ssandra-operator/templates/clusterrole.yaml # Generate the leader election role from the RBAC generated manifests -cat charts/templates/leader-role.tmpl.yaml | tee build/helm/leader-role.yaml -cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 2).rules' | tee -a build/helm/leader-role.yaml +cat charts/templates/leader-role.tmpl.yaml | tee build/helm/leader-role.yaml > /dev/null +cat build/helm/k8ssandra-operator-rbac.yaml | yq 'select(di == 2).rules' | tee -a build/helm/leader-role.yaml > /dev/null cp build/helm/leader-role.yaml charts/k8ssandra-operator/templates/leader-role.yaml \ No newline at end of file From fe60b213410018501190ae3149924dbb253e0d03 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 30 Apr 2024 17:01:50 +0300 Subject: [PATCH 04/26] Add new refresh K8ssandraTask --- .../control/k8ssandratask_controller.go | 109 ++++++++++++++++++ .../control/k8ssandratask_controller_test.go | 84 +++++++++++++- 2 files changed, 189 insertions(+), 4 deletions(-) diff --git a/controllers/control/k8ssandratask_controller.go b/controllers/control/k8ssandratask_controller.go index 71388fbec..ac2cf4aa6 100644 --- a/controllers/control/k8ssandratask_controller.go +++ b/controllers/control/k8ssandratask_controller.go @@ -23,6 +23,7 @@ import ( "time" "github.com/go-logr/logr" + cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" cassapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" k8capi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" @@ -54,6 +55,8 @@ import ( const ( k8ssandraTaskFinalizer = "control.k8ssandra.io/finalizer" + InternalTaskAnnotation = "control.k8ssandra.io/internal-command" + internalRefreshCommand = "refresh" defaultTTL = time.Duration(86400) * time.Second ) @@ -148,6 +151,13 @@ func (r *K8ssandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques if !kcExists { return r.reportInvalidSpec(ctx, kTask, "unknown K8ssandraCluster %s.%s", kcKey.Namespace, kcKey.Name) } + + if kTask.Spec.Template.Jobs[0].Command == "refresh" { + if !metav1.HasAnnotation(kc.ObjectMeta, InternalTaskAnnotation) { + return r.executeRefreshTask(ctx, kTask, kc) + } + } + if dcs, err := filterDcs(kc, kTask.Spec.Datacenters); err != nil { return r.reportInvalidSpec(ctx, kTask, err.Error()) } else { @@ -200,6 +210,92 @@ func (r *K8ssandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } +func (r *K8ssandraTaskReconciler) executeRefreshTask(ctx context.Context, kTask *api.K8ssandraTask, kc *k8capi.K8ssandraCluster) (ctrl.Result, error) { + if kTask.Status.StartTime == nil { + patch := client.MergeFrom(kTask.DeepCopy()) + now := metav1.Now() + kTask.Status.StartTime = &now + kTask.Status.Active = 1 + kTask.SetCondition(cassapi.JobRunning, metav1.ConditionTrue) + if err := r.Status().Patch(ctx, kTask, patch); err != nil { + return ctrl.Result{}, err + } + } + + // First verify if K8ssandraCluster itself has "UpdateRequired" and process it until it no longer has it. + if kc.Status.GetConditionStatus(k8capi.ClusterRequiresUpdate) == corev1.ConditionTrue { + if metav1.HasAnnotation(kc.ObjectMeta, k8capi.AutomatedUpdateAnnotation) { + return ctrl.Result{Requeue: true}, nil + } else { + patch := client.MergeFrom(kc.DeepCopy()) + metav1.SetMetaDataAnnotation(&kc.ObjectMeta, k8capi.AutomatedUpdateAnnotation, string(k8capi.AllowUpdateOnce)) + if err := r.Patch(ctx, kc, patch); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + } + + internalTask := &api.K8ssandraTask{} + err := r.Get(ctx, types.NamespacedName{Namespace: kTask.Namespace, Name: kTask.Name + "-refresh-internal"}, internalTask) + // If task wasn't found, create it and if task is still running, requeue + if k8serrors.IsNotFound(err) { + // Then verify that no CassandraDatacenter has "UpdateRequired" and if they do, create new tasks to execute them + dcs, err := r.datacenters(ctx, kc) + if err != nil { + return ctrl.Result{}, err + } + + dcsRequiringUpdate := make([]string, 0, len(dcs)) + for _, dc := range dcs { + if dc.Status.GetConditionStatus("RequiresUpdate") == corev1.ConditionTrue { // TODO Update this to cassdcapi const when cass-operator is updatedß + dcsRequiringUpdate = append(dcsRequiringUpdate, dc.DatacenterName()) // TODO Ís this the correct reference? + } + } + + if len(dcsRequiringUpdate) > 0 { + // Delegate work to the task controller for Datacenter operations + task := &api.K8ssandraTask{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kTask.Namespace, + Name: kTask.Name + "-refresh-internal", + Annotations: map[string]string{ + InternalTaskAnnotation: internalRefreshCommand, + }, + }, + Spec: api.K8ssandraTaskSpec{ + Datacenters: make([]string, len(dcsRequiringUpdate)), + Template: kTask.Spec.Template, + }, + } + + if err := r.Create(ctx, task); err != nil { + return ctrl.Result{}, err + } + } + } else if err != nil { + return ctrl.Result{}, err + } else { + // Verify if the job is completed + if internalTask.Status.CompletionTime.IsZero() { + return ctrl.Result{Requeue: true}, nil + } + } + + patch := client.MergeFrom(kTask.DeepCopy()) + now := metav1.Now() + kTask.Status.CompletionTime = &now + kTask.Status.Succeeded = 1 + kTask.Status.Active = 0 + kTask.SetCondition(cassapi.JobComplete, metav1.ConditionTrue) + kTask.SetCondition(cassapi.JobRunning, metav1.ConditionFalse) + if err := r.Status().Patch(ctx, kTask, patch); err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + func (r *K8ssandraTaskReconciler) getCluster(ctx context.Context, kcKey client.ObjectKey) (*k8capi.K8ssandraCluster, bool, error) { kc := &k8capi.K8ssandraCluster{} if err := r.Get(ctx, kcKey, kc); k8serrors.IsNotFound(err) { @@ -210,6 +306,19 @@ func (r *K8ssandraTaskReconciler) getCluster(ctx context.Context, kcKey client.O return kc, true, nil } +func (r *K8ssandraTaskReconciler) datacenters(ctx context.Context, kc *k8capi.K8ssandraCluster) ([]cassdcapi.CassandraDatacenter, error) { + dcs := make([]cassdcapi.CassandraDatacenter, 0, len(kc.Spec.Cassandra.Datacenters)) + for _, dcTemplate := range kc.Spec.Cassandra.Datacenters { + dcKey := client.ObjectKey{Namespace: utils.FirstNonEmptyString(dcTemplate.Meta.Namespace, kc.Namespace), Name: dcTemplate.Meta.Name} + dc := &cassdcapi.CassandraDatacenter{} + if err := r.Get(ctx, dcKey, dc); err != nil { + return nil, err + } + dcs = append(dcs, *dc) + } + return dcs, nil +} + func (r *K8ssandraTaskReconciler) deleteCassandraTasks( ctx context.Context, kTask *api.K8ssandraTask, diff --git a/controllers/control/k8ssandratask_controller_test.go b/controllers/control/k8ssandratask_controller_test.go index d0b4f0d5c..2c8b0ae04 100644 --- a/controllers/control/k8ssandratask_controller_test.go +++ b/controllers/control/k8ssandratask_controller_test.go @@ -2,6 +2,7 @@ package control import ( "context" + "fmt" "testing" "time" @@ -9,6 +10,7 @@ import ( cassapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" k8capi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" + "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" "github.com/k8ssandra/k8ssandra-operator/pkg/config" testutils "github.com/k8ssandra/k8ssandra-operator/pkg/test" @@ -67,10 +69,13 @@ func TestK8ssandraTask(t *testing.T) { defer testEnv.Stop(t) defer cancel() - t.Run("ExecuteParallelK8ssandraTask", testEnv.ControllerTest(ctx, executeParallelK8ssandraTask)) - t.Run("ExecuteSequentialK8ssandraTask", testEnv.ControllerTest(ctx, executeSequentialK8ssandraTask)) - t.Run("DeleteK8ssandraTask", testEnv.ControllerTest(ctx, deleteK8ssandraTask)) - t.Run("ExpireK8ssandraTask", testEnv.ControllerTest(ctx, expireK8ssandraTask)) + /* + t.Run("ExecuteParallelK8ssandraTask", testEnv.ControllerTest(ctx, executeParallelK8ssandraTask)) + t.Run("ExecuteSequentialK8ssandraTask", testEnv.ControllerTest(ctx, executeSequentialK8ssandraTask)) + t.Run("DeleteK8ssandraTask", testEnv.ControllerTest(ctx, deleteK8ssandraTask)) + t.Run("ExpireK8ssandraTask", testEnv.ControllerTest(ctx, expireK8ssandraTask)) + */ + t.Run("RefreshK8ssandraCluster", testEnv.ControllerTest(ctx, refreshK8ssandraTask)) } // executeParallelK8ssandraTask creates and runs a K8ssandraTask with parallel DC processing. @@ -378,6 +383,65 @@ func expireK8ssandraTask(t *testing.T, ctx context.Context, f *framework.Framewo require.Eventually(func() bool { return !f.K8ssandraTaskExists(ctx, k8TaskKey)() }, timeout, interval) } +func refreshK8ssandraTask(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) { + require := require.New(t) + + kc := newCluster(namespace, "kc", + newDc("dc1", f.DataPlaneContexts[0])) + require.NoError(f.Client.Create(ctx, kc), "failed to create K8ssandraCluster") + + kc.Status.SetConditionStatus(k8capi.ClusterRequiresUpdate, corev1.ConditionTrue) + require.NoError(f.Client.Status().Update(ctx, kc)) + + dcConfig := cassandra.Coalesce(kc.CassClusterName(), kc.Spec.Cassandra.DeepCopy(), kc.Spec.Cassandra.Datacenters[0].DeepCopy()) + dc, err := cassandra.NewDatacenter(types.NamespacedName{Namespace: kc.Namespace, Name: kc.Name}, dcConfig) + require.NoError(err) + require.NoError(f.Client.Create(ctx, dc)) + + t.Log("Create a K8ssandraTask with TTL") + ttl := new(int32) + *ttl = 1 + k8Task := &api.K8ssandraTask{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "refresh", + }, + Spec: api.K8ssandraTaskSpec{ + Cluster: corev1.ObjectReference{ + Name: "kc", + }, + Template: cassapi.CassandraTaskTemplate{ + TTLSecondsAfterFinished: ttl, + Jobs: []cassapi.CassandraJob{{ + Name: "job1", + Command: "refresh", + }}, + }, + DcConcurrencyPolicy: batchv1.ForbidConcurrent, + }, + } + require.NoError(f.Client.Create(ctx, k8Task), "failed to create K8ssandraTask") + + require.Eventually(func() bool { + if err := f.Client.Get(ctx, types.NamespacedName{Namespace: kc.Namespace, Name: kc.Name}, kc); err != nil { + return false + } + if !metav1.HasAnnotation(kc.ObjectMeta, k8capi.AutomatedUpdateAnnotation) { + return false + } + return kc.Annotations[k8capi.AutomatedUpdateAnnotation] == string(k8capi.AllowUpdateOnce) + }, timeout, interval) + // First case, there's only changes in K8ssandraCluster + t.Log("Mark the K8ssandraCluster as updated if the annotation was added") + require.Equal(corev1.ConditionTrue, kc.Status.GetConditionStatus(k8capi.ClusterRequiresUpdate)) + kc.Status.SetConditionStatus(k8capi.ClusterRequiresUpdate, corev1.ConditionFalse) + require.NoError(f.Client.Status().Update(ctx, kc)) + + waitForTaskCompletion(ctx, t, f, newClusterKey(f.ControlPlaneContext, namespace, "refresh")) + // Second case, we also have changes in the CassandraDatacenter, even after updating K8ssandraCluster + +} + func newCluster(namespace, name string, dcs ...k8capi.CassandraDatacenterTemplate) *k8capi.K8ssandraCluster { return &k8capi.K8ssandraCluster{ ObjectMeta: metav1.ObjectMeta{ @@ -410,6 +474,18 @@ func newDc(name string, k8sContext string) k8capi.CassandraDatacenterTemplate { } } +func waitForTaskCompletion(ctx context.Context, t *testing.T, f *framework.Framework, taskKey framework.ClusterKey) { + require.Eventually(t, func() bool { + k8Task := &api.K8ssandraTask{} + require.NoError(t, f.Get(ctx, taskKey, k8Task)) + fmt.Printf("k8Task.Status: %+v\n", k8Task.Status) + return k8Task.Status.Active == 0 && + k8Task.Status.Succeeded > 0 && + k8Task.GetConditionStatus(cassapi.JobRunning) == metav1.ConditionFalse && + k8Task.GetConditionStatus(cassapi.JobComplete) == metav1.ConditionTrue + }, timeout, interval) +} + func newClusterKey(k8sContext, namespace, name string) framework.ClusterKey { return framework.ClusterKey{ NamespacedName: types.NamespacedName{Namespace: namespace, Name: name}, From 46022d47e4b0fc8768d7edd89197461b7550f4d7 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 30 Apr 2024 17:05:38 +0300 Subject: [PATCH 05/26] Re-enable tests --- controllers/control/k8ssandratask_controller_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/controllers/control/k8ssandratask_controller_test.go b/controllers/control/k8ssandratask_controller_test.go index 2c8b0ae04..101b672f8 100644 --- a/controllers/control/k8ssandratask_controller_test.go +++ b/controllers/control/k8ssandratask_controller_test.go @@ -69,12 +69,10 @@ func TestK8ssandraTask(t *testing.T) { defer testEnv.Stop(t) defer cancel() - /* - t.Run("ExecuteParallelK8ssandraTask", testEnv.ControllerTest(ctx, executeParallelK8ssandraTask)) - t.Run("ExecuteSequentialK8ssandraTask", testEnv.ControllerTest(ctx, executeSequentialK8ssandraTask)) - t.Run("DeleteK8ssandraTask", testEnv.ControllerTest(ctx, deleteK8ssandraTask)) - t.Run("ExpireK8ssandraTask", testEnv.ControllerTest(ctx, expireK8ssandraTask)) - */ + t.Run("ExecuteParallelK8ssandraTask", testEnv.ControllerTest(ctx, executeParallelK8ssandraTask)) + t.Run("ExecuteSequentialK8ssandraTask", testEnv.ControllerTest(ctx, executeSequentialK8ssandraTask)) + t.Run("DeleteK8ssandraTask", testEnv.ControllerTest(ctx, deleteK8ssandraTask)) + t.Run("ExpireK8ssandraTask", testEnv.ControllerTest(ctx, expireK8ssandraTask)) t.Run("RefreshK8ssandraCluster", testEnv.ControllerTest(ctx, refreshK8ssandraTask)) } From 12f0d630529bee01123612eb5ed6bb059ba088be Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 8 May 2024 18:11:47 +0300 Subject: [PATCH 06/26] Make the process entirely in the annotations, no more task --- CHANGELOG/CHANGELOG-1.18.md | 1 + .../control/k8ssandratask_controller.go | 108 ------------------ .../control/k8ssandratask_controller_test.go | 74 ------------ controllers/k8ssandra/datacenters.go | 52 +++++++++ .../k8ssandra/k8ssandracluster_controller.go | 2 - 5 files changed, 53 insertions(+), 184 deletions(-) diff --git a/CHANGELOG/CHANGELOG-1.18.md b/CHANGELOG/CHANGELOG-1.18.md index eadb85000..d19e07dde 100644 --- a/CHANGELOG/CHANGELOG-1.18.md +++ b/CHANGELOG/CHANGELOG-1.18.md @@ -18,5 +18,6 @@ When cutting a new release, update the `unreleased` heading to the tag being gen * [CHANGE] Update cassandra-medusa to 0.22.0 * [CHANGE] Update cass-operator to v1.22.0 * [FEATURE] [#1310](https://github.com/k8ssandra/k8ssandra-operator/issues/1310) Enhance the MedusaBackupSchedule API to allow scheduling purge tasks +* [ENHANCEMENT] [#1274](https://github.com/k8ssandra/k8ssandra-operator/issues/1274) On upgrade, do not modify the CassandraDatacenter object unless instructed with an annotation `k8ssandra.io/autoupdate-spec` with value `once` or `always` * [BUGFIX] [#1222](https://github.com/k8ssandra/k8ssandra-operator/issues/1222) Consider DC-level config when validating numToken updates in webhook * [BUGFIX] [#1366](https://github.com/k8ssandra/k8ssandra-operator/issues/1366) Reaper deployment can't be created on OpenShift due to missing RBAC rule diff --git a/controllers/control/k8ssandratask_controller.go b/controllers/control/k8ssandratask_controller.go index ac2cf4aa6..e63fc30a7 100644 --- a/controllers/control/k8ssandratask_controller.go +++ b/controllers/control/k8ssandratask_controller.go @@ -23,7 +23,6 @@ import ( "time" "github.com/go-logr/logr" - cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" cassapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" k8capi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" @@ -55,8 +54,6 @@ import ( const ( k8ssandraTaskFinalizer = "control.k8ssandra.io/finalizer" - InternalTaskAnnotation = "control.k8ssandra.io/internal-command" - internalRefreshCommand = "refresh" defaultTTL = time.Duration(86400) * time.Second ) @@ -152,12 +149,6 @@ func (r *K8ssandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques return r.reportInvalidSpec(ctx, kTask, "unknown K8ssandraCluster %s.%s", kcKey.Namespace, kcKey.Name) } - if kTask.Spec.Template.Jobs[0].Command == "refresh" { - if !metav1.HasAnnotation(kc.ObjectMeta, InternalTaskAnnotation) { - return r.executeRefreshTask(ctx, kTask, kc) - } - } - if dcs, err := filterDcs(kc, kTask.Spec.Datacenters); err != nil { return r.reportInvalidSpec(ctx, kTask, err.Error()) } else { @@ -210,92 +201,6 @@ func (r *K8ssandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } -func (r *K8ssandraTaskReconciler) executeRefreshTask(ctx context.Context, kTask *api.K8ssandraTask, kc *k8capi.K8ssandraCluster) (ctrl.Result, error) { - if kTask.Status.StartTime == nil { - patch := client.MergeFrom(kTask.DeepCopy()) - now := metav1.Now() - kTask.Status.StartTime = &now - kTask.Status.Active = 1 - kTask.SetCondition(cassapi.JobRunning, metav1.ConditionTrue) - if err := r.Status().Patch(ctx, kTask, patch); err != nil { - return ctrl.Result{}, err - } - } - - // First verify if K8ssandraCluster itself has "UpdateRequired" and process it until it no longer has it. - if kc.Status.GetConditionStatus(k8capi.ClusterRequiresUpdate) == corev1.ConditionTrue { - if metav1.HasAnnotation(kc.ObjectMeta, k8capi.AutomatedUpdateAnnotation) { - return ctrl.Result{Requeue: true}, nil - } else { - patch := client.MergeFrom(kc.DeepCopy()) - metav1.SetMetaDataAnnotation(&kc.ObjectMeta, k8capi.AutomatedUpdateAnnotation, string(k8capi.AllowUpdateOnce)) - if err := r.Patch(ctx, kc, patch); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{Requeue: true}, nil - } - } - - internalTask := &api.K8ssandraTask{} - err := r.Get(ctx, types.NamespacedName{Namespace: kTask.Namespace, Name: kTask.Name + "-refresh-internal"}, internalTask) - // If task wasn't found, create it and if task is still running, requeue - if k8serrors.IsNotFound(err) { - // Then verify that no CassandraDatacenter has "UpdateRequired" and if they do, create new tasks to execute them - dcs, err := r.datacenters(ctx, kc) - if err != nil { - return ctrl.Result{}, err - } - - dcsRequiringUpdate := make([]string, 0, len(dcs)) - for _, dc := range dcs { - if dc.Status.GetConditionStatus("RequiresUpdate") == corev1.ConditionTrue { // TODO Update this to cassdcapi const when cass-operator is updatedß - dcsRequiringUpdate = append(dcsRequiringUpdate, dc.DatacenterName()) // TODO Ís this the correct reference? - } - } - - if len(dcsRequiringUpdate) > 0 { - // Delegate work to the task controller for Datacenter operations - task := &api.K8ssandraTask{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: kTask.Namespace, - Name: kTask.Name + "-refresh-internal", - Annotations: map[string]string{ - InternalTaskAnnotation: internalRefreshCommand, - }, - }, - Spec: api.K8ssandraTaskSpec{ - Datacenters: make([]string, len(dcsRequiringUpdate)), - Template: kTask.Spec.Template, - }, - } - - if err := r.Create(ctx, task); err != nil { - return ctrl.Result{}, err - } - } - } else if err != nil { - return ctrl.Result{}, err - } else { - // Verify if the job is completed - if internalTask.Status.CompletionTime.IsZero() { - return ctrl.Result{Requeue: true}, nil - } - } - - patch := client.MergeFrom(kTask.DeepCopy()) - now := metav1.Now() - kTask.Status.CompletionTime = &now - kTask.Status.Succeeded = 1 - kTask.Status.Active = 0 - kTask.SetCondition(cassapi.JobComplete, metav1.ConditionTrue) - kTask.SetCondition(cassapi.JobRunning, metav1.ConditionFalse) - if err := r.Status().Patch(ctx, kTask, patch); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - func (r *K8ssandraTaskReconciler) getCluster(ctx context.Context, kcKey client.ObjectKey) (*k8capi.K8ssandraCluster, bool, error) { kc := &k8capi.K8ssandraCluster{} if err := r.Get(ctx, kcKey, kc); k8serrors.IsNotFound(err) { @@ -306,19 +211,6 @@ func (r *K8ssandraTaskReconciler) getCluster(ctx context.Context, kcKey client.O return kc, true, nil } -func (r *K8ssandraTaskReconciler) datacenters(ctx context.Context, kc *k8capi.K8ssandraCluster) ([]cassdcapi.CassandraDatacenter, error) { - dcs := make([]cassdcapi.CassandraDatacenter, 0, len(kc.Spec.Cassandra.Datacenters)) - for _, dcTemplate := range kc.Spec.Cassandra.Datacenters { - dcKey := client.ObjectKey{Namespace: utils.FirstNonEmptyString(dcTemplate.Meta.Namespace, kc.Namespace), Name: dcTemplate.Meta.Name} - dc := &cassdcapi.CassandraDatacenter{} - if err := r.Get(ctx, dcKey, dc); err != nil { - return nil, err - } - dcs = append(dcs, *dc) - } - return dcs, nil -} - func (r *K8ssandraTaskReconciler) deleteCassandraTasks( ctx context.Context, kTask *api.K8ssandraTask, diff --git a/controllers/control/k8ssandratask_controller_test.go b/controllers/control/k8ssandratask_controller_test.go index 101b672f8..d0b4f0d5c 100644 --- a/controllers/control/k8ssandratask_controller_test.go +++ b/controllers/control/k8ssandratask_controller_test.go @@ -2,7 +2,6 @@ package control import ( "context" - "fmt" "testing" "time" @@ -10,7 +9,6 @@ import ( cassapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" k8capi "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" - "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "github.com/k8ssandra/k8ssandra-operator/pkg/clientcache" "github.com/k8ssandra/k8ssandra-operator/pkg/config" testutils "github.com/k8ssandra/k8ssandra-operator/pkg/test" @@ -73,7 +71,6 @@ func TestK8ssandraTask(t *testing.T) { t.Run("ExecuteSequentialK8ssandraTask", testEnv.ControllerTest(ctx, executeSequentialK8ssandraTask)) t.Run("DeleteK8ssandraTask", testEnv.ControllerTest(ctx, deleteK8ssandraTask)) t.Run("ExpireK8ssandraTask", testEnv.ControllerTest(ctx, expireK8ssandraTask)) - t.Run("RefreshK8ssandraCluster", testEnv.ControllerTest(ctx, refreshK8ssandraTask)) } // executeParallelK8ssandraTask creates and runs a K8ssandraTask with parallel DC processing. @@ -381,65 +378,6 @@ func expireK8ssandraTask(t *testing.T, ctx context.Context, f *framework.Framewo require.Eventually(func() bool { return !f.K8ssandraTaskExists(ctx, k8TaskKey)() }, timeout, interval) } -func refreshK8ssandraTask(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) { - require := require.New(t) - - kc := newCluster(namespace, "kc", - newDc("dc1", f.DataPlaneContexts[0])) - require.NoError(f.Client.Create(ctx, kc), "failed to create K8ssandraCluster") - - kc.Status.SetConditionStatus(k8capi.ClusterRequiresUpdate, corev1.ConditionTrue) - require.NoError(f.Client.Status().Update(ctx, kc)) - - dcConfig := cassandra.Coalesce(kc.CassClusterName(), kc.Spec.Cassandra.DeepCopy(), kc.Spec.Cassandra.Datacenters[0].DeepCopy()) - dc, err := cassandra.NewDatacenter(types.NamespacedName{Namespace: kc.Namespace, Name: kc.Name}, dcConfig) - require.NoError(err) - require.NoError(f.Client.Create(ctx, dc)) - - t.Log("Create a K8ssandraTask with TTL") - ttl := new(int32) - *ttl = 1 - k8Task := &api.K8ssandraTask{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "refresh", - }, - Spec: api.K8ssandraTaskSpec{ - Cluster: corev1.ObjectReference{ - Name: "kc", - }, - Template: cassapi.CassandraTaskTemplate{ - TTLSecondsAfterFinished: ttl, - Jobs: []cassapi.CassandraJob{{ - Name: "job1", - Command: "refresh", - }}, - }, - DcConcurrencyPolicy: batchv1.ForbidConcurrent, - }, - } - require.NoError(f.Client.Create(ctx, k8Task), "failed to create K8ssandraTask") - - require.Eventually(func() bool { - if err := f.Client.Get(ctx, types.NamespacedName{Namespace: kc.Namespace, Name: kc.Name}, kc); err != nil { - return false - } - if !metav1.HasAnnotation(kc.ObjectMeta, k8capi.AutomatedUpdateAnnotation) { - return false - } - return kc.Annotations[k8capi.AutomatedUpdateAnnotation] == string(k8capi.AllowUpdateOnce) - }, timeout, interval) - // First case, there's only changes in K8ssandraCluster - t.Log("Mark the K8ssandraCluster as updated if the annotation was added") - require.Equal(corev1.ConditionTrue, kc.Status.GetConditionStatus(k8capi.ClusterRequiresUpdate)) - kc.Status.SetConditionStatus(k8capi.ClusterRequiresUpdate, corev1.ConditionFalse) - require.NoError(f.Client.Status().Update(ctx, kc)) - - waitForTaskCompletion(ctx, t, f, newClusterKey(f.ControlPlaneContext, namespace, "refresh")) - // Second case, we also have changes in the CassandraDatacenter, even after updating K8ssandraCluster - -} - func newCluster(namespace, name string, dcs ...k8capi.CassandraDatacenterTemplate) *k8capi.K8ssandraCluster { return &k8capi.K8ssandraCluster{ ObjectMeta: metav1.ObjectMeta{ @@ -472,18 +410,6 @@ func newDc(name string, k8sContext string) k8capi.CassandraDatacenterTemplate { } } -func waitForTaskCompletion(ctx context.Context, t *testing.T, f *framework.Framework, taskKey framework.ClusterKey) { - require.Eventually(t, func() bool { - k8Task := &api.K8ssandraTask{} - require.NoError(t, f.Get(ctx, taskKey, k8Task)) - fmt.Printf("k8Task.Status: %+v\n", k8Task.Status) - return k8Task.Status.Active == 0 && - k8Task.Status.Succeeded > 0 && - k8Task.GetConditionStatus(cassapi.JobRunning) == metav1.ConditionFalse && - k8Task.GetConditionStatus(cassapi.JobComplete) == metav1.ConditionTrue - }, timeout, interval) -} - func newClusterKey(k8sContext, namespace, name string) framework.ClusterKey { return framework.ClusterKey{ NamespacedName: types.NamespacedName{Namespace: namespace, Name: name}, diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 378e69be4..fd25376d3 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -6,19 +6,23 @@ import ( "sort" "strconv" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/go-logr/logr" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" cassctlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" + ktaskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" telemetryapi "github.com/k8ssandra/k8ssandra-operator/apis/telemetry/v1alpha1" "github.com/k8ssandra/k8ssandra-operator/pkg/annotations" "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" + "github.com/k8ssandra/k8ssandra-operator/pkg/labels" "github.com/k8ssandra/k8ssandra-operator/pkg/result" "github.com/k8ssandra/k8ssandra-operator/pkg/secret" agent "github.com/k8ssandra/k8ssandra-operator/pkg/telemetry/cassandra_agent" "github.com/k8ssandra/k8ssandra-operator/pkg/utils" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -243,6 +247,54 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k } } + if AllowUpdate(kc) { + dcsRequiringUpdate := make([]string, 0, len(actualDcs)) + for _, dc := range actualDcs { + if dc.Status.GetConditionStatus("RequiresUpdate") == corev1.ConditionTrue { // TODO Update this to cassdcapi const when cass-operator is updatedß + dcsRequiringUpdate = append(dcsRequiringUpdate, dc.DatacenterName()) + } + } + + if len(dcsRequiringUpdate) > 0 { + generatedName := fmt.Sprintf("refresh-%d", time.Now().Unix()) + internalTask := &ktaskapi.K8ssandraTask{} + err := r.Get(ctx, types.NamespacedName{Namespace: kc.Namespace, Name: generatedName}, internalTask) + // If task wasn't found, create it and if task is still running, requeue + if errors.IsNotFound(err) { + // Delegate work to the task controller for Datacenter operations + task := &ktaskapi.K8ssandraTask{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kc.Namespace, + Name: generatedName, + Labels: labels.WatchedByK8ssandraClusterLabels(kcKey), + }, + Spec: ktaskapi.K8ssandraTaskSpec{ + Cluster: corev1.ObjectReference{ + Name: kc.Name, + }, + Datacenters: make([]string, len(dcsRequiringUpdate)), + Template: cassctlapi.CassandraTaskTemplate{ + Jobs: []cassctlapi.CassandraJob{{ + Name: fmt.Sprintf("refresh-%s", kc.Name), + Command: "refresh", + }}, + }, + DcConcurrencyPolicy: batchv1.ForbidConcurrent, + }, + } + + if err := r.Create(ctx, task); err != nil { + return result.Error(err), actualDcs + } + + return result.RequeueSoon(r.DefaultDelay), actualDcs + + } else if internalTask.Status.CompletionTime.IsZero() { + return result.RequeueSoon(r.DefaultDelay), actualDcs + } + } + } + // If we reach this point all CassandraDatacenters are ready. We only set the // CassandraInitialized condition if it is unset, i.e., only once. This allows us to // distinguish whether we are deploying a CassandraDatacenter as part of a new cluster diff --git a/controllers/k8ssandra/k8ssandracluster_controller.go b/controllers/k8ssandra/k8ssandracluster_controller.go index d9162ab55..dd3372d6d 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller.go +++ b/controllers/k8ssandra/k8ssandracluster_controller.go @@ -211,8 +211,6 @@ func (r *K8ssandraClusterReconciler) SetupWithManager(mgr ctrl.Manager, clusters cb := ctrl.NewControllerManagedBy(mgr). For(&api.K8ssandraCluster{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{}))) - // We need also annotation changes due to autoupdate-spec - clusterLabelFilter := func(ctx context.Context, mapObj client.Object) []reconcile.Request { requests := make([]reconcile.Request, 0) From b012b84047a308d171c056fbed36aa3a590edd40 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 31 May 2024 13:13:47 +0300 Subject: [PATCH 07/26] Change the name of the refresh task to be predictable --- controllers/k8ssandra/datacenters.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index fd25376d3..2969d3c7d 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -6,7 +6,6 @@ import ( "sort" "strconv" "strings" - "time" "github.com/Masterminds/semver/v3" "github.com/go-logr/logr" @@ -256,7 +255,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k } if len(dcsRequiringUpdate) > 0 { - generatedName := fmt.Sprintf("refresh-%d", time.Now().Unix()) + generatedName := fmt.Sprintf("refresh-%d-%d", kc.Generation, kc.Status.ObservedGeneration) internalTask := &ktaskapi.K8ssandraTask{} err := r.Get(ctx, types.NamespacedName{Namespace: kc.Namespace, Name: generatedName}, internalTask) // If task wasn't found, create it and if task is still running, requeue From bef0d3ad48c13f12f8bc7a7ea833a82277047944 Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Wed, 29 May 2024 17:22:55 -0700 Subject: [PATCH 08/26] Fix missing import --- controllers/k8ssandra/k8ssandracluster_controller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/k8ssandra/k8ssandracluster_controller_test.go b/controllers/k8ssandra/k8ssandracluster_controller_test.go index db576754c..59b3562a8 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller_test.go +++ b/controllers/k8ssandra/k8ssandracluster_controller_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "k8s.io/utils/pointer" "math/rand" "reflect" "testing" From 6e55caa4c41e1d9abaeedf86d852a4b588a31752 Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Fri, 31 May 2024 11:24:11 -0700 Subject: [PATCH 09/26] Use correct pointer lib --- controllers/k8ssandra/k8ssandracluster_controller_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/controllers/k8ssandra/k8ssandracluster_controller_test.go b/controllers/k8ssandra/k8ssandracluster_controller_test.go index 59b3562a8..7fd3c30c5 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller_test.go +++ b/controllers/k8ssandra/k8ssandracluster_controller_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "k8s.io/utils/pointer" "math/rand" "reflect" "testing" @@ -2690,7 +2689,7 @@ func testGenerationCheck(t *testing.T, ctx context.Context, f *framework.Framewo }, }, PodSecurityContext: &corev1.PodSecurityContext{ - RunAsUser: pointer.Int64(999), + RunAsUser: ptr.To(int64(999)), }, ManagementApiAuth: &cassdcapi.ManagementApiAuthConfig{ Insecure: &cassdcapi.ManagementApiAuthInsecureConfig{}, From a3ce8bc93492282484a97208286d27fde58eb023 Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Fri, 31 May 2024 11:47:13 -0700 Subject: [PATCH 10/26] Re-add GenerationChanged() improvement Since ObservedGeneration is being added by this PR, it will be 0 when the operator gets upgraded to this version. We want to interpret that as "the generation didn't change". This was agreed upon previously but got lost in a force-push. --- apis/k8ssandra/v1alpha1/k8ssandracluster_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go index 7e44c5cdf..f60306743 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go @@ -566,5 +566,5 @@ func (kc *K8ssandraCluster) GetClusterIdHash() string { } func (k *K8ssandraCluster) GenerationChanged() bool { - return k.Status.ObservedGeneration < k.Generation + return k.Status.ObservedGeneration != 0 && k.Status.ObservedGeneration < k.Generation } From cdf373bb26cb2f5353f374091313149275d4aac9 Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Fri, 31 May 2024 15:54:51 -0700 Subject: [PATCH 11/26] Fix DC list in task --- controllers/k8ssandra/datacenters.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 2969d3c7d..0c9295e80 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -249,8 +249,8 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k if AllowUpdate(kc) { dcsRequiringUpdate := make([]string, 0, len(actualDcs)) for _, dc := range actualDcs { - if dc.Status.GetConditionStatus("RequiresUpdate") == corev1.ConditionTrue { // TODO Update this to cassdcapi const when cass-operator is updatedß - dcsRequiringUpdate = append(dcsRequiringUpdate, dc.DatacenterName()) + if dc.Status.GetConditionStatus("RequiresUpdate") == corev1.ConditionTrue { // TODO Update this to cassdcapi const when cass-operator is updated + dcsRequiringUpdate = append(dcsRequiringUpdate, dc.ObjectMeta.Name) } } @@ -271,7 +271,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k Cluster: corev1.ObjectReference{ Name: kc.Name, }, - Datacenters: make([]string, len(dcsRequiringUpdate)), + Datacenters: dcsRequiringUpdate, Template: cassctlapi.CassandraTaskTemplate{ Jobs: []cassctlapi.CassandraJob{{ Name: fmt.Sprintf("refresh-%s", kc.Name), From 7bbb1feaf475f7f3acb8a03cafdd4e29f0bf0b1c Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Fri, 31 May 2024 16:11:36 -0700 Subject: [PATCH 12/26] Revert unnecessary change --- controllers/control/k8ssandratask_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/control/k8ssandratask_controller.go b/controllers/control/k8ssandratask_controller.go index e63fc30a7..71388fbec 100644 --- a/controllers/control/k8ssandratask_controller.go +++ b/controllers/control/k8ssandratask_controller.go @@ -148,7 +148,6 @@ func (r *K8ssandraTaskReconciler) Reconcile(ctx context.Context, req ctrl.Reques if !kcExists { return r.reportInvalidSpec(ctx, kTask, "unknown K8ssandraCluster %s.%s", kcKey.Namespace, kcKey.Name) } - if dcs, err := filterDcs(kc, kTask.Spec.Datacenters); err != nil { return r.reportInvalidSpec(ctx, kTask, err.Error()) } else { From f1b82944d1460f41313add19531b836e35cb91f2 Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Fri, 31 May 2024 16:12:42 -0700 Subject: [PATCH 13/26] Fix error message --- controllers/k8ssandra/datacenters.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 0c9295e80..6cd6d4935 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -156,7 +156,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k patch := client.MergeFrom(kc.DeepCopy()) kc.Status.SetConditionStatus(api.ClusterRequiresUpdate, corev1.ConditionTrue) if err := r.Client.Status().Patch(ctx, kc, patch); err != nil { - return result.Error(fmt.Errorf("failed to set %s annotation: %v", api.AutomatedUpdateAnnotation, err)), actualDcs + return result.Error(fmt.Errorf("failed to set %s condition: %v", api.ClusterRequiresUpdate, err)), actualDcs } } else if !annotations.CompareHashAnnotations(actualDc, desiredDc) { if actualDc.Spec.SuperuserSecretName != desiredDc.Spec.SuperuserSecretName { From 55c07cfa23c80bb2d70048b03563be194cc83ce0 Mon Sep 17 00:00:00 2001 From: Olivier Michallat Date: Fri, 31 May 2024 17:15:14 -0700 Subject: [PATCH 14/26] Fix envtests --- controllers/k8ssandra/add_dc_test.go | 10 ++++++---- controllers/k8ssandra/stop_dc_test.go | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/controllers/k8ssandra/add_dc_test.go b/controllers/k8ssandra/add_dc_test.go index 71daec2b0..6c30a6855 100644 --- a/controllers/k8ssandra/add_dc_test.go +++ b/controllers/k8ssandra/add_dc_test.go @@ -50,8 +50,9 @@ func addDcSetupForSingleDc(ctx context.Context, t *testing.T, f *framework.Frame require := require.New(t) kc := &api.K8ssandraCluster{ ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "add-dc-test", + Namespace: namespace, + Name: "add-dc-test", + Annotations: map[string]string{api.AutomatedUpdateAnnotation: string(api.AllowUpdateAlways)}, }, Spec: api.K8ssandraClusterSpec{ Cassandra: &api.CassandraClusterTemplate{ @@ -119,8 +120,9 @@ func addDcSetupForMultiDc(ctx context.Context, t *testing.T, f *framework.Framew require := require.New(t) kc := &api.K8ssandraCluster{ ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "add-dc-test", + Namespace: namespace, + Name: "add-dc-test", + Annotations: map[string]string{api.AutomatedUpdateAnnotation: string(api.AllowUpdateAlways)}, }, Spec: api.K8ssandraClusterSpec{ Cassandra: &api.CassandraClusterTemplate{ diff --git a/controllers/k8ssandra/stop_dc_test.go b/controllers/k8ssandra/stop_dc_test.go index f25dd1784..812a46c3b 100644 --- a/controllers/k8ssandra/stop_dc_test.go +++ b/controllers/k8ssandra/stop_dc_test.go @@ -49,8 +49,9 @@ func stopDcTestSetup(t *testing.T, f *framework.Framework, ctx context.Context, kc := &api.K8ssandraCluster{ ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "stop-dc-test", + Namespace: namespace, + Name: "stop-dc-test", + Annotations: map[string]string{api.AutomatedUpdateAnnotation: string(api.AllowUpdateAlways)}, }, Spec: api.K8ssandraClusterSpec{ Cassandra: &api.CassandraClusterTemplate{ From c40d32ff37a95103aee9c55612cd2ab61e7bae0b Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 25 Jun 2024 13:25:39 +0300 Subject: [PATCH 15/26] Update to cass-operator v1.21.1 --- controllers/k8ssandra/datacenters.go | 2 +- pkg/cassandra/datacenter.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 6cd6d4935..8478f4e34 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -249,7 +249,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k if AllowUpdate(kc) { dcsRequiringUpdate := make([]string, 0, len(actualDcs)) for _, dc := range actualDcs { - if dc.Status.GetConditionStatus("RequiresUpdate") == corev1.ConditionTrue { // TODO Update this to cassdcapi const when cass-operator is updated + if dc.Status.GetConditionStatus(cassdcapi.DatacenterRequiresUpdate) == corev1.ConditionTrue { dcsRequiringUpdate = append(dcsRequiringUpdate, dc.ObjectMeta.Name) } } diff --git a/pkg/cassandra/datacenter.go b/pkg/cassandra/datacenter.go index 1162a96e3..660b6e644 100644 --- a/pkg/cassandra/datacenter.go +++ b/pkg/cassandra/datacenter.go @@ -158,9 +158,6 @@ func NewDatacenter(klusterKey types.NamespacedName, template *DatacenterConfig) ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name: template.Meta.Name, - Annotations: map[string]string{ - cassdcapi.UpdateAllowedAnnotation: string(cassdcapi.AllowUpdateAlways), - }, Labels: utils.MergeMap(map[string]string{ api.NameLabel: api.NameLabelValue, api.PartOfLabel: api.PartOfLabelValue, From 911e17359626831da835d284990cf81af4652990 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 23 Jul 2024 12:30:24 +0300 Subject: [PATCH 16/26] Add documentation --- docs/content/en/install/_index.md | 4 ++++ docs/content/en/install/upgrade/_index.md | 29 +++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 docs/content/en/install/upgrade/_index.md diff --git a/docs/content/en/install/_index.md b/docs/content/en/install/_index.md index b541226ef..c57530484 100644 --- a/docs/content/en/install/_index.md +++ b/docs/content/en/install/_index.md @@ -25,4 +25,8 @@ If you are using a cloud provider, explore the following topics for cloud-specif * [DigitalOcean Kubernetes]({{< relref "install/doks/" >}}) (DOKS) * [Google Kubernetes Engine]({{< relref "install/gke/" >}}) (GKE) +## Upgrade + +* [Upgrade notes]({{< relref "install/upgrade/" >}}) (Upgrade notes) + **Tip:** For an architectural overview of K8ssandra Operator and its new `K8ssandraCluster` custom resource, see the [K8ssandra Operator]({{< relref "components/k8ssandra-operator/" >}}) component page. diff --git a/docs/content/en/install/upgrade/_index.md b/docs/content/en/install/upgrade/_index.md new file mode 100644 index 000000000..13eb07096 --- /dev/null +++ b/docs/content/en/install/upgrade/_index.md @@ -0,0 +1,29 @@ +--- +title: "Upgrade notes" +linkTitle: "Docs" +no_list: true +weight: 2 +description: "Notes on upgrading existing installation" +--- + +Upgrading the operators is usually a straight-forward operation based on the standard installation method's upgrade procedure. In certain cases however, updates will require certain manual processing to avoid disruptions to the running clusters. + +## Updates after operator upgrade to running Cassandra clusters + +Sometimes the updates to operators might bring new features or improvements to existing running Cassandra clusters. However, starting from release 1.18 we will no longer update them automatically when the operators are upgraded to prevent a rolling restart on inconvenient time. If there are changes to be applied after upgrading, the ``K8ssandraCluster`` instances are marked with a Status Condition ``RequiresUpdate`` set to True. + +The updates are applied automatically if the ``K8ssandraCluster`` Spec is modified. However, since this is not often necessary the alternative way to apply the updates is to place an annotation on the ``K8ssandraCluster`` (in ``metadata.annotations``). The annotation ``k8ssandra.io/autoupdate-spec`` has two accepted values, ``once`` and ``always``. When setting the value to ``once``, the clusters are upgraded with a rolling restart (if needed) and then the annotation is removed. If set to ``always`` the cluster is upgraded yet the annotation is not removed and the old behavior of updating the clusters as soon as operator is upgraded will be used. + +Example of setting the annotation: + +```yaml +apiVersion: k8ssandra.io/v1alpha1 +kind: K8ssandraCluster +metadata: + name: test-cluster + annotations: + k8ssandra.io/autoupdate-spec: "always" +spec: +``` + +We recommend updating the clusters after upgrading the operators as it will also apply newer images to running clusters which could have CVEs or bugs fixed. From 7501db7b1aa87f7369b12042ebb908483e3320dd Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 23 Jul 2024 13:22:08 +0300 Subject: [PATCH 17/26] Fix grammar --- docs/content/en/install/upgrade/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/en/install/upgrade/_index.md b/docs/content/en/install/upgrade/_index.md index 13eb07096..f42e2df55 100644 --- a/docs/content/en/install/upgrade/_index.md +++ b/docs/content/en/install/upgrade/_index.md @@ -10,7 +10,7 @@ Upgrading the operators is usually a straight-forward operation based on the sta ## Updates after operator upgrade to running Cassandra clusters -Sometimes the updates to operators might bring new features or improvements to existing running Cassandra clusters. However, starting from release 1.18 we will no longer update them automatically when the operators are upgraded to prevent a rolling restart on inconvenient time. If there are changes to be applied after upgrading, the ``K8ssandraCluster`` instances are marked with a Status Condition ``RequiresUpdate`` set to True. +Sometimes the updates to operators might bring new features or improvements to existing running Cassandra clusters. However, starting from release 1.18 we will no longer update them automatically when the operators are upgraded to prevent a rolling restart at an inconvenient time. If there are changes to be applied after upgrading, the ``K8ssandraCluster`` instances are marked with a Status Condition ``RequiresUpdate`` set to True. The updates are applied automatically if the ``K8ssandraCluster`` Spec is modified. However, since this is not often necessary the alternative way to apply the updates is to place an annotation on the ``K8ssandraCluster`` (in ``metadata.annotations``). The annotation ``k8ssandra.io/autoupdate-spec`` has two accepted values, ``once`` and ``always``. When setting the value to ``once``, the clusters are upgraded with a rolling restart (if needed) and then the annotation is removed. If set to ``always`` the cluster is upgraded yet the annotation is not removed and the old behavior of updating the clusters as soon as operator is upgraded will be used. From 7a4ac6984ce9e727f862618ec07a0cbb8a8f6516 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 23 Jul 2024 17:21:00 +0300 Subject: [PATCH 18/26] Add reconcile request after updating the Datacenter --- controllers/k8ssandra/datacenters.go | 2 ++ controllers/k8ssandra/k8ssandracluster_controller.go | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 8478f4e34..b96d4fa07 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -191,6 +191,8 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k dcLogger.Error(err, "Failed to update datacenter") return result.Error(err), actualDcs } + + return result.RequeueSoon(r.DefaultDelay), actualDcs } if actualDc.Spec.Stopped { diff --git a/controllers/k8ssandra/k8ssandracluster_controller.go b/controllers/k8ssandra/k8ssandracluster_controller.go index dd3372d6d..f7235e77f 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller.go +++ b/controllers/k8ssandra/k8ssandracluster_controller.go @@ -18,6 +18,7 @@ package k8ssandra import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -156,7 +157,7 @@ func (r *K8ssandraClusterReconciler) reconcile(ctx context.Context, kc *api.K8ss return recResult.Output() } - if res := updateStatus(ctx, r.Client, kc); res.Completed() { + if res := updateStatus(ctx, r.Client, kc, kcLogger); res.Completed() { return res.Output() } @@ -185,7 +186,7 @@ func (r *K8ssandraClusterReconciler) afterCassandraReconciled(ctx context.Contex return result.Continue() } -func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster) result.ReconcileResult { +func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster, kcLogger logr.Logger) result.ReconcileResult { if AllowUpdate(kc) { if metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) { if kc.Annotations[api.AutomatedUpdateAnnotation] == string(api.AllowUpdateOnce) { @@ -198,6 +199,7 @@ func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster kc.Status.SetConditionStatus(api.ClusterRequiresUpdate, corev1.ConditionFalse) } + kcLogger.Info(fmt.Sprintf("Updating observed generation to %d", kc.Generation)) kc.Status.ObservedGeneration = kc.Generation if err := r.Status().Update(ctx, kc); err != nil { return result.Error(err) From a42e6c1030cf87e1470bd6cc9655def52ba0c851 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 23 Jul 2024 18:35:55 +0300 Subject: [PATCH 19/26] More logging.. --- controllers/k8ssandra/datacenters.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index b96d4fa07..83ae0b2bd 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -33,7 +33,8 @@ const ( rebuildNodesLabel = "k8ssandra.io/rebuild-nodes" ) -func AllowUpdate(kc *api.K8ssandraCluster) bool { +func AllowUpdate(kc *api.K8ssandraCluster, logger logr.Logger) bool { + logger.Info(fmt.Sprintf("Generation: %d, ObservedGeneration: %d", kc.Generation, kc.Status.ObservedGeneration)) return kc.GenerationChanged() || metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) } @@ -150,7 +151,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k r.setStatusForDatacenter(kc, actualDc) - if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc) { + if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc, logger) { logger.Info("Datacenter requires an update, but we're not allowed to do it", "CassandraDatacenter", dcKey) // We're not allowed to update, but need to patch := client.MergeFrom(kc.DeepCopy()) @@ -248,7 +249,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k } } - if AllowUpdate(kc) { + if AllowUpdate(kc, logger) { dcsRequiringUpdate := make([]string, 0, len(actualDcs)) for _, dc := range actualDcs { if dc.Status.GetConditionStatus(cassdcapi.DatacenterRequiresUpdate) == corev1.ConditionTrue { From 94593106a8c2fd9f9e667778b5cc06e36b227d18 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Jul 2024 13:35:48 +0300 Subject: [PATCH 20/26] Fix rebase --- controllers/k8ssandra/k8ssandracluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/k8ssandra/k8ssandracluster_controller.go b/controllers/k8ssandra/k8ssandracluster_controller.go index f7235e77f..b948a8800 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller.go +++ b/controllers/k8ssandra/k8ssandracluster_controller.go @@ -187,7 +187,7 @@ func (r *K8ssandraClusterReconciler) afterCassandraReconciled(ctx context.Contex } func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster, kcLogger logr.Logger) result.ReconcileResult { - if AllowUpdate(kc) { + if AllowUpdate(kc, kcLogger) { if metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) { if kc.Annotations[api.AutomatedUpdateAnnotation] == string(api.AllowUpdateOnce) { delete(kc.ObjectMeta.Annotations, api.AutomatedUpdateAnnotation) From 3f3383e5502adb1f14194284b700fb18d71f5c95 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Jul 2024 14:53:43 +0300 Subject: [PATCH 21/26] Even more logging.. --- controllers/k8ssandra/datacenters.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index 83ae0b2bd..d7f60e4cd 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -9,6 +9,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" cassctlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" ktaskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" @@ -151,6 +152,9 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k r.setStatusForDatacenter(kc, actualDc) + dcDiff := cmp.Diff(actualDc, desiredDc) + logger.Info(fmt.Sprintf("ObservedGeneration: %d, Generation: %d, Datacenter: %v, dcDiff: %s", actualDc.Status.ObservedGeneration, actualDc.GetGeneration(), dcKey, dcDiff)) + if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc, logger) { logger.Info("Datacenter requires an update, but we're not allowed to do it", "CassandraDatacenter", dcKey) // We're not allowed to update, but need to From c5dff9857dcb2dcef550247656c2f8d7e092c3aa Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Jul 2024 16:19:10 +0300 Subject: [PATCH 22/26] Revert changes in a3ce8bc93492282484a97208286d27fde58eb023 but add tests to ensure the field is working properly --- .../v1alpha1/k8ssandracluster_types.go | 2 +- .../v1alpha1/k8ssandracluster_types_test.go | 21 +++++++++++++++ controllers/k8ssandra/datacenters.go | 11 +++----- controllers/k8ssandra/datacenters_test.go | 27 ++++++++++++++++++- .../k8ssandra/k8ssandracluster_controller.go | 8 +++--- 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go index f60306743..7e44c5cdf 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go @@ -566,5 +566,5 @@ func (kc *K8ssandraCluster) GetClusterIdHash() string { } func (k *K8ssandraCluster) GenerationChanged() bool { - return k.Status.ObservedGeneration != 0 && k.Status.ObservedGeneration < k.Generation + return k.Status.ObservedGeneration < k.Generation } diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_types_test.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_types_test.go index 3ab6bafd7..154620021 100644 --- a/apis/k8ssandra/v1alpha1/k8ssandracluster_types_test.go +++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_types_test.go @@ -3,6 +3,7 @@ package v1alpha1 import ( "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" "sigs.k8s.io/yaml" @@ -194,3 +195,23 @@ metadata: assert.Equal(t, "nodePortSvcLabelValue1", dc.Meta.ServiceConfig.NodePortService.Labels["nodePortSvcLabel1"]) assert.Equal(t, "nodePortSvcAnnotationValue1", dc.Meta.ServiceConfig.NodePortService.Annotations["nodePortSvcAnnotation1"]) } + +func TestGenerationChanged(t *testing.T) { + assert := assert.New(t) + kc := &K8ssandraCluster{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + }, + Spec: K8ssandraClusterSpec{}, + } + + kc.Status = K8ssandraClusterStatus{ + ObservedGeneration: 0, + } + + assert.True(kc.GenerationChanged()) + kc.Status.ObservedGeneration = 2 + assert.False(kc.GenerationChanged()) + kc.ObjectMeta.Generation = 3 + assert.True(kc.GenerationChanged()) +} diff --git a/controllers/k8ssandra/datacenters.go b/controllers/k8ssandra/datacenters.go index d7f60e4cd..b96d4fa07 100644 --- a/controllers/k8ssandra/datacenters.go +++ b/controllers/k8ssandra/datacenters.go @@ -9,7 +9,6 @@ import ( "github.com/Masterminds/semver/v3" "github.com/go-logr/logr" - "github.com/google/go-cmp/cmp" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" cassctlapi "github.com/k8ssandra/cass-operator/apis/control/v1alpha1" ktaskapi "github.com/k8ssandra/k8ssandra-operator/apis/control/v1alpha1" @@ -34,8 +33,7 @@ const ( rebuildNodesLabel = "k8ssandra.io/rebuild-nodes" ) -func AllowUpdate(kc *api.K8ssandraCluster, logger logr.Logger) bool { - logger.Info(fmt.Sprintf("Generation: %d, ObservedGeneration: %d", kc.Generation, kc.Status.ObservedGeneration)) +func AllowUpdate(kc *api.K8ssandraCluster) bool { return kc.GenerationChanged() || metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) } @@ -152,10 +150,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k r.setStatusForDatacenter(kc, actualDc) - dcDiff := cmp.Diff(actualDc, desiredDc) - logger.Info(fmt.Sprintf("ObservedGeneration: %d, Generation: %d, Datacenter: %v, dcDiff: %s", actualDc.Status.ObservedGeneration, actualDc.GetGeneration(), dcKey, dcDiff)) - - if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc, logger) { + if !annotations.CompareHashAnnotations(actualDc, desiredDc) && !AllowUpdate(kc) { logger.Info("Datacenter requires an update, but we're not allowed to do it", "CassandraDatacenter", dcKey) // We're not allowed to update, but need to patch := client.MergeFrom(kc.DeepCopy()) @@ -253,7 +248,7 @@ func (r *K8ssandraClusterReconciler) reconcileDatacenters(ctx context.Context, k } } - if AllowUpdate(kc, logger) { + if AllowUpdate(kc) { dcsRequiringUpdate := make([]string, 0, len(actualDcs)) for _, dc := range actualDcs { if dc.Status.GetConditionStatus(cassdcapi.DatacenterRequiresUpdate) == corev1.ConditionTrue { diff --git a/controllers/k8ssandra/datacenters_test.go b/controllers/k8ssandra/datacenters_test.go index 7f13a92c2..dd6e35b2b 100644 --- a/controllers/k8ssandra/datacenters_test.go +++ b/controllers/k8ssandra/datacenters_test.go @@ -1,6 +1,8 @@ package k8ssandra import ( + "testing" + "github.com/Masterminds/semver/v3" cassdcapi "github.com/k8ssandra/cass-operator/apis/cassandra/v1beta1" api "github.com/k8ssandra/k8ssandra-operator/apis/k8ssandra/v1alpha1" @@ -8,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" ) var ( @@ -263,3 +264,27 @@ func TestGetSourceDatacenterName_Conflict(t *testing.T) { } } + +func TestAllowUpdate(t *testing.T) { + assert := assert.New(t) + kc := &api.K8ssandraCluster{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 2, + }, + Spec: api.K8ssandraClusterSpec{}, + } + + kc.Status = api.K8ssandraClusterStatus{ + ObservedGeneration: 0, + } + + assert.True(AllowUpdate(kc)) + kc.Status.ObservedGeneration = 1 + assert.True(AllowUpdate(kc)) + kc.Status.ObservedGeneration = 2 + assert.False(AllowUpdate(kc)) + metav1.SetMetaDataAnnotation(&kc.ObjectMeta, api.AutomatedUpdateAnnotation, string(api.AllowUpdateOnce)) + assert.True(AllowUpdate(kc)) + metav1.SetMetaDataAnnotation(&kc.ObjectMeta, api.AutomatedUpdateAnnotation, string(api.AllowUpdateAlways)) + assert.True(AllowUpdate(kc)) +} diff --git a/controllers/k8ssandra/k8ssandracluster_controller.go b/controllers/k8ssandra/k8ssandracluster_controller.go index b948a8800..dd3372d6d 100644 --- a/controllers/k8ssandra/k8ssandracluster_controller.go +++ b/controllers/k8ssandra/k8ssandracluster_controller.go @@ -18,7 +18,6 @@ package k8ssandra import ( "context" - "fmt" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" @@ -157,7 +156,7 @@ func (r *K8ssandraClusterReconciler) reconcile(ctx context.Context, kc *api.K8ss return recResult.Output() } - if res := updateStatus(ctx, r.Client, kc, kcLogger); res.Completed() { + if res := updateStatus(ctx, r.Client, kc); res.Completed() { return res.Output() } @@ -186,8 +185,8 @@ func (r *K8ssandraClusterReconciler) afterCassandraReconciled(ctx context.Contex return result.Continue() } -func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster, kcLogger logr.Logger) result.ReconcileResult { - if AllowUpdate(kc, kcLogger) { +func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster) result.ReconcileResult { + if AllowUpdate(kc) { if metav1.HasAnnotation(kc.ObjectMeta, api.AutomatedUpdateAnnotation) { if kc.Annotations[api.AutomatedUpdateAnnotation] == string(api.AllowUpdateOnce) { delete(kc.ObjectMeta.Annotations, api.AutomatedUpdateAnnotation) @@ -199,7 +198,6 @@ func updateStatus(ctx context.Context, r client.Client, kc *api.K8ssandraCluster kc.Status.SetConditionStatus(api.ClusterRequiresUpdate, corev1.ConditionFalse) } - kcLogger.Info(fmt.Sprintf("Updating observed generation to %d", kc.Generation)) kc.Status.ObservedGeneration = kc.Generation if err := r.Status().Update(ctx, kc); err != nil { return result.Error(err) From a9a9a802981427900c77a29027a559c4d8e26a29 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Jul 2024 17:33:56 +0300 Subject: [PATCH 23/26] Add ObservedGeneration set in the upgrade test --- test/e2e/suite_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 0ad85657c..79fa569a7 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -955,6 +955,11 @@ func createSingleDatacenterClusterWithUpgrade(t *testing.T, ctx context.Context, require.NoError(f.Get(ctx, dcKey, cassdc)) dcHash := cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] dcPrefix := DcPrefix(t, f, dcKey) + verifyClusterReconcileFinished(ctx, t, f, k8ssandra) + require.NoError(f.Get(ctx, kcKey, k8ssandra)) + // We have to do this, because the old version being installed at first doesn't have this field + k8ssandra.Status.ObservedGeneration = k8ssandra.Generation + require.NoError(f.Client.Status().Update(ctx, k8ssandra)) // Perform the upgrade err = upgradeToLatest(t, ctx, f, namespace, dcPrefix) From 753259d5a0d47f404c39c517c5d5e20b119dc2f1 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Wed, 24 Jul 2024 18:05:06 +0300 Subject: [PATCH 24/26] Remove the hash check part from test after the ObservedGeneration != 0 change --- test/e2e/suite_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 79fa569a7..877d559fe 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -953,9 +953,7 @@ func createSingleDatacenterClusterWithUpgrade(t *testing.T, ctx context.Context, assertCassandraDatacenterK8cStatusReady(ctx, t, f, kcKey.NamespacedName, dcKey.Name) cassdc := &cassdcapi.CassandraDatacenter{} require.NoError(f.Get(ctx, dcKey, cassdc)) - dcHash := cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] dcPrefix := DcPrefix(t, f, dcKey) - verifyClusterReconcileFinished(ctx, t, f, k8ssandra) require.NoError(f.Get(ctx, kcKey, k8ssandra)) // We have to do this, because the old version being installed at first doesn't have this field k8ssandra.Status.ObservedGeneration = k8ssandra.Generation @@ -967,9 +965,7 @@ func createSingleDatacenterClusterWithUpgrade(t *testing.T, ctx context.Context, verifyClusterReconcileFinished(ctx, t, f, k8ssandra) require.NoError(f.Get(ctx, dcKey, cassdc)) - newDcHash := cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] - require.Equal(dcHash, newDcHash, "CassandraDatacenter resource hash changed after upgrade") require.NoError(f.Get(ctx, kcKey, k8ssandra)) require.Equal(corev1.ConditionTrue, k8ssandra.Status.GetConditionStatus(api.ClusterRequiresUpdate)) @@ -977,8 +973,6 @@ func createSingleDatacenterClusterWithUpgrade(t *testing.T, ctx context.Context, require.NoError(f.Update(ctx, kcKey, k8ssandra)) verifyClusterReconcileFinished(ctx, t, f, k8ssandra) require.NoError(f.Get(ctx, dcKey, cassdc)) - newDcHash = cassdc.ObjectMeta.Annotations[api.ResourceHashAnnotation] - require.NotEqual(dcHash, newDcHash, "CassandraDatacenter resource hash hasn't changed after upgrade") } // createSingleDatacenterCluster creates a K8ssandraCluster with one CassandraDatacenter From afd620195c9426094b867adf815cfeec5eadbc97 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 25 Jul 2024 12:21:49 +0300 Subject: [PATCH 25/26] Return upgrade test to the original one --- test/e2e/suite_test.go | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 877d559fe..36e07acee 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -44,7 +44,6 @@ import ( "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" ) @@ -943,36 +942,19 @@ func createSingleDatacenterClusterWithUpgrade(t *testing.T, ctx context.Context, require := require.New(t) t.Log("check that the K8ssandraCluster was created") - kcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "test"}} k8ssandra := &api.K8ssandraCluster{} - err := f.Get(ctx, kcKey, k8ssandra) + kcKey := types.NamespacedName{Namespace: namespace, Name: "test"} + err := f.Client.Get(ctx, kcKey, k8ssandra) require.NoError(err, "failed to get K8ssandraCluster in namespace %s", namespace) dcKey := framework.ClusterKey{K8sContext: f.DataPlaneContexts[0], NamespacedName: types.NamespacedName{Namespace: namespace, Name: "dc1"}} checkDatacenterReady(t, ctx, dcKey, f) - assertCassandraDatacenterK8cStatusReady(ctx, t, f, kcKey.NamespacedName, dcKey.Name) - cassdc := &cassdcapi.CassandraDatacenter{} - require.NoError(f.Get(ctx, dcKey, cassdc)) + assertCassandraDatacenterK8cStatusReady(ctx, t, f, kcKey, dcKey.Name) dcPrefix := DcPrefix(t, f, dcKey) - require.NoError(f.Get(ctx, kcKey, k8ssandra)) - // We have to do this, because the old version being installed at first doesn't have this field - k8ssandra.Status.ObservedGeneration = k8ssandra.Generation - require.NoError(f.Client.Status().Update(ctx, k8ssandra)) // Perform the upgrade err = upgradeToLatest(t, ctx, f, namespace, dcPrefix) require.NoError(err, "failed to upgrade to latest version") - - verifyClusterReconcileFinished(ctx, t, f, k8ssandra) - require.NoError(f.Get(ctx, dcKey, cassdc)) - - require.NoError(f.Get(ctx, kcKey, k8ssandra)) - - require.Equal(corev1.ConditionTrue, k8ssandra.Status.GetConditionStatus(api.ClusterRequiresUpdate)) - metav1.SetMetaDataAnnotation(&k8ssandra.ObjectMeta, api.AutomatedUpdateAnnotation, "always") - require.NoError(f.Update(ctx, kcKey, k8ssandra)) - verifyClusterReconcileFinished(ctx, t, f, k8ssandra) - require.NoError(f.Get(ctx, dcKey, cassdc)) } // createSingleDatacenterCluster creates a K8ssandraCluster with one CassandraDatacenter From e2f21d362728d0f04052eb0faa75a99b9bff10e0 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Thu, 25 Jul 2024 13:02:36 +0300 Subject: [PATCH 26/26] Remove verifyReconcileFinished --- test/e2e/suite_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 36e07acee..7ead381c4 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -2251,17 +2251,3 @@ func CheckLabelsAnnotationsCreated(dcKey framework.ClusterKey, t *testing.T, ctx assert.True(t, cassDC.Spec.AdditionalAnnotations["anAnnotationKeyClusterLevel"] == "anAnnotationValueClusterLevel") return nil } - -func verifyClusterReconcileFinished(ctx context.Context, t *testing.T, f *framework.E2eFramework, kc *api.K8ssandraCluster) { - t.Log("check K8ssandraCluster reconciliation finished") - key := client.ObjectKey{Namespace: kc.Namespace, Name: kc.Name} - - assert.Eventually(t, func() bool { - kc := &api.K8ssandraCluster{} - if err := f.Client.Get(ctx, key, kc); err != nil { - t.Logf("failed to get K8ssandraCluster: %v", err) - return false - } - return kc.ObjectMeta.Generation == kc.Status.ObservedGeneration - }, polling.k8ssandraClusterStatus.timeout, polling.k8ssandraClusterStatus.interval, "cluster hasn't finished reconciliation") -}