Deploy Reaper in Control Plane mode + register k8ssandra clusters int…

…o it

CHANGELOG/CHANGELOG-1.19.md

@@ -14,3 +14,5 @@ Changelog for the K8ssandra Operator, new PRs should update the `unreleased` sec
 When cutting a new release, update the `unreleased` heading to the tag being generated and date, like `## vX.Y.Z - YYYY-MM-DD` and create a new placeholder section for  `unreleased` entries.
 
 ## unreleased
+
+* [FEATURE] [#1277](https://github.com/k8ssandra/k8ssandra-operator/issues/1277) Introduce Reaper's Control Plane deployment mode

apis/k8ssandra/v1alpha1/k8ssandracluster_types.go

@@ -55,7 +55,8 @@ type K8ssandraClusterSpec struct {
 	Stargate *stargateapi.StargateClusterTemplate `json:"stargate,omitempty"`
 
 	// Reaper defines the desired deployment characteristics for Reaper in this K8ssandraCluster.
-	// If this is non-nil, Reaper will be deployed on every Cassandra datacenter in this K8ssandraCluster.
+	// If this is non-nil, Reaper might be deployed on every Cassandra datacenter in this K8ssandraCluster, unless
+	// there is a Control Plane Reaper present. In that case, the K8ssandraCluster will get registered to it.
 	// +optional
 	Reaper *reaperapi.ReaperClusterTemplate `json:"reaper,omitempty"`
 

apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go

@@ -46,6 +46,7 @@ var (
 	ErrNoReaperAccessMode       = fmt.Errorf("reaper StorageConfig.AccessModes not set")
 	ErrNoReaperResourceRequests = fmt.Errorf("reaper StorageConfig.Resources.Requests not set")
 	ErrNoReaperStorageRequest   = fmt.Errorf("reaper StorageConfig.Resources.Requests.Storage not set")
+	ErrNoReaperPerDcWithLocal   = fmt.Errorf("reaper DeploymentModePerDc is only supported when using cassandra storage")
 )
 
 // log is for logging in this package.
@@ -294,21 +295,25 @@ func (r *K8ssandraCluster) validateReaper() error {
 	if r.Spec.Reaper == nil {
 		return nil
 	}
-	if r.Spec.Reaper.StorageType != reaperapi.StorageTypeLocal {
-		return nil
-	}
-	if r.Spec.Reaper.StorageConfig == nil {
+	if r.Spec.Reaper.StorageType == reaperapi.StorageTypeLocal && r.Spec.Reaper.StorageConfig == nil {
 		return ErrNoReaperStorageConfig
 	}
-	// not checking StorageClassName because Kubernetes will use a default one if it's not set
-	if r.Spec.Reaper.StorageConfig.AccessModes == nil {
-		return ErrNoReaperAccessMode
-	}
-	if r.Spec.Reaper.StorageConfig.Resources.Requests == nil {
-		return ErrNoReaperResourceRequests
+	if r.Spec.Reaper.StorageType == reaperapi.StorageTypeLocal {
+		// we're checking for validity of the storage config in Reaper's webhook too, so this is a duplicate of that
+		// for now, I don't see a better way of reusing code to validate the storage config
+		// not checking StorageClassName because Kubernetes will use a default one if it's not set
+		if r.Spec.Reaper.StorageConfig.AccessModes == nil {
+			return ErrNoReaperAccessMode
+		}
+		if r.Spec.Reaper.StorageConfig.Resources.Requests == nil {
+			return ErrNoReaperResourceRequests
+		}
+		if r.Spec.Reaper.StorageConfig.Resources.Requests.Storage().IsZero() {
+			return ErrNoReaperStorageRequest
+		}
 	}
-	if r.Spec.Reaper.StorageConfig.Resources.Requests.Storage().IsZero() {
-		return ErrNoReaperStorageRequest
+	if r.Spec.Reaper.StorageType == reaperapi.StorageTypeLocal && r.Spec.Reaper.DeploymentMode == reaperapi.DeploymentModePerDc {
+		return ErrNoReaperPerDcWithLocal
 	}
 	return nil
 }

apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go

@@ -543,6 +543,12 @@ func testReaperStorage(t *testing.T) {
 	reaperWithoutStorageSize.Spec.Reaper.StorageConfig.Resources.Requests = corev1.ResourceList{}
 	err = reaperWithoutStorageSize.validateK8ssandraCluster()
 	required.Error(err)
+
+	kc := createClusterObjWithCassandraConfig("kc-with-per-dc-reaper-and-local-storage", "ns")
+	kc.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
+	kc.Spec.Reaper.DeploymentMode = reaperapi.DeploymentModePerDc
+	err = kc.validateK8ssandraCluster()
+	required.Equal(ErrNoReaperPerDcWithLocal, err)
 }
 
 // TestValidateUpdateNumTokens is a unit test for numTokens updates.

apis/reaper/v1alpha1/reaper_types.go

@@ -30,12 +30,13 @@ import (
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
 const (
-	DeploymentModeSingle = "SINGLE"
-	DeploymentModePerDc  = "PER_DC"
-	ReaperLabel          = "k8ssandra.io/reaper"
-	DefaultKeyspace      = "reaper_db"
-	StorageTypeCassandra = "cassandra"
-	StorageTypeLocal     = "local"
+	DeploymentModeSingle       = "SINGLE"
+	DeploymentModePerDc        = "PER_DC"
+	DeploymentModeControlPlane = "CONTROL_PLANE"
+	ReaperLabel                = "k8ssandra.io/reaper"
+	DefaultKeyspace            = "reaper_db"
+	StorageTypeCassandra       = "cassandra"
+	StorageTypeLocal           = "local"
 )
 
 type ReaperTemplate struct {
@@ -235,16 +236,30 @@ type ReaperClusterTemplate struct {
 
 	// +optional
 	// +kubebuilder:default="PER_DC"
-	// +kubebuilder:validation:Enum:=PER_DC;SINGLE
+	// +kubebuilder:validation:Enum:=PER_DC;SINGLE;CONTROL_PLANE
 	DeploymentMode string `json:"deploymentMode,omitempty"`
+
+	// When there is a CONTROL_PLANE Reaper out there, this field allows registering a K8ssandra cluster to it.
+	// Populating this field disables some operator behaviour related to setting Reaper up.
+	// +optional
+	ReaperRef corev1.ObjectReference `json:"reaperRef,omitempty"`
+}
+
+func (rct *ReaperClusterTemplate) HasReaperRef() bool {
+	return rct != nil && rct.ReaperRef.Name != ""
 }
 
-// EnsureDeploymentMode ensures that a deployment mode is SINGLE if we use the local storage type. This is to prevent
-// several instances of Reapers with local storage that would interfere with each other.
+func (rct *ReaperClusterTemplate) IsControlPlane() bool {
+	return rct != nil && rct.DeploymentMode == DeploymentModeControlPlane
+}
+
+// EnsureDeploymentMode ensures that a deployment mode is no PER_DC if we use the local storage type. This is to prevent
+// several instances of Reapers with local storage that would interfere with each other. Deployment mode CONTROL_PLANE
+// is allowed.
 func (t *ReaperClusterTemplate) EnsureDeploymentMode() bool {
 	if t != nil {
 		if t.StorageType == StorageTypeLocal {
-			if t.DeploymentMode != DeploymentModeSingle {
+			if t.DeploymentMode == DeploymentModePerDc {
 				t.DeploymentMode = DeploymentModeSingle
 				return true
 			}
@@ -258,8 +273,8 @@ func (t *ReaperClusterTemplate) EnsureDeploymentMode() bool {
 type CassandraDatacenterRef struct {
 
 	// The datacenter name.
-	// +kubebuilder:validation:Required
-	Name string `json:"name"`
+	// +optional
+	Name string `json:"name,omitempty"`
 
 	// The datacenter namespace. If empty, the datacenter will be assumed to reside in the same namespace as the Reaper
 	// instance.
@@ -274,8 +289,8 @@ type ReaperSpec struct {
 	// DatacenterRef is the reference of a CassandraDatacenter resource that this Reaper instance should manage. It will
 	// also be used as the backend for persisting Reaper's state. Reaper must be able to access the JMX port (7199 by
 	// default) and the CQL port (9042 by default) on this DC.
-	// +kubebuilder:validation:Required
-	DatacenterRef CassandraDatacenterRef `json:"datacenterRef"`
+	// +optional
+	DatacenterRef CassandraDatacenterRef `json:"datacenterRef,omitempty"`
 
 	// DatacenterAvailability indicates to Reaper its deployment in relation to the target datacenter's network.
 	// For single-DC clusters, the default (ALL) is fine. For multi-DC clusters, it is recommended to use EACH,

apis/reaper/v1alpha1/reaper_types_test.go

@@ -41,4 +41,10 @@ func TestEnsureDeploymentMode(t *testing.T) {
 	changed = rct.EnsureDeploymentMode()
 	assert.False(t, changed)
 	assert.Equal(t, DeploymentModePerDc, rct.DeploymentMode)
+
+	cpReaper := rct.DeepCopy()
+	cpReaper.DeploymentMode = DeploymentModeControlPlane
+	changed = cpReaper.EnsureDeploymentMode()
+	assert.False(t, changed)
+	assert.Equal(t, DeploymentModeControlPlane, cpReaper.DeploymentMode)
 }

charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml

@@ -26559,7 +26559,8 @@ spec:
               reaper:
                 description: |-
                   Reaper defines the desired deployment characteristics for Reaper in this K8ssandraCluster.
-                  If this is non-nil, Reaper will be deployed on every Cassandra datacenter in this K8ssandraCluster.
+                  If this is non-nil, Reaper might be deployed on every Cassandra datacenter in this K8ssandraCluster, unless
+                  there is a Control Plane Reaper present. In that case, the K8ssandraCluster will get registered to it.
                 properties:
                   ServiceAccountName:
                     default: default
@@ -27600,6 +27601,7 @@ spec:
                     enum:
                     - PER_DC
                     - SINGLE
+                    - CONTROL_PLANE
                     type: string
                   heapSize:
                     anyOf:
@@ -28439,6 +28441,52 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  reaperRef:
+                    description: |-
+                      When there is a CONTROL_PLANE Reaper out there, this field allows registering a K8ssandra cluster to it.
+                      Populating this field disables some operator behaviour related to setting Reaper up.
+                    properties:
+                      apiVersion:
+                        description: API version of the referent.
+                        type: string
+                      fieldPath:
+                        description: |-
+                          If referring to a piece of an object instead of an entire object, this string
+                          should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                          For example, if the object reference is to a container within a pod, this would take on a value like:
+                          "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                          the event) or if no container name is specified "spec.containers[2]" (container with
+                          index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                          referencing a part of an object.
+                          TODO: this design is not final and this field is subject to change in the future.
+                        type: string
+                      kind:
+                        description: |-
+                          Kind of the referent.
+                          More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                        type: string
+                      name:
+                        description: |-
+                          Name of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                        type: string
+                      namespace:
+                        description: |-
+                          Namespace of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                        type: string
+                      resourceVersion:
+                        description: |-
+                          Specific resourceVersion to which this reference is made, if any.
+                          More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                        type: string
+                      uid:
+                        description: |-
+                          UID of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
                   resources:
                     description: Main Container resources.
                     properties:
@@ -33695,8 +33743,6 @@ spec:
                       The datacenter namespace. If empty, the datacenter will be assumed to reside in the same namespace as the Reaper
                       instance.
                     type: string
-                required:
-                - name
                 type: object
               heapSize:
                 anyOf:
@@ -35282,8 +35328,6 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
-            required:
-            - datacenterRef
             type: object
           status:
             description: ReaperStatus defines the observed state of Reaper

config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml

@@ -26497,7 +26497,8 @@ spec:
               reaper:
                 description: |-
                   Reaper defines the desired deployment characteristics for Reaper in this K8ssandraCluster.
-                  If this is non-nil, Reaper will be deployed on every Cassandra datacenter in this K8ssandraCluster.
+                  If this is non-nil, Reaper might be deployed on every Cassandra datacenter in this K8ssandraCluster, unless
+                  there is a Control Plane Reaper present. In that case, the K8ssandraCluster will get registered to it.
                 properties:
                   ServiceAccountName:
                     default: default
@@ -27538,6 +27539,7 @@ spec:
                     enum:
                     - PER_DC
                     - SINGLE
+                    - CONTROL_PLANE
                     type: string
                   heapSize:
                     anyOf:
@@ -28377,6 +28379,52 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  reaperRef:
+                    description: |-
+                      When there is a CONTROL_PLANE Reaper out there, this field allows registering a K8ssandra cluster to it.
+                      Populating this field disables some operator behaviour related to setting Reaper up.
+                    properties:
+                      apiVersion:
+                        description: API version of the referent.
+                        type: string
+                      fieldPath:
+                        description: |-
+                          If referring to a piece of an object instead of an entire object, this string
+                          should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                          For example, if the object reference is to a container within a pod, this would take on a value like:
+                          "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                          the event) or if no container name is specified "spec.containers[2]" (container with
+                          index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                          referencing a part of an object.
+                          TODO: this design is not final and this field is subject to change in the future.
+                        type: string
+                      kind:
+                        description: |-
+                          Kind of the referent.
+                          More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                        type: string
+                      name:
+                        description: |-
+                          Name of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                        type: string
+                      namespace:
+                        description: |-
+                          Namespace of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                        type: string
+                      resourceVersion:
+                        description: |-
+                          Specific resourceVersion to which this reference is made, if any.
+                          More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                        type: string
+                      uid:
+                        description: |-
+                          UID of the referent.
+                          More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                        type: string
+                    type: object
+                    x-kubernetes-map-type: atomic
                   resources:
                     description: Main Container resources.
                     properties:

config/crd/bases/reaper.k8ssandra.io_reapers.yaml

@@ -1181,8 +1181,6 @@ spec:
                       The datacenter namespace. If empty, the datacenter will be assumed to reside in the same namespace as the Reaper
                       instance.
                     type: string
-                required:
-                - name
                 type: object
               heapSize:
                 anyOf:
@@ -2768,8 +2766,6 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
-            required:
-            - datacenterRef
             type: object
           status:
             description: ReaperStatus defines the observed state of Reaper

controllers/k8ssandra/dcconfigs.go

@@ -71,8 +71,8 @@ func (r *K8ssandraClusterReconciler) createDatacenterConfigs(
 			cassandra.AllowAlterRfDuringRangeMovement(dcConfig)
 		}
 
-		// Inject Reaper settings
-		if kc.Spec.Reaper != nil {
+		// Inject Reaper settings, unless we just reference an existing Reaper
+		if kc.Spec.Reaper != nil && kc.Spec.Reaper.ReaperRef.Name == "" {
 			reaper.AddReaperSettingsToDcConfig(kc.Spec.Reaper.DeepCopy(), dcConfig, kc.Spec.IsAuthEnabled())
 		}