Skip to content

Commit

Permalink
Make Reaper's storage configurable + adjust its deployment accordingly
Browse files Browse the repository at this point in the history
  • Loading branch information
rzvoncek committed Aug 27, 2024
1 parent d60620a commit 8d4039f
Show file tree
Hide file tree
Showing 24 changed files with 1,733 additions and 154 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG/CHANGELOG-1.18.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,5 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
* [ENHANCEMENT] [#1274](https://github.com/k8ssandra/k8ssandra-operator/issues/1274) On upgrade, do not modify the CassandraDatacenter object unless instructed with an annotation `k8ssandra.io/autoupdate-spec` with value `once` or `always`
* [BUGFIX] [#1222](https://github.com/k8ssandra/k8ssandra-operator/issues/1222) Consider DC-level config when validating numToken updates in webhook
* [BUGFIX] [#1366](https://github.com/k8ssandra/k8ssandra-operator/issues/1366) Reaper deployment can't be created on OpenShift due to missing RBAC rule
* [CHANGE] Update cassandra-medusa to 0.22.0
* [FEATURE] [#1275](https://github.com/k8ssandra/k8ssandra-operator/issues/1275) Allow configuring Reaper to use a memory storage backend
46 changes: 39 additions & 7 deletions apis/k8ssandra/v1alpha1/k8ssandracluster_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package v1alpha1
import (
"fmt"
"github.com/Masterminds/semver/v3"
reaperapi "github.com/k8ssandra/k8ssandra-operator/apis/reaper/v1alpha1"
"strings"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -34,13 +35,17 @@ import (
)

var (
clientCache *clientcache.ClientCache
ErrNumTokens = fmt.Errorf("num_tokens value can't be changed")
ErrReaperKeyspace = fmt.Errorf("reaper keyspace can not be changed")
ErrNoStorageConfig = fmt.Errorf("storageConfig must be defined at cluster level or dc level")
ErrNoResourcesSet = fmt.Errorf("softPodAntiAffinity requires Resources to be set")
ErrClusterName = fmt.Errorf("cluster name can not be changed")
ErrNoStoragePrefix = fmt.Errorf("medusa storage prefix must be set when a medusaConfigurationRef is used")
clientCache *clientcache.ClientCache
ErrNumTokens = fmt.Errorf("num_tokens value can't be changed")
ErrReaperKeyspace = fmt.Errorf("reaper keyspace can not be changed")
ErrNoStorageConfig = fmt.Errorf("storageConfig must be defined at cluster level or dc level")
ErrNoResourcesSet = fmt.Errorf("softPodAntiAffinity requires Resources to be set")
ErrClusterName = fmt.Errorf("cluster name can not be changed")
ErrNoStoragePrefix = fmt.Errorf("medusa storage prefix must be set when a medusaConfigurationRef is used")
ErrNoReaperStorageConfig = fmt.Errorf("reaper StorageConfig not set")
ErrNoReaperAccessMode = fmt.Errorf("reaper StorageConfig.AccessModes not set")
ErrNoReaperResourceRequests = fmt.Errorf("reaper StorageConfig.Resources.Requests not set")
ErrNoReaperStorageRequest = fmt.Errorf("reaper StorageConfig.Resources.Requests.Storage not set")
)

// log is for logging in this package.
Expand Down Expand Up @@ -113,6 +118,10 @@ func (r *K8ssandraCluster) validateK8ssandraCluster() error {
return err
}

if err := r.validateReaper(); err != nil {
return err
}

if err := r.validateStatefulsetNameSize(); err != nil {
return err
}
Expand Down Expand Up @@ -280,3 +289,26 @@ func (r *K8ssandraCluster) ValidateMedusa() error {

return nil
}

func (r *K8ssandraCluster) validateReaper() error {
if r.Spec.Reaper == nil {
return nil
}
if r.Spec.Reaper.StorageType != reaperapi.StorageTypeLocal {
return nil
}
if r.Spec.Reaper.StorageConfig == nil {
return ErrNoReaperStorageConfig
}
// not checking StorageClassName because Kubernetes will use a default one if it's not set
if r.Spec.Reaper.StorageConfig.AccessModes == nil {
return ErrNoReaperAccessMode
}
if r.Spec.Reaper.StorageConfig.Resources.Requests == nil {
return ErrNoReaperResourceRequests
}
if r.Spec.Reaper.StorageConfig.Resources.Requests.Storage().IsZero() {
return ErrNoReaperStorageRequest
}
return nil
}
49 changes: 49 additions & 0 deletions apis/k8ssandra/v1alpha1/k8ssandracluster_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"crypto/tls"
"fmt"
"k8s.io/apimachinery/pkg/api/resource"
"net"
"path/filepath"
"testing"
Expand Down Expand Up @@ -53,6 +54,23 @@ var testEnv *envtest.Environment
var ctx context.Context
var cancel context.CancelFunc

var minimalInMemoryReaperStorageConfig = &corev1.PersistentVolumeClaimSpec{
StorageClassName: func() *string { s := "test"; return &s }(),
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.VolumeResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
},
}

var minimalInMemoryReaperConfig = &reaperapi.ReaperClusterTemplate{
ReaperTemplate: reaperapi.ReaperTemplate{
StorageType: reaperapi.StorageTypeLocal,
StorageConfig: minimalInMemoryReaperStorageConfig,
},
}

func TestWebhook(t *testing.T) {
required := require.New(t)
ctx, cancel = context.WithCancel(context.TODO())
Expand Down Expand Up @@ -160,6 +178,7 @@ func TestWebhook(t *testing.T) {
t.Run("InvalidDcName", testInvalidDcName)
t.Run("MedusaConfigNonLocalNamespace", testMedusaNonLocalNamespace)
t.Run("AutomatedUpdateAnnotation", testAutomatedUpdateAnnotation)
t.Run("ReaperStorage", testReaperStorage)
}

func testContextValidation(t *testing.T) {
Expand Down Expand Up @@ -496,6 +515,36 @@ func testMedusaNonLocalNamespace(t *testing.T) {
required.Contains(err.Error(), "Medusa config must be namespace local")
}

func testReaperStorage(t *testing.T) {
required := require.New(t)

reaperWithNoStorageConfig := createMinimalClusterObj("reaper-no-storage-config", "ns")
reaperWithNoStorageConfig.Spec.Reaper = &reaperapi.ReaperClusterTemplate{
ReaperTemplate: reaperapi.ReaperTemplate{
StorageType: reaperapi.StorageTypeLocal,
},
}
err := reaperWithNoStorageConfig.validateK8ssandraCluster()
required.Error(err)

reaperWithDefaultConfig := createClusterObjWithCassandraConfig("reaper-default-storage-config", "ns")
reaperWithDefaultConfig.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
err = reaperWithDefaultConfig.validateK8ssandraCluster()
required.NoError(err)

reaperWithoutAccessMode := createClusterObjWithCassandraConfig("reaper-no-access-mode", "ns")
reaperWithoutAccessMode.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
reaperWithoutAccessMode.Spec.Reaper.StorageConfig.AccessModes = nil
err = reaperWithoutAccessMode.validateK8ssandraCluster()
required.Error(err)

reaperWithoutStorageSize := createClusterObjWithCassandraConfig("reaper-no-storage-size", "ns")
reaperWithoutStorageSize.Spec.Reaper = minimalInMemoryReaperConfig.DeepCopy()
reaperWithoutStorageSize.Spec.Reaper.StorageConfig.Resources.Requests = corev1.ResourceList{}
err = reaperWithoutStorageSize.validateK8ssandraCluster()
required.Error(err)
}

// TestValidateUpdateNumTokens is a unit test for numTokens updates.
func TestValidateUpdateNumTokens(t *testing.T) {
type config struct {
Expand Down
35 changes: 33 additions & 2 deletions apis/reaper/v1alpha1/reaper_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,29 @@ import (
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.

const (
ReaperLabel = "k8ssandra.io/reaper"
DefaultKeyspace = "reaper_db"
DeploymentModeSingle = "SINGLE"
DeploymentModePerDc = "PER_DC"
ReaperLabel = "k8ssandra.io/reaper"
DefaultKeyspace = "reaper_db"
StorageTypeCassandra = "cassandra"
StorageTypeLocal = "local"
)

type ReaperTemplate struct {

// The storage backend to store Reaper's data. Defaults to "cassandra" which causes Reaper to be stateless and store
// its state to a Cassandra cluster it repairs (implying there must be one Reaper for each Cassandra cluster).
// The "local" option makes Reaper to store its state locally, allowing a single Reaper to repair several clusters.
// +kubebuilder:validation:Enum=cassandra;local
// +kubebuilder:default="cassandra"
// +optional
StorageType string `json:"storageType,omitempty"`

// If StorageType is "local", Reaper will need a Persistent Volume to persist its data. This field allows
// configuring that Persistent Volume.
// +optional
StorageConfig *corev1.PersistentVolumeClaimSpec `json:"storageConfig,omitempty"`

// The keyspace to use to store Reaper's state. Will default to "reaper_db" if unspecified. Will be created if it
// does not exist, and if this Reaper resource is managed by K8ssandra.
// +kubebuilder:default="reaper_db"
Expand Down Expand Up @@ -222,6 +239,20 @@ type ReaperClusterTemplate struct {
DeploymentMode string `json:"deploymentMode,omitempty"`
}

// EnsureDeploymentMode ensures that a deployment mode is SINGLE if we use the local storage type. This is to prevent
// several instances of Reapers with local storage that would interfere with each other.
func (t *ReaperClusterTemplate) EnsureDeploymentMode() bool {
if t != nil {
if t.StorageType == StorageTypeLocal {
if t.DeploymentMode != DeploymentModeSingle {
t.DeploymentMode = DeploymentModeSingle
return true
}
}
}
return false
}

// CassandraDatacenterRef references the target Cassandra DC that Reaper should manage.
// TODO this object could be used by Stargate too; which currently cannot locate DCs outside of its own namespace.
type CassandraDatacenterRef struct {
Expand Down
44 changes: 44 additions & 0 deletions apis/reaper/v1alpha1/reaper_types_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
Copyright 2021.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
"github.com/stretchr/testify/assert"
"testing"
)

func TestEnsureDeploymentMode(t *testing.T) {
rct := &ReaperClusterTemplate{
ReaperTemplate: ReaperTemplate{
StorageType: StorageTypeLocal,
},
DeploymentMode: DeploymentModePerDc,
}
changed := rct.EnsureDeploymentMode()
assert.True(t, changed)
assert.Equal(t, DeploymentModeSingle, rct.DeploymentMode)

rct = &ReaperClusterTemplate{
ReaperTemplate: ReaperTemplate{
StorageType: StorageTypeCassandra,
},
DeploymentMode: DeploymentModePerDc,
}
changed = rct.EnsureDeploymentMode()
assert.False(t, changed)
assert.Equal(t, DeploymentModePerDc, rct.DeploymentMode)
}
5 changes: 5 additions & 0 deletions apis/reaper/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 8d4039f

Please sign in to comment.