From a9b2d07d6a66f04f0d1492ad05f12c6ef4420b0a Mon Sep 17 00:00:00 2001 From: Radovan Zvoncek Date: Mon, 25 Nov 2024 17:43:13 +0200 Subject: [PATCH] K8OP-295 Expose Medusa's gRPC server port configuration --- CHANGELOG/CHANGELOG-1.21.md | 1 + apis/medusa/v1alpha1/medusa_types.go | 10 +++++++ .../crds/k8ssandra-operator-crds.yaml | 9 +++++++ .../bases/k8ssandra.io_k8ssandraclusters.yaml | 9 +++++++ .../medusa/medusabackupjob_controller.go | 15 +++++++++-- .../medusa/medusabackupjob_controller_test.go | 4 +++ .../medusa/medusarestorejob_controller.go | 8 +++++- .../medusarestorejob_controller_test.go | 3 +++ controllers/medusa/medusatask_controller.go | 22 ++++++++++++--- .../medusa/medusatask_controller_test.go | 3 +++ .../content/en/tasks/backup-restore/_index.md | 3 +++ pkg/cassandra/datacenter.go | 27 +++++++++++++++++++ pkg/medusa/reconcile.go | 23 ++++++++++------ pkg/medusa/reconcile_test.go | 24 ++++++++++++++--- .../fixtures/multi-dc-medusa/k8ssandra.yaml | 4 ++- .../k8ssandra.yaml | 2 ++ 16 files changed, 149 insertions(+), 18 deletions(-) diff --git a/CHANGELOG/CHANGELOG-1.21.md b/CHANGELOG/CHANGELOG-1.21.md index 6d4d84c69..c8671ad8f 100644 --- a/CHANGELOG/CHANGELOG-1.21.md +++ b/CHANGELOG/CHANGELOG-1.21.md @@ -18,3 +18,4 @@ When cutting a new release, update the `unreleased` heading to the tag being gen * [CHANGE] [#1441](https://github.com/k8ssandra/k8ssandra-operator/issues/1441) Use k8ssandra-client instead of k8ssandra-tools for CRD upgrades * [BUGFIX] [#1383](https://github.com/k8ssandra/k8ssandra-operator/issues/1383) Do not create MedusaBackup if MadusaBakupJob did not fully succeed * [ENHANCEMENT] [#1667](https://github.com/k8ssahttps://github.com/k8ssandra/k8ssandra/issues/1667) Add `skipSchemaMigration` option to `K8ssandraCluster.spec.reaper` +* [ENHANCEMENT] [#1455](https://github.com/k8ssandra/k8ssandra-operator/issues/1455) Expose configuration of Medusa's gRPC server port diff --git a/apis/medusa/v1alpha1/medusa_types.go b/apis/medusa/v1alpha1/medusa_types.go index db88f7d8c..d49262785 100644 --- a/apis/medusa/v1alpha1/medusa_types.go +++ b/apis/medusa/v1alpha1/medusa_types.go @@ -117,6 +117,13 @@ type Storage struct { PodStorage *PodStorageSettings `json:"podStorage,omitempty"` } +type Service struct { + // GrpcPort to listen on when running as gRPC service + // Included grpc in the field name to avoid misunderstanding with storage.port + // +optional + GrpcPort int `json:"grpcPort,omitempty"` +} + type PodStorageSettings struct { // Settings for the pod's storage when backups use the local storage provider. @@ -160,6 +167,9 @@ type MedusaClusterTemplate struct { // Provides all storage backend related properties for backups. StorageProperties Storage `json:"storageProperties,omitempty"` + // Provides all service related properties for Medusa. + ServiceProperties Service `json:"serviceProperties,omitempty"` + // Certificates for Medusa if client encryption is enabled in Cassandra. // The secret must be in the same namespace as Cassandra and must contain three keys: "rootca.crt", "client.crt_signed" and "client.key". // See https://docs.datastax.com/en/developer/python-driver/latest/security/ for more information on the required files. diff --git a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml index db9f5e04b..8b05fca4f 100644 --- a/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml +++ b/charts/k8ssandra-operator/crds/k8ssandra-operator-crds.yaml @@ -26443,6 +26443,15 @@ spec: type: string type: object type: object + serviceProperties: + description: Provides all service related properties for Medusa. + properties: + grpcPort: + description: |- + GrpcPort to listen on when running as gRPC service + Included grpc in the field name to avoid misunderstanding with storage.port + type: integer + type: object storageProperties: description: Provides all storage backend related properties for backups. diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml index 4bccdf4bd..5c92239b3 100644 --- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml +++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml @@ -26381,6 +26381,15 @@ spec: type: string type: object type: object + serviceProperties: + description: Provides all service related properties for Medusa. + properties: + grpcPort: + description: |- + GrpcPort to listen on when running as gRPC service + Included grpc in the field name to avoid misunderstanding with storage.port + type: integer + type: object storageProperties: description: Provides all storage backend related properties for backups. diff --git a/controllers/medusa/medusabackupjob_controller.go b/controllers/medusa/medusabackupjob_controller.go index 36607440f..826f7b419 100644 --- a/controllers/medusa/medusabackupjob_controller.go +++ b/controllers/medusa/medusabackupjob_controller.go @@ -19,6 +19,7 @@ package medusa import ( "context" "fmt" + "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "net" "strings" @@ -305,7 +306,12 @@ func (r *MedusaBackupJobReconciler) createMedusaBackup(ctx context.Context, back } func doMedusaBackup(ctx context.Context, name string, backupType shared.BackupType, pod *corev1.Pod, clientFactory medusa.ClientFactory, logger logr.Logger) (string, error) { - addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + medusaPort := shared.BackupSidecarPort + explicitPort, found := cassandra.FindContainerPort(pod, "medusa", "grpc") + if found { + medusaPort = explicitPort + } + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(medusaPort)) logger.Info("connecting to backup sidecar", "Pod", pod.Name, "Address", addr) if medusaClient, err := clientFactory.NewClient(ctx, addr); err != nil { return "", err @@ -322,7 +328,12 @@ func doMedusaBackup(ctx context.Context, name string, backupType shared.BackupTy } func backupStatus(ctx context.Context, name string, pod *corev1.Pod, clientFactory medusa.ClientFactory, logger logr.Logger) (medusa.StatusType, error) { - addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + medusaPort := shared.BackupSidecarPort + explicitPort, found := cassandra.FindContainerPort(pod, "medusa", "grpc") + if found { + medusaPort = explicitPort + } + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(medusaPort)) logger.Info("connecting to backup sidecar", "Pod", pod.Name, "Address", addr) if medusaClient, err := clientFactory.NewClient(ctx, addr); err != nil { logger.Error(err, "Could not make a new medusa client") diff --git a/controllers/medusa/medusabackupjob_controller_test.go b/controllers/medusa/medusabackupjob_controller_test.go index e20ce82f8..1a48eb5b7 100644 --- a/controllers/medusa/medusabackupjob_controller_test.go +++ b/controllers/medusa/medusabackupjob_controller_test.go @@ -83,6 +83,10 @@ func testMedusaBackupDatacenter(t *testing.T, ctx context.Context, f *framework. Name: cassandraUserSecret, }, }, + // adding this did not actually break any assertions + ServiceProperties: api.Service{ + GrpcPort: 1234, + }, CassandraUserSecretRef: corev1.LocalObjectReference{ Name: cassandraUserSecret, }, diff --git a/controllers/medusa/medusarestorejob_controller.go b/controllers/medusa/medusarestorejob_controller.go index 0c85551e1..84901a19e 100644 --- a/controllers/medusa/medusarestorejob_controller.go +++ b/controllers/medusa/medusarestorejob_controller.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/go-logr/logr" "github.com/k8ssandra/k8ssandra-operator/pkg/shared" + "k8s.io/utils/ptr" "net" "time" @@ -273,7 +274,12 @@ func (r *MedusaRestoreJobReconciler) prepareRestore(ctx context.Context, request } for _, pod := range pods { - addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + medusaPort := shared.BackupSidecarPort + explicitPort, found := cassandra.FindContainerPort(ptr.To(pod), "medusa", "grpc") + if found { + medusaPort = explicitPort + } + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(medusaPort)) if medusaClient, err := r.ClientFactory.NewClient(ctx, addr); err != nil { logger.Error(err, "Failed to create Medusa client", "address", addr) } else { diff --git a/controllers/medusa/medusarestorejob_controller_test.go b/controllers/medusa/medusarestorejob_controller_test.go index bd602b2e9..a9d8a9494 100644 --- a/controllers/medusa/medusarestorejob_controller_test.go +++ b/controllers/medusa/medusarestorejob_controller_test.go @@ -67,6 +67,9 @@ func testMedusaRestoreDatacenter(t *testing.T, ctx context.Context, f *framework Name: cassandraUserSecret, }, }, + ServiceProperties: api.Service{ + GrpcPort: 4567, + }, CassandraUserSecretRef: corev1.LocalObjectReference{ Name: cassandraUserSecret, }, diff --git a/controllers/medusa/medusatask_controller.go b/controllers/medusa/medusatask_controller.go index 9648415ff..fb16c544f 100644 --- a/controllers/medusa/medusatask_controller.go +++ b/controllers/medusa/medusatask_controller.go @@ -19,6 +19,7 @@ package medusa import ( "context" "fmt" + "github.com/k8ssandra/k8ssandra-operator/pkg/cassandra" "net" "sync" @@ -413,7 +414,12 @@ func (r *MedusaTaskReconciler) scheduleSyncForPurge(task *medusav1alpha1.MedusaT } func doPurge(ctx context.Context, task *medusav1alpha1.MedusaTask, pod *corev1.Pod, clientFactory medusa.ClientFactory) (*medusa.PurgeBackupsResponse, error) { - addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + medusaPort := shared.BackupSidecarPort + explicitPort, found := cassandra.FindContainerPort(pod, "medusa", "grpc") + if found { + medusaPort = explicitPort + } + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(medusaPort)) if medusaClient, err := clientFactory.NewClient(ctx, addr); err != nil { return nil, err } else { @@ -423,7 +429,12 @@ func doPurge(ctx context.Context, task *medusav1alpha1.MedusaTask, pod *corev1.P } func prepareRestore(ctx context.Context, task *medusav1alpha1.MedusaTask, pod *corev1.Pod, clientFactory medusa.ClientFactory) (*medusa.PurgeBackupsResponse, error) { - addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + medusaPort := shared.BackupSidecarPort + explicitPort, found := cassandra.FindContainerPort(pod, "medusa", "grpc") + if found { + medusaPort = explicitPort + } + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(medusaPort)) if medusaClient, err := clientFactory.NewClient(ctx, addr); err != nil { return nil, err } else { @@ -434,7 +445,12 @@ func prepareRestore(ctx context.Context, task *medusav1alpha1.MedusaTask, pod *c } func GetBackups(ctx context.Context, pod *corev1.Pod, clientFactory medusa.ClientFactory) ([]*medusa.BackupSummary, error) { - addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(shared.BackupSidecarPort)) + medusaPort := shared.BackupSidecarPort + explicitPort, found := cassandra.FindContainerPort(pod, "medusa", "grpc") + if found { + medusaPort = explicitPort + } + addr := net.JoinHostPort(pod.Status.PodIP, fmt.Sprint(medusaPort)) if medusaClient, err := clientFactory.NewClient(ctx, addr); err != nil { return nil, err } else { diff --git a/controllers/medusa/medusatask_controller_test.go b/controllers/medusa/medusatask_controller_test.go index 9ef9beadb..1c817b778 100644 --- a/controllers/medusa/medusatask_controller_test.go +++ b/controllers/medusa/medusatask_controller_test.go @@ -79,6 +79,9 @@ func testMedusaTasks(t *testing.T, ctx context.Context, f *framework.Framework, }, MaxBackupCount: 1, }, + ServiceProperties: api.Service{ + GrpcPort: 7890, + }, CassandraUserSecretRef: corev1.LocalObjectReference{ Name: cassandraUserSecret, }, diff --git a/docs/content/en/tasks/backup-restore/_index.md b/docs/content/en/tasks/backup-restore/_index.md index b7268ef9a..d8fb63faa 100644 --- a/docs/content/en/tasks/backup-restore/_index.md +++ b/docs/content/en/tasks/backup-restore/_index.md @@ -86,6 +86,9 @@ spec: # accessModes: # - ReadWriteOnce # size: 100Mi + serviceProperties: + # which port will Medusa's gRPC server listen on + grpcPort: 50051 ``` The definition above requires a secret named `medusa-bucket-key` to be present in the target namespace before the `K8ssandraCluster` object gets created. Use the following format for this secret: diff --git a/pkg/cassandra/datacenter.go b/pkg/cassandra/datacenter.go index 78d24dd6b..11690df9a 100644 --- a/pkg/cassandra/datacenter.go +++ b/pkg/cassandra/datacenter.go @@ -485,6 +485,33 @@ func FindInitContainer(dcPodTemplateSpec *corev1.PodTemplateSpec, containerName return -1, false } +func FindPort(container *corev1.Container, portName string) (int32, bool) { + if container.Ports != nil { + for _, port := range container.Ports { + if port.Name == portName { + return port.ContainerPort, true + } + } + } + return -1, false +} +func FindContainerPort(pod *corev1.Pod, containerName, podName string) (int, bool) { + if pod.Spec.Containers != nil { + for _, container := range pod.Spec.Containers { + if container.Name == containerName { + if container.Ports != nil { + for _, port := range container.Ports { + if port.Name == podName { + return int(port.ContainerPort), true + } + } + } + } + } + } + return -1, false +} + func FindVolume(dcPodTemplateSpec *corev1.PodTemplateSpec, volumeName string) (int, bool) { if dcPodTemplateSpec != nil { for i, volume := range dcPodTemplateSpec.Spec.Volumes { diff --git a/pkg/medusa/reconcile.go b/pkg/medusa/reconcile.go index 929c1459b..78953deb4 100644 --- a/pkg/medusa/reconcile.go +++ b/pkg/medusa/reconcile.go @@ -25,7 +25,7 @@ import ( const ( DefaultMedusaImageRepository = "k8ssandra" DefaultMedusaImageName = "medusa" - DefaultMedusaVersion = "0.22.3" + DefaultMedusaVersion = "a1f7647-tmp" DefaultMedusaPort = 50051 DefaultProbeInitialDelay = 10 DefaultProbeTimeout = 1 @@ -114,6 +114,9 @@ func CreateMedusaIni(kc *k8ss.K8ssandraCluster, dcConfig *cassandra.DatacenterCo [grpc] enabled = 1 + {{- if .Spec.Medusa.ServiceProperties.GrpcPort }} + port = {{ .Spec.Medusa.ServiceProperties.GrpcPort }} + {{- end }} [logging] level = DEBUG @@ -226,19 +229,23 @@ func CreateMedusaMainContainer(dcConfig *cassandra.DatacenterConfig, medusaSpec setImage(medusaSpec.ContainerImage, medusaContainer) medusaContainer.SecurityContext = medusaSpec.SecurityContext medusaContainer.Env = medusaEnvVars(medusaSpec, k8cName, useExternalSecrets, "GRPC") + var grpcPort = DefaultMedusaPort + if medusaSpec.ServiceProperties.GrpcPort != 0 { + grpcPort = medusaSpec.ServiceProperties.GrpcPort + } medusaContainer.Ports = []corev1.ContainerPort{ { Name: "grpc", - ContainerPort: DefaultMedusaPort, + ContainerPort: int32(grpcPort), Protocol: "TCP", }, } - readinessProbe, err := generateMedusaProbe(medusaSpec.ReadinessProbe) + readinessProbe, err := generateMedusaProbe(medusaSpec.ReadinessProbe, grpcPort) if err != nil { return nil, err } - livenessProbe, err := generateMedusaProbe(medusaSpec.LivenessProbe) + livenessProbe, err := generateMedusaProbe(medusaSpec.LivenessProbe, grpcPort) if err != nil { return nil, err } @@ -545,9 +552,9 @@ func PurgeCronJob(dcConfig *cassandra.DatacenterConfig, clusterName, namespace s return purgeCronJob, nil } -func generateMedusaProbe(configuredProbe *corev1.Probe) (*corev1.Probe, error) { +func generateMedusaProbe(configuredProbe *corev1.Probe, grpcPort int) (*corev1.Probe, error) { // Goalesce the custom probe with the default probe, - defaultProbe := defaultMedusaProbe() + defaultProbe := defaultMedusaProbe(grpcPort) if configuredProbe == nil { return defaultProbe, nil } @@ -561,12 +568,12 @@ func generateMedusaProbe(configuredProbe *corev1.Probe) (*corev1.Probe, error) { return &mergedProbe, nil } -func defaultMedusaProbe() *corev1.Probe { +func defaultMedusaProbe(grpcPort int) *corev1.Probe { // Goalesce the custom probe with the default probe, probe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ - Command: []string{"/bin/grpc_health_probe", fmt.Sprintf("--addr=:%d", DefaultMedusaPort)}, + Command: []string{"/bin/grpc_health_probe", fmt.Sprintf("--addr=:%d", grpcPort)}, }, }, InitialDelaySeconds: DefaultProbeInitialDelay, diff --git a/pkg/medusa/reconcile_test.go b/pkg/medusa/reconcile_test.go index 9d75d31cd..7a8472096 100644 --- a/pkg/medusa/reconcile_test.go +++ b/pkg/medusa/reconcile_test.go @@ -66,6 +66,9 @@ func testMedusaIniFull(t *testing.T) { Secure: false, BackupGracePeriodInDays: 7, }, + ServiceProperties: medusaapi.Service{ + GrpcPort: 55055, + }, CassandraUserSecretRef: corev1.LocalObjectReference{ Name: "test-superuser", }, @@ -90,6 +93,7 @@ func testMedusaIniFull(t *testing.T) { assert.Contains(t, medusaIni, "port = 9001") assert.Contains(t, medusaIni, "secure = False") assert.Contains(t, medusaIni, "backup_grace_period_in_days = 7") + assert.Contains(t, medusaIni, "port = 55055") } func testMedusaIniNoPrefix(t *testing.T) { @@ -512,6 +516,10 @@ func TestInitContainerDefaultResources(t *testing.T) { logger := logr.New(logr.Discard().GetSink()) medusaContainer, err := CreateMedusaMainContainer(&dcConfig, medusaSpec, false, "test", logger) + medusaPort, found := cassandra.FindPort(medusaContainer, "grpc") + assert.True(t, found, "Couldn't find medusa grpc port") + assert.Equal(t, int32(50051), medusaPort, "expected medusa grpc port to NOT be set") + assert.NoError(t, err) UpdateMedusaInitContainer(&dcConfig, medusaSpec, false, "test", logger) UpdateMedusaMainContainer(&dcConfig, medusaContainer) @@ -542,6 +550,9 @@ func TestInitContainerCustomResources(t *testing.T) { }, BucketName: "bucket", }, + ServiceProperties: medusaapi.Service{ + GrpcPort: 55055, + }, CassandraUserSecretRef: corev1.LocalObjectReference{ Name: "test-superuser", }, @@ -573,6 +584,11 @@ func TestInitContainerCustomResources(t *testing.T) { medusaContainer, err := CreateMedusaMainContainer(&dcConfig, medusaSpec, false, "test", logger) assert.NoError(t, err) + + medusaPort, found := cassandra.FindPort(medusaContainer, "grpc") + assert.True(t, found, "Couldn't find medusa grpc port") + assert.Equal(t, int32(55055), medusaPort, "expected medusa grpc port to be set") + UpdateMedusaInitContainer(&dcConfig, medusaSpec, false, "test", logger) UpdateMedusaMainContainer(&dcConfig, medusaContainer) @@ -641,21 +657,23 @@ func TestGenerateMedusaProbe(t *testing.T) { FailureThreshold: 500, } - customProbe, err := generateMedusaProbe(customProbeSettings) + customProbe, err := generateMedusaProbe(customProbeSettings, 55055) assert.NoError(t, err) assert.Equal(t, int32(100), customProbe.InitialDelaySeconds) assert.Equal(t, int32(200), customProbe.TimeoutSeconds) assert.Equal(t, int32(300), customProbe.PeriodSeconds) assert.Equal(t, int32(400), customProbe.SuccessThreshold) assert.Equal(t, int32(500), customProbe.FailureThreshold) + assert.Contains(t, customProbe.Exec.Command[1], "55055") - defaultProbe, err := generateMedusaProbe(nil) + defaultProbe, err := generateMedusaProbe(nil, 55155) assert.NoError(t, err) assert.Equal(t, int32(DefaultProbeInitialDelay), defaultProbe.InitialDelaySeconds) assert.Equal(t, int32(DefaultProbeTimeout), defaultProbe.TimeoutSeconds) assert.Equal(t, int32(DefaultProbePeriod), defaultProbe.PeriodSeconds) assert.Equal(t, int32(DefaultProbeSuccessThreshold), defaultProbe.SuccessThreshold) assert.Equal(t, int32(DefaultProbeFailureThreshold), defaultProbe.FailureThreshold) + assert.Contains(t, defaultProbe.Exec.Command[1], "55155") // Test that changing the probe handler is rejected rejectedProbe := &corev1.Probe{ @@ -670,7 +688,7 @@ func TestGenerateMedusaProbe(t *testing.T) { }, }, } - probe, err := generateMedusaProbe(rejectedProbe) + probe, err := generateMedusaProbe(rejectedProbe, 55055) assert.Error(t, err) assert.Nil(t, probe) } diff --git a/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml b/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml index 8768ea427..9774213db 100644 --- a/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml +++ b/test/testdata/fixtures/multi-dc-medusa/k8ssandra.yaml @@ -41,4 +41,6 @@ spec: name: medusa-bucket-key host: test-hl.minio.svc.cluster.local port: 9000 - secure: false + secure: false + serviceProperties: + grpcPort: 55055 diff --git a/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml b/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml index 2df832a08..d2be1dd4e 100644 --- a/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml +++ b/test/testdata/fixtures/single-dc-multi-cluster-medusa/k8ssandra.yaml @@ -81,3 +81,5 @@ spec: host: minio-service.minio.svc.cluster.local port: 9000 secure: false + serviceProperties: + grpcPort: 55055