From aacbcc5f12c252f4ea9a93a2f84e520c78de43a3 Mon Sep 17 00:00:00 2001 From: Veronika Fisarova Date: Thu, 18 Jan 2024 14:14:53 +0100 Subject: [PATCH] [tlse] tls for ManilaAPI pod configuration Public/Internal service cert secrets and the CA bundle secret can be passed to configure httpd virtual hosts for tls termination. The certs get direct mounted to the appropriate place in etc/pki/tls/certs/%s.crt|key and a CA bundle to /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem . Job deployments for bootstrap/cron get the CA bundle added if configured. Depends-On: openstack-k8s-operators/lib-common#428 Signed-off-by: Veronika Fisarova --- .../manila.openstack.org_manilaapis.yaml | 18 ++ api/bases/manila.openstack.org_manilas.yaml | 18 ++ ...manila.openstack.org_manilaschedulers.yaml | 5 + .../manila.openstack.org_manilashares.yaml | 5 + api/v1beta1/manilaapi_types.go | 6 + api/v1beta1/manilascheduler_types.go | 6 + api/v1beta1/manilashare_types.go | 6 + api/v1beta1/zz_generated.deepcopy.go | 3 + .../manila.openstack.org_manilaapis.yaml | 18 ++ .../bases/manila.openstack.org_manilas.yaml | 18 ++ ...manila.openstack.org_manilaschedulers.yaml | 5 + .../manila.openstack.org_manilashares.yaml | 5 + config/samples/manila_v1beta1_manila_tls.yaml | 25 +++ config/samples/manila_v1beta1_manilaapi.yaml | 10 + controllers/manila_controller.go | 39 ++++ controllers/manilaapi_controller.go | 187 ++++++++++++++++-- controllers/manilascheduler_controller.go | 100 ++++++++++ controllers/manilashare_controller.go | 100 ++++++++++ pkg/manila/cronjob.go | 7 + pkg/manila/dbsync.go | 10 + pkg/manilaapi/statefulset.go | 54 ++++- pkg/manilascheduler/statefulset.go | 17 +- pkg/manilashare/statefulset.go | 18 +- templates/manila/config/10-manila_wsgi.conf | 20 +- .../manila/config/manila-api-config.json | 22 +++ templates/manila/config/ssl.conf | 21 ++ test/functional/base_test.go | 30 +++ test/functional/manila_controller_test.go | 58 ++++++ test/functional/manila_test_data.go | 23 +++ test/functional/manilaapi_controller_test.go | 4 +- 30 files changed, 818 insertions(+), 40 deletions(-) create mode 100644 config/samples/manila_v1beta1_manila_tls.yaml create mode 100644 templates/manila/config/ssl.conf diff --git a/api/bases/manila.openstack.org_manilaapis.yaml b/api/bases/manila.openstack.org_manilaapis.yaml index facebc88..ef7c61c0 100644 --- a/api/bases/manila.openstack.org_manilaapis.yaml +++ b/api/bases/manila.openstack.org_manilaapis.yaml @@ -931,6 +931,24 @@ spec: serviceUser: default: manila type: string + tls: + properties: + api: + properties: + internal: + properties: + secretName: + type: string + type: object + public: + properties: + secretName: + type: string + type: object + type: object + caBundleSecretName: + type: string + type: object transportURLSecret: type: string required: diff --git a/api/bases/manila.openstack.org_manilas.yaml b/api/bases/manila.openstack.org_manilas.yaml index 230287f0..677b4ba1 100644 --- a/api/bases/manila.openstack.org_manilas.yaml +++ b/api/bases/manila.openstack.org_manilas.yaml @@ -931,6 +931,24 @@ spec: x-kubernetes-int-or-string: true type: object type: object + tls: + properties: + api: + properties: + internal: + properties: + secretName: + type: string + type: object + public: + properties: + secretName: + type: string + type: object + type: object + caBundleSecretName: + type: string + type: object required: - containerImage type: object diff --git a/api/bases/manila.openstack.org_manilaschedulers.yaml b/api/bases/manila.openstack.org_manilaschedulers.yaml index dacaf67d..5c279442 100644 --- a/api/bases/manila.openstack.org_manilaschedulers.yaml +++ b/api/bases/manila.openstack.org_manilaschedulers.yaml @@ -880,6 +880,11 @@ spec: serviceUser: default: manila type: string + tls: + properties: + caBundleSecretName: + type: string + type: object transportURLSecret: type: string required: diff --git a/api/bases/manila.openstack.org_manilashares.yaml b/api/bases/manila.openstack.org_manilashares.yaml index 697247db..51b6f953 100644 --- a/api/bases/manila.openstack.org_manilashares.yaml +++ b/api/bases/manila.openstack.org_manilashares.yaml @@ -880,6 +880,11 @@ spec: serviceUser: default: manila type: string + tls: + properties: + caBundleSecretName: + type: string + type: object transportURLSecret: type: string required: diff --git a/api/v1beta1/manilaapi_types.go b/api/v1beta1/manilaapi_types.go index efb82126..b766fa78 100644 --- a/api/v1beta1/manilaapi_types.go +++ b/api/v1beta1/manilaapi_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,6 +39,11 @@ type ManilaAPITemplate struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` } // APIOverrideSpec to override the generated manifest of several child resources. diff --git a/api/v1beta1/manilascheduler_types.go b/api/v1beta1/manilascheduler_types.go index 6e61a9d1..91ed3c9d 100644 --- a/api/v1beta1/manilascheduler_types.go +++ b/api/v1beta1/manilascheduler_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -61,6 +62,11 @@ type ManilaSchedulerSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide the default SA name ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // ManilaSchedulerStatus defines the observed state of ManilaScheduler diff --git a/api/v1beta1/manilashare_types.go b/api/v1beta1/manilashare_types.go index b46fedb6..a5478566 100644 --- a/api/v1beta1/manilashare_types.go +++ b/api/v1beta1/manilashare_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -61,6 +62,11 @@ type ManilaShareSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide the default SA name ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` } // ManilaShareStatus defines the observed state of ManilaShare diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 1ea2a8a8..55d97ce5 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -229,6 +229,7 @@ func (in *ManilaAPITemplate) DeepCopyInto(out *ManilaAPITemplate) { **out = **in } in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManilaAPITemplate. @@ -396,6 +397,7 @@ func (in *ManilaSchedulerSpec) DeepCopyInto(out *ManilaSchedulerSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManilaSchedulerSpec. @@ -593,6 +595,7 @@ func (in *ManilaShareSpec) DeepCopyInto(out *ManilaShareSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManilaShareSpec. diff --git a/config/crd/bases/manila.openstack.org_manilaapis.yaml b/config/crd/bases/manila.openstack.org_manilaapis.yaml index facebc88..ef7c61c0 100644 --- a/config/crd/bases/manila.openstack.org_manilaapis.yaml +++ b/config/crd/bases/manila.openstack.org_manilaapis.yaml @@ -931,6 +931,24 @@ spec: serviceUser: default: manila type: string + tls: + properties: + api: + properties: + internal: + properties: + secretName: + type: string + type: object + public: + properties: + secretName: + type: string + type: object + type: object + caBundleSecretName: + type: string + type: object transportURLSecret: type: string required: diff --git a/config/crd/bases/manila.openstack.org_manilas.yaml b/config/crd/bases/manila.openstack.org_manilas.yaml index 230287f0..677b4ba1 100644 --- a/config/crd/bases/manila.openstack.org_manilas.yaml +++ b/config/crd/bases/manila.openstack.org_manilas.yaml @@ -931,6 +931,24 @@ spec: x-kubernetes-int-or-string: true type: object type: object + tls: + properties: + api: + properties: + internal: + properties: + secretName: + type: string + type: object + public: + properties: + secretName: + type: string + type: object + type: object + caBundleSecretName: + type: string + type: object required: - containerImage type: object diff --git a/config/crd/bases/manila.openstack.org_manilaschedulers.yaml b/config/crd/bases/manila.openstack.org_manilaschedulers.yaml index dacaf67d..5c279442 100644 --- a/config/crd/bases/manila.openstack.org_manilaschedulers.yaml +++ b/config/crd/bases/manila.openstack.org_manilaschedulers.yaml @@ -880,6 +880,11 @@ spec: serviceUser: default: manila type: string + tls: + properties: + caBundleSecretName: + type: string + type: object transportURLSecret: type: string required: diff --git a/config/crd/bases/manila.openstack.org_manilashares.yaml b/config/crd/bases/manila.openstack.org_manilashares.yaml index 697247db..51b6f953 100644 --- a/config/crd/bases/manila.openstack.org_manilashares.yaml +++ b/config/crd/bases/manila.openstack.org_manilashares.yaml @@ -880,6 +880,11 @@ spec: serviceUser: default: manila type: string + tls: + properties: + caBundleSecretName: + type: string + type: object transportURLSecret: type: string required: diff --git a/config/samples/manila_v1beta1_manila_tls.yaml b/config/samples/manila_v1beta1_manila_tls.yaml new file mode 100644 index 00000000..e12d554e --- /dev/null +++ b/config/samples/manila_v1beta1_manila_tls.yaml @@ -0,0 +1,25 @@ +apiVersion: manila.openstack.org/v1beta1 +kind: Manila +metadata: + name: manila + namespace: openstack +spec: + serviceUser: manila + customServiceConfig: | + [DEFAULT] + debug = true + databaseInstance: openstack + secret: osp-secret + databaseUser: manila + rabbitMqClusterName: rabbitmq + manilaAPI: + tls: + api: + internal: + secretName: cert-manila-internal-svc + public: + secretName: cert-manila-public-svc + caBundleSecretName: combined-ca-bundle + manilaScheduler: {} + manilaShares: + share1: {} diff --git a/config/samples/manila_v1beta1_manilaapi.yaml b/config/samples/manila_v1beta1_manilaapi.yaml index 6f3be559..5ec01628 100644 --- a/config/samples/manila_v1beta1_manilaapi.yaml +++ b/config/samples/manila_v1beta1_manilaapi.yaml @@ -4,3 +4,13 @@ metadata: name: manilaapi-sample spec: # TODO(user): Add fields here + #tls: + # api: + # # secret holding tls.crt and tls.key for the APIs internal k8s service + # internal: + # secretName: cert-internal-svc + # # secret holding tls.crt and tls.key for the APIs public k8s service + # public: + # secretName: cert-public-svc + # # secret holding the tls-ca-bundle.pem to be used as a deploymend env CA bundle + # caBundleSecretName: combined-ca-bundle diff --git a/controllers/manila_controller.go b/controllers/manila_controller.go index 6055a36a..34d54ab4 100644 --- a/controllers/manila_controller.go +++ b/controllers/manila_controller.go @@ -37,6 +37,7 @@ import ( nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" manilav1beta1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" "github.com/openstack-k8s-operators/manila-operator/pkg/manila" @@ -209,6 +210,27 @@ func (r *ManilaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res return r.reconcileNormal(ctx, instance, helper) } +// fields to index to reconcile when change +const ( + passwordSecretField = ".spec.secret" + caBundleSecretNameField = ".spec.tls.caBundleSecretName" + tlsAPIInternalField = ".spec.tls.api.internal.secretName" + tlsAPIPublicField = ".spec.tls.api.public.secretName" +) + +var ( + commonWatchFields = []string{ + passwordSecretField, + caBundleSecretNameField, + } + manilaAPIWatchFields = []string{ + passwordSecretField, + caBundleSecretNameField, + tlsAPIInternalField, + tlsAPIPublicField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *ManilaReconciler) SetupWithManager(mgr ctrl.Manager) error { // transportURLSecretFn - Watch for changes made to the secret associated with the RabbitMQ @@ -872,6 +894,21 @@ func (r *ManilaReconciler) generateServiceConfig( "MemcachedServersWithInet": strings.Join(memcached.Status.ServerListWithInet, ","), } + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("manila-%s.%s.svc", endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.ManilaAPI.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + configTemplates := []util.Template{ // ScriptsConfigMap { @@ -971,6 +1008,7 @@ func (r *ManilaReconciler) schedulerDeploymentCreateOrUpdate(ctx context.Context DatabaseHostname: instance.Status.DatabaseHostname, TransportURLSecret: instance.Status.TransportURLSecret, ServiceAccount: instance.RbacResourceName(), + TLS: instance.Spec.ManilaAPI.TLS.Ca, } op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { @@ -1007,6 +1045,7 @@ func (r *ManilaReconciler) shareDeploymentCreateOrUpdate(ctx context.Context, in DatabaseHostname: instance.Status.DatabaseHostname, TransportURLSecret: instance.Status.TransportURLSecret, ServiceAccount: instance.RbacResourceName(), + TLS: instance.Spec.ManilaAPI.TLS.Ca, } op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { diff --git a/controllers/manilaapi_controller.go b/controllers/manilaapi_controller.go index 8f637060..a055528f 100644 --- a/controllers/manilaapi_controller.go +++ b/controllers/manilaapi_controller.go @@ -21,11 +21,16 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/go-logr/logr" @@ -40,6 +45,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" manilav1beta1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" "github.com/openstack-k8s-operators/manila-operator/pkg/manila" @@ -178,6 +184,7 @@ func (r *ManilaAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, ""), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -256,6 +263,54 @@ func (r *ManilaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaAPI{}, passwordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaAPI) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaAPI{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaAPI{}, tlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaAPI{}, tlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&manilav1beta1.ManilaAPI{}). Owns(&keystonev1.KeystoneService{}). @@ -266,9 +321,47 @@ func (r *ManilaAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } +func (r *ManilaAPIReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("ManilaAPI") + + for _, field := range manilaAPIWatchFields { + crList := &manilav1beta1.ManilaAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + func (r *ManilaAPIReconciler) reconcileDelete(ctx context.Context, instance *manilav1beta1.ManilaAPI, helper *helper.Helper) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) @@ -422,7 +515,12 @@ func (r *ManilaAPIReconciler) reconcileInit( } // create service - end - // TODO: TLS, pass in https as protocol + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + apiEndpointsV2[string(endpointType)], err = svc.GetAPIEndpoint( svcOverride.EndpointURL, data.Protocol, data.Path) if err != nil { @@ -540,9 +638,6 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man fmt.Sprintf("%s-config-data", parentManilaName), //ConfigSecret } - // - // Create Secrets required as input for the Service and calculate an overall hash of hashes - // for _, parentSecret := range parentSecrets { ctrlResult, err = r.getSecret(ctx, helper, instance, parentSecret, &configVars) if err != nil { @@ -551,15 +646,66 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man } instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + + // Validate API service certs secrets + certsHash, ctrlResult, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, helper, instance.Namespace) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + configVars[tls.TLSHashName] = env.SetValue(certsHash) + } + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create secrets required as input for the Service and calculate an overall hash of hashes + // serviceLabels := map[string]string{ common.AppSelector: manila.ServiceName, common.ComponentSelector: manilaapi.Component, } - - // - // create Secrets for manila-api service + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. // - err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels) + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -568,13 +714,16 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man condition.ServiceConfigReadyErrorMessage, err.Error())) return ctrl.Result{}, err + } else if hashChanged { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil } // - // create hash over all the different input resources to identify if any those changed - // and a restart/recreate is required. + // create custom config for this manila service // - inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + err = r.generateServiceConfig(ctx, helper, instance, &configVars, serviceLabels) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -652,10 +801,18 @@ func (r *ManilaAPIReconciler) reconcileNormal(ctx context.Context, instance *man // // Define a new Statefulset - ssDef := statefulset.NewStatefulSet( - manilaapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations), - time.Duration(5)*time.Second, - ) + ss, err := manilaapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + ssDef := statefulset.NewStatefulSet(ss, time.Duration(5)*time.Second) ctrlResult, err = ssDef.CreateOrPatch(ctx, helper) if err != nil { diff --git a/controllers/manilascheduler_controller.go b/controllers/manilascheduler_controller.go index 18108536..7e62d076 100644 --- a/controllers/manilascheduler_controller.go +++ b/controllers/manilascheduler_controller.go @@ -24,13 +24,17 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -43,6 +47,7 @@ import ( nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" manilav1beta1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" "github.com/openstack-k8s-operators/manila-operator/pkg/manila" @@ -151,6 +156,7 @@ func (r *ManilaSchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Requ condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -227,6 +233,30 @@ func (r *ManilaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaScheduler{}, passwordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaScheduler) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaScheduler{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaScheduler) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&manilav1beta1.ManilaScheduler{}). Owns(&appsv1.StatefulSet{}). @@ -234,9 +264,47 @@ func (r *ManilaSchedulerReconciler) SetupWithManager(mgr ctrl.Manager) error { // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } +func (r *ManilaSchedulerReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("ManilaScheduler") + + for _, field := range commonWatchFields { + crList := &manilav1beta1.ManilaSchedulerList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + func (r *ManilaSchedulerReconciler) reconcileDelete(ctx context.Context, instance *manilav1beta1.ManilaScheduler, helper *helper.Helper) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) @@ -304,6 +372,38 @@ func (r *ManilaSchedulerReconciler) reconcileNormal(ctx context.Context, instanc } instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + // // Create Secrets required as input for the Service and calculate an overall hash of hashes // diff --git a/controllers/manilashare_controller.go b/controllers/manilashare_controller.go index 734b7f8f..5e87f73e 100644 --- a/controllers/manilashare_controller.go +++ b/controllers/manilashare_controller.go @@ -24,13 +24,17 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -43,6 +47,7 @@ import ( nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" manilav1beta1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" "github.com/openstack-k8s-operators/manila-operator/pkg/manila" @@ -152,6 +157,7 @@ func (r *ManilaShareReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -225,6 +231,30 @@ func (r *ManilaShareReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaShare{}, passwordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaShare) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &manilav1beta1.ManilaShare{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*manilav1beta1.ManilaShare) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&manilav1beta1.ManilaShare{}). Owns(&appsv1.StatefulSet{}). @@ -232,9 +262,47 @@ func (r *ManilaShareReconciler) SetupWithManager(mgr ctrl.Manager) error { // watch the secrets we don't own Watches(&source.Kind{Type: &corev1.Secret{}}, handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } +func (r *ManilaShareReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("CinderBackup") + + for _, field := range commonWatchFields { + crList := &manilav1beta1.ManilaShareList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + func (r *ManilaShareReconciler) reconcileDelete(ctx context.Context, instance *manilav1beta1.ManilaShare, helper *helper.Helper) (ctrl.Result, error) { r.Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) @@ -303,6 +371,38 @@ func (r *ManilaShareReconciler) reconcileNormal(ctx context.Context, instance *m } instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + serviceLabels := map[string]string{ common.AppSelector: manila.ServiceName, common.ComponentSelector: manilashare.Component, diff --git a/pkg/manila/cronjob.go b/pkg/manila/cronjob.go index 0b158897..1c4a54f9 100644 --- a/pkg/manila/cronjob.go +++ b/pkg/manila/cronjob.go @@ -17,6 +17,7 @@ package manila import ( "fmt" + manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" batchv1 "k8s.io/api/batch/v1" @@ -73,6 +74,12 @@ func CronJob( }, } + // add CA cert if defined + if instance.Spec.ManilaAPI.TLS.CaBundleSecretName != "" { + cronJobVolume = append(cronJobVolume, instance.Spec.ManilaAPI.TLS.CreateVolume()) + cronJobVolumeMounts = append(cronJobVolumeMounts, instance.Spec.ManilaAPI.TLS.CreateVolumeMounts(nil)...) + } + cronjob := &batchv1.CronJob{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-db-purge", ServiceName), diff --git a/pkg/manila/dbsync.go b/pkg/manila/dbsync.go index fa644148..b4447aa6 100644 --- a/pkg/manila/dbsync.go +++ b/pkg/manila/dbsync.go @@ -35,6 +35,10 @@ func DbSyncJob(instance *manilav1.Manila, labels map[string]string, annotations Key: DefaultsConfigFileName, Path: DefaultsConfigFileName, }, + { + Key: CustomConfigFileName, + Path: CustomConfigFileName, + }, }, }, }, @@ -64,6 +68,12 @@ func DbSyncJob(instance *manilav1.Manila, labels map[string]string, annotations }, } + // add CA cert if defined + if instance.Spec.ManilaAPI.TLS.CaBundleSecretName != "" { + dbSyncVolume = append(dbSyncVolume, instance.Spec.ManilaAPI.TLS.CreateVolume()) + dbSyncMounts = append(dbSyncMounts, instance.Spec.ManilaAPI.TLS.CreateVolumeMounts(nil)...) + } + args := []string{"-c"} if instance.Spec.Debug.DBSync { args = append(args, common.DebugCommand) diff --git a/pkg/manilaapi/statefulset.go b/pkg/manilaapi/statefulset.go index e3756792..2e85786c 100644 --- a/pkg/manilaapi/statefulset.go +++ b/pkg/manilaapi/statefulset.go @@ -16,6 +16,8 @@ import ( common "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" manilav1 "github.com/openstack-k8s-operators/manila-operator/api/v1beta1" manila "github.com/openstack-k8s-operators/manila-operator/pkg/manila" @@ -36,7 +38,7 @@ func StatefulSet( configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.StatefulSet { +) (*appsv1.StatefulSet, error) { runAsUser := int64(0) livenessProbe := &corev1.Probe{ @@ -70,6 +72,43 @@ func StatefulSet( Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(manila.ManilaPublicPort)}, } readinessProbe.HTTPGet = livenessProbe.HTTPGet + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + } + + // create Volume and VolumeMounts + volumes := GetVolumes( + manila.GetOwningManilaName(instance), + instance.Name, + instance.Spec.ExtraMounts) + volumeMounts := GetVolumeMounts(instance.Spec.ExtraMounts) + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } } envVars := map[string]env.Setter{} @@ -128,23 +167,20 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &runAsUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: append(GetVolumeMounts(instance.Spec.ExtraMounts), - []corev1.VolumeMount{GetLogVolumeMount()}...), + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, }, }, NodeSelector: instance.Spec.NodeSelector, + Volumes: volumes, }, }, }, } - statefulset.Spec.Template.Spec.Volumes = append(GetVolumes( - manila.GetOwningManilaName(instance), - instance.Name, - instance.Spec.ExtraMounts), GetLogVolume()) + // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. @@ -159,5 +195,5 @@ func StatefulSet( statefulset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector } - return statefulset + return statefulset, nil } diff --git a/pkg/manilascheduler/statefulset.go b/pkg/manilascheduler/statefulset.go index 21d7d198..ef537f47 100644 --- a/pkg/manilascheduler/statefulset.go +++ b/pkg/manilascheduler/statefulset.go @@ -89,8 +89,18 @@ func StatefulSet( envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) + volumes := GetVolumes( + manila.GetOwningManilaName(instance), + instance.Name, + instance.Spec.ExtraMounts) + volumeMounts := GetVolumeMounts(instance.Spec.ExtraMounts) + // Add the CA bundle + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -138,15 +148,12 @@ func StatefulSet( }, }, NodeSelector: instance.Spec.NodeSelector, + Volumes: volumes, }, }, }, } - statefulset.Spec.Template.Spec.Volumes = GetVolumes( - manila.GetOwningManilaName(instance), - instance.Name, - instance.Spec.ExtraMounts, - ) + // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. diff --git a/pkg/manilashare/statefulset.go b/pkg/manilashare/statefulset.go index d06d9e9d..017984ad 100644 --- a/pkg/manilashare/statefulset.go +++ b/pkg/manilashare/statefulset.go @@ -99,11 +99,22 @@ func StatefulSet( envVars["MALLOC_MMAP_THRESHOLD_"] = env.SetValue("131072") envVars["MALLOC_TRIM_THRESHOLD_"] = env.SetValue("262144") + volumes := GetVolumes( + manila.GetOwningManilaName(instance), + instance.Name, + instance.Spec.ExtraMounts) + volumeMounts := GetVolumeMounts( instance.Name, instance.Spec.ExtraMounts, ) + // Add the CA bundle + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + statefulset := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, @@ -152,15 +163,12 @@ func StatefulSet( }, }, NodeSelector: instance.Spec.NodeSelector, + Volumes: volumes, }, }, }, } - statefulset.Spec.Template.Spec.Volumes = GetVolumes( - manila.GetOwningManilaName(instance), - instance.Name, - instance.Spec.ExtraMounts, - ) + // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. diff --git a/templates/manila/config/10-manila_wsgi.conf b/templates/manila/config/10-manila_wsgi.conf index 7c170151..eb3bf215 100644 --- a/templates/manila/config/10-manila_wsgi.conf +++ b/templates/manila/config/10-manila_wsgi.conf @@ -1,5 +1,7 @@ +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration - ServerName manilaapi.openstack.svc + ServerName {{ $vhost.ServerName }} ## Vhost docroot DocumentRoot "/var/www/cgi-bin/manila" @@ -16,11 +18,21 @@ ErrorLog /dev/stdout ServerSignature Off CustomLog /dev/stdout combined - SetEnvIf X-Forwarded-Proto https HTTPS=1 + + {{- if $vhost.TLS }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" + {{- end }} ## WSGI configuration WSGIApplicationGroup %{GLOBAL} - WSGIDaemonProcess manila-api display-name=manila_wsgi group=manila processes=4 threads=1 user=manila - WSGIProcessGroup manila-api + WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cinder processes=4 threads=1 user=cinder + WSGIProcessGroup {{ $endpt }} WSGIScriptAlias / "/var/www/cgi-bin/manila/manila-wsgi" + WSGIPassAuthorization On +{{ end }} diff --git a/templates/manila/config/manila-api-config.json b/templates/manila/config/manila-api-config.json index f58c45b0..8359fd86 100644 --- a/templates/manila/config/manila-api-config.json +++ b/templates/manila/config/manila-api-config.json @@ -12,6 +12,28 @@ "dest": "/etc/httpd/conf.d/10-manila_wsgi.conf", "owner": "root", "perm": "0644" + }, + { + "source": "/var/lib/config-data/merged/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "root", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "root", + "perm": "0600", + "optional": true, + "merge": true } ], "permissions": [ diff --git a/templates/manila/config/ssl.conf b/templates/manila/config/ssl.conf new file mode 100644 index 00000000..e3da4ecb --- /dev/null +++ b/templates/manila/config/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + diff --git a/test/functional/base_test.go b/test/functional/base_test.go index e5f67a6e..bfd3104c 100644 --- a/test/functional/base_test.go +++ b/test/functional/base_test.go @@ -15,7 +15,9 @@ package functional import ( "fmt" + . "github.com/onsi/gomega" + "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -103,6 +105,15 @@ func GetManilaCommonSpec(spec map[string]interface{}) map[string]interface{} { return defaultSpec } +func GetTLSManilaSpec() map[string]interface{} { + return map[string]interface{}{ + "databaseInstance": "openstack", + "secret": SecretName, + "manilaAPI": GetTLSManilaAPISpec(), + "manilaScheduler": GetDefaultManilaSchedulerSpec(), + } +} + func GetDefaultManilaAPISpec() map[string]interface{} { return map[string]interface{}{ "secret": SecretName, @@ -234,6 +245,25 @@ func GetManilaShare(name types.NamespacedName) *manilav1.ManilaShare { return instance } +func GetTLSManilaAPISpec() map[string]interface{} { + spec := GetDefaultManilaAPISpec() + maps.Copy(spec, map[string]interface{}{ + "tls": map[string]interface{}{ + "api": map[string]interface{}{ + "internal": map[string]interface{}{ + "secretName": InternalCertSecretName, + }, + "public": map[string]interface{}{ + "secretName": PublicCertSecretName, + }, + }, + "caBundleSecretName": CABundleSecretName, + }, + }) + + return spec +} + func ManilaAPIConditionGetter(name types.NamespacedName) condition.Conditions { instance := GetManilaAPI(name) return instance.Status.Conditions diff --git a/test/functional/manila_controller_test.go b/test/functional/manila_controller_test.go index c60d7c3e..6db30cbc 100644 --- a/test/functional/manila_controller_test.go +++ b/test/functional/manila_controller_test.go @@ -437,4 +437,62 @@ var _ = Describe("Manila controller", func() { Expect(endpoints).To(HaveKeyWithValue("internal", "http://manila-internal."+manila.Namespace+".svc:8786/v2")) }) }) + + When("A Manila with TLS is created", func() { + BeforeEach(func() { + DeferCleanup(th.DeleteInstance, CreateManila(manilaTest.Instance, GetTLSManilaSpec())) + DeferCleanup(k8sClient.Delete, ctx, CreateManilaMessageBusSecret(manilaTest.Instance.Namespace, manilaTest.RabbitmqSecretName)) + DeferCleanup(th.DeleteInstance, CreateManilaAPI(manilaTest.Instance, GetDefaultManilaAPISpec())) + DeferCleanup(th.DeleteInstance, CreateManilaScheduler(manilaTest.Instance, GetDefaultManilaSchedulerSpec())) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + manilaTest.Instance.Namespace, + GetManila(manilaName).Spec.DatabaseInstance, + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + infra.SimulateTransportURLReady(manilaTest.ManilaTransportURL) + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(namespace, manilaTest.MemcachedInstance, memcachedSpec)) + infra.SimulateMemcachedReady(manilaTest.ManilaMemcached) + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(manilaTest.Instance.Namespace)) + mariadb.SimulateMariaDBDatabaseCompleted(manilaTest.Instance) + th.SimulateJobSuccess(manilaTest.ManilaDBSync) + }) + + It("Creates ManilaAPI", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(manilaTest.CABundleSecret)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(manilaTest.InternalCertSecret)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(manilaTest.PublicCertSecret)) + //keystone.SimulateKeystoneServiceReady(manilaTest.ManilaKeystoneService) + keystone.SimulateKeystoneEndpointReady(manilaTest.ManilaKeystoneEndpoint) + + ManilaAPIExists(manilaTest.Instance) + + d := th.GetStatefulSet(manilaTest.ManilaAPI) + // Check the resulting deployment fields + Expect(int(*d.Spec.Replicas)).To(Equal(1)) + + Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(8)) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(2)) + + // cert deployment volumes + th.AssertVolumeExists(manilaTest.CABundleSecret.Name, d.Spec.Template.Spec.Volumes) + th.AssertVolumeExists(manilaTest.InternalCertSecret.Name, d.Spec.Template.Spec.Volumes) + th.AssertVolumeExists(manilaTest.PublicCertSecret.Name, d.Spec.Template.Spec.Volumes) + + // cert volumeMounts + container := d.Spec.Template.Spec.Containers[1] + th.AssertVolumeMountExists(manilaTest.InternalCertSecret.Name, "tls.key", container.VolumeMounts) + th.AssertVolumeMountExists(manilaTest.InternalCertSecret.Name, "tls.crt", container.VolumeMounts) + th.AssertVolumeMountExists(manilaTest.PublicCertSecret.Name, "tls.key", container.VolumeMounts) + th.AssertVolumeMountExists(manilaTest.PublicCertSecret.Name, "tls.crt", container.VolumeMounts) + th.AssertVolumeMountExists(manilaTest.CABundleSecret.Name, "tls-ca-bundle.pem", container.VolumeMounts) + + Expect(container.ReadinessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS)) + Expect(container.LivenessProbe.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS)) + }) + }) }) diff --git a/test/functional/manila_test_data.go b/test/functional/manila_test_data.go index 13b90e34..479d7db6 100644 --- a/test/functional/manila_test_data.go +++ b/test/functional/manila_test_data.go @@ -19,12 +19,19 @@ package functional import ( "fmt" + "k8s.io/apimachinery/pkg/types" ) const ( // MemcachedInstance - name of the memcached instance MemcachedInstance = "memcached" + //PublicCertSecretName - + PublicCertSecretName = "public-tls-certs" + //InternalCertSecretName - + InternalCertSecretName = "internal-tls-certs" + //CABundleSecretName - + CABundleSecretName = "combined-ca-bundle" ) // ManilaTestData is the data structure used to provide input data to envTest @@ -57,6 +64,9 @@ type ManilaTestData struct { ManilaDefaultShare types.NamespacedName InternalAPINAD types.NamespacedName ContainerImage string + CABundleSecret types.NamespacedName + InternalCertSecret types.NamespacedName + PublicCertSecret types.NamespacedName } // GetManilaTestData is a function that initialize the ManilaTestData @@ -159,5 +169,18 @@ func GetManilaTestData(manilaName types.NamespacedName) ManilaTestData { ManilaPassword: "12345678", ManilaServiceUser: "manila", ContainerImage: "test://manila", + CABundleSecret: types.NamespacedName{ + Namespace: manilaName.Namespace, + Name: CABundleSecretName, + }, + + InternalCertSecret: types.NamespacedName{ + Namespace: manilaName.Namespace, + Name: InternalCertSecretName, + }, + PublicCertSecret: types.NamespacedName{ + Namespace: manilaName.Namespace, + Name: PublicCertSecretName, + }, } } diff --git a/test/functional/manilaapi_controller_test.go b/test/functional/manilaapi_controller_test.go index eb11aaeb..3cd18706 100644 --- a/test/functional/manilaapi_controller_test.go +++ b/test/functional/manilaapi_controller_test.go @@ -123,11 +123,11 @@ var _ = Describe("ManilaAPI controller", func() { ss := th.GetStatefulSet(manilaTest.ManilaAPI) // Check the resulting deployment fields Expect(int(*ss.Spec.Replicas)).To(Equal(1)) - Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(6)) + Expect(ss.Spec.Template.Spec.Volumes).To(HaveLen(5)) Expect(ss.Spec.Template.Spec.Containers).To(HaveLen(2)) container := ss.Spec.Template.Spec.Containers[1] - Expect(container.VolumeMounts).To(HaveLen(7)) + Expect(container.VolumeMounts).To(HaveLen(6)) Expect(container.Image).To(Equal(manilaTest.ContainerImage)) Expect(container.LivenessProbe.HTTPGet.Port.IntVal).To(Equal(int32(8786))) Expect(container.ReadinessProbe.HTTPGet.Port.IntVal).To(Equal(int32(8786)))