diff --git a/api/bases/placement.openstack.org_placementapis.yaml b/api/bases/placement.openstack.org_placementapis.yaml index d7ed94b2..fb0367d3 100644 --- a/api/bases/placement.openstack.org_placementapis.yaml +++ b/api/bases/placement.openstack.org_placementapis.yaml @@ -365,6 +365,39 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + disabled: + description: Disabled TLS for the deployment of the service + type: boolean + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - containerImage - databaseInstance diff --git a/api/go.mod b/api/go.mod index 9bcc5072..8f3c2ee0 100644 --- a/api/go.mod +++ b/api/go.mod @@ -35,6 +35,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.13.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -47,6 +48,7 @@ require ( golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.16.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect @@ -67,3 +69,14 @@ require ( // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace ( //allow-merging + github.com/google/gnostic => github.com/google/gnostic v0.6.9 + // pin to k8s 0.26.x for now + k8s.io/api => k8s.io/api v0.26.11 + k8s.io/apimachinery => k8s.io/apimachinery v0.26.11 + k8s.io/client-go => k8s.io/client-go v0.26.11 + sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.14.7 +) + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20231207114330-c01061824e7b diff --git a/api/go.sum b/api/go.sum index de9218f0..e8992457 100644 --- a/api/go.sum +++ b/api/go.sum @@ -220,8 +220,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231209173030-f7a552f208e7 h1:fZ0YF3m9kJDPjHg2rC6XJbGPxv00Nc9N/1/Cdg1Die0= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231209173030-f7a552f208e7/go.mod h1:gcsno+cczP8lUANyOQ/jHKHv3QgzmZvLv8cMSS0rMws= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -276,6 +274,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stuggi/lib-common/modules/common v0.0.0-20231207114330-c01061824e7b h1:k7QWu0G96ZWeHiQnIcEEWQ39EDNsiZRg5zXI9DSk1aY= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/api/v1beta1/placementapi_types.go b/api/v1beta1/placementapi_types.go index 910ef42d..5abf7493 100644 --- a/api/v1beta1/placementapi_types.go +++ b/api/v1beta1/placementapi_types.go @@ -19,6 +19,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" @@ -115,6 +116,11 @@ type PlacementAPISpec struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` } // APIOverrideSpec to override the generated manifest of several child resources. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5067acac..202b2c1d 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -184,6 +184,7 @@ func (in *PlacementAPISpec) DeepCopyInto(out *PlacementAPISpec) { copy(*out, *in) } in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementAPISpec. diff --git a/config/crd/bases/placement.openstack.org_placementapis.yaml b/config/crd/bases/placement.openstack.org_placementapis.yaml index d7ed94b2..fb0367d3 100644 --- a/config/crd/bases/placement.openstack.org_placementapis.yaml +++ b/config/crd/bases/placement.openstack.org_placementapis.yaml @@ -365,6 +365,39 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + disabled: + description: Disabled TLS for the deployment of the service + type: boolean + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - containerImage - databaseInstance diff --git a/config/samples/placement_v1beta1_placementapi.yaml b/config/samples/placement_v1beta1_placementapi.yaml index c9e4d698..cd0907b1 100644 --- a/config/samples/placement_v1beta1_placementapi.yaml +++ b/config/samples/placement_v1beta1_placementapi.yaml @@ -14,6 +14,14 @@ spec: service: false preserveJobs: false replicas: 1 + # tls: + # api: + # disabled: false + # internal: + # secretName: cert-internal-svc + # public: + # secretName: cert-public-svc + # caBundleSecretName: combined-ca-bundle secret: placement-secret #resources: # requests: diff --git a/controllers/placementapi_controller.go b/controllers/placementapi_controller.go index 2e25041a..13891c1d 100644 --- a/controllers/placementapi_controller.go +++ b/controllers/placementapi_controller.go @@ -21,12 +21,20 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -44,6 +52,7 @@ import ( common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" util "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -192,8 +201,73 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request return r.reconcileNormal(ctx, instance, helper) } +// fields to index to reconcile when change +const ( + passwordSecretField = ".spec.secret" + caBundleSecretNameField = ".spec.tls.caBundleSecretName" + tlsAPIInternalField = ".spec.tls.api.internal.secretName" + tlsAPIPublicField = ".spec.tls.api.public.secretName" +) + +var ( + allWatchFields = []string{ + passwordSecretField, + caBundleSecretNameField, + tlsAPIInternalField, + tlsAPIPublicField, + } +) + // SetupWithManager sets up the controller with the Manager. func (r *PlacementAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &placementv1.PlacementAPI{}, passwordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*placementv1.PlacementAPI) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &placementv1.PlacementAPI{}, caBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*placementv1.PlacementAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &placementv1.PlacementAPI{}, tlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*placementv1.PlacementAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &placementv1.PlacementAPI{}, tlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*placementv1.PlacementAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). For(&placementv1.PlacementAPI{}). Owns(&mariadbv1.MariaDBDatabase{}). @@ -207,9 +281,47 @@ func (r *PlacementAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Watches( + &source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } +func (r *PlacementAPIReconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(context.Background()).WithName("Controllers").WithName("PlacementAPI") + + for _, field := range allWatchFields { + crList := &placementv1.PlacementAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + func (r *PlacementAPIReconciler) reconcileDelete(ctx context.Context, instance *placementv1.PlacementAPI, helper *helper.Helper) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info("Reconciling Service delete") @@ -445,7 +557,12 @@ func (r *PlacementAPIReconciler) reconcileInit( } // create service - end - // TODO: TLS, pass in https as protocol, create TLS cert + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( svcOverride.EndpointURL, data.Protocol, data.Path) if err != nil { @@ -617,6 +734,9 @@ func (r *PlacementAPIReconciler) reconcileNormal(ctx context.Context, instance * // Create ConfigMaps and Secrets required as input for the Service and calculate an overall hash of hashes // + // TODO: should create cert secret before creating the ServiceConfigMaps, otherwise we'll have another job runs + // since config changes + // // create Configmap required for placement input // - %-scripts configmap holding scripts to e.g. bootstrap the service @@ -634,10 +754,43 @@ func (r *PlacementAPIReconciler) reconcileNormal(ctx context.Context, instance * return ctrl.Result{}, err } + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, ctrlResult, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + if hash != "" { + configMapVars[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API service certs secrets + certsHash, ctrlResult, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, helper, instance.Namespace) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + configMapVars[tls.TLSHashName] = env.SetValue(certsHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. // + // inputEnvMap := util.MergeMaps(configMapVars, initEnvVars) inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configMapVars) if err != nil { return ctrl.Result{}, err @@ -654,7 +807,8 @@ func (r *PlacementAPIReconciler) reconcileNormal(ctx context.Context, instance * // serviceLabels := map[string]string{ - common.AppSelector: placement.ServiceName, + common.AppSelector: placement.ServiceName, + common.OwnerSelector: instance.Name, } // networks to attach to @@ -687,7 +841,7 @@ func (r *PlacementAPIReconciler) reconcileNormal(ctx context.Context, instance * } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -715,8 +869,14 @@ func (r *PlacementAPIReconciler) reconcileNormal(ctx context.Context, instance * // // Define a new Deployment object - deplDef := placement.Deployment(instance, inputHash, serviceLabels, serviceAnnotations) + deplDef, err := placement.Deployment(ctx, helper, instance, inputHash, serviceLabels, serviceAnnotations) if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) return ctrl.Result{}, err } depl := deployment.NewDeployment( @@ -816,6 +976,21 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps( templateParameters["KeystoneInternalURL"] = keystoneInternalURL templateParameters["KeystonePublicURL"] = keystonePublicURL + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("placement-%s.%s.svc", endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + cms := []util.Template{ // ScriptsConfigMap { diff --git a/go.mod b/go.mod index 593e22f3..41b1b573 100644 --- a/go.mod +++ b/go.mod @@ -90,3 +90,5 @@ replace github.com/openstack-k8s-operators/placement-operator/api => ./api // mschuppert: map to latest commit from release-4.13 tag // must consistent within modules and service operators replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging + +replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/stuggi/lib-common/modules/common v0.0.0-20231207114330-c01061824e7b diff --git a/go.sum b/go.sum index 2c05abb8..074b9997 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,6 @@ github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxC github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4= github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20231208104910-f8433c1c9399 h1:Te7JSPGGUhkzjig/1CjlPmQgMpHT0+yHWoTxbVJGJ74= github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20231208104910-f8433c1c9399/go.mod h1:kDtQ2LCkf28F7xgK8GBFAMPDhXnL6iRb8NztHhrYaO0= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231209173030-f7a552f208e7 h1:fZ0YF3m9kJDPjHg2rC6XJbGPxv00Nc9N/1/Cdg1Die0= -github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231209173030-f7a552f208e7/go.mod h1:gcsno+cczP8lUANyOQ/jHKHv3QgzmZvLv8cMSS0rMws= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20231209173030-f7a552f208e7 h1:qTcC8VRNJYgmyrzEZxMa0NNq8t/Uii8OTSjWN8vvZGA= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20231209173030-f7a552f208e7/go.mod h1:SfdtKhpn1MGom0ZEOsgY27QFu+7SdDhO/LiuDe5cDXk= github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20231209173030-f7a552f208e7 h1:3Z0LR2duuWlZ0V2YTWI+2hMZR25fkcOc0xuYY41U/5U= @@ -300,6 +298,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stuggi/lib-common/modules/common v0.0.0-20231207114330-c01061824e7b h1:k7QWu0G96ZWeHiQnIcEEWQ39EDNsiZRg5zXI9DSk1aY= +github.com/stuggi/lib-common/modules/common v0.0.0-20231207114330-c01061824e7b/go.mod h1:ImxqioQ1ID+d7fMMD4lK8CxJqNTB5tsQ+lGKcN/xx5M= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= diff --git a/pkg/placement/dbsync.go b/pkg/placement/dbsync.go index 87e28b59..08b3a22b 100644 --- a/pkg/placement/dbsync.go +++ b/pkg/placement/dbsync.go @@ -44,6 +44,18 @@ func DbSyncJob( envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true") + // create Volume and VolumeMounts + volumes := getVolumes(instance.Name) + volumeMounts := getVolumeMounts() + initVolumeMounts := getInitVolumeMounts() + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(getVolumes(instance.Name), instance.Spec.TLS.CreateVolume()) + volumeMounts = append(getVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) + initVolumeMounts = append(getInitVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: ServiceName + "-db-sync", @@ -70,16 +82,15 @@ func DbSyncJob( RunAsUser: ptr.To(PlacementUserID), }, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: getVolumeMounts("dbsync"), + VolumeMounts: volumeMounts, }, }, + Volumes: volumes, }, }, }, } - job.Spec.Template.Spec.Volumes = getVolumes(ServiceName) - initContainerDetails := APIDetails{ ContainerImage: instance.Spec.ContainerImage, DatabaseHost: instance.Status.DatabaseHostname, @@ -88,7 +99,7 @@ func DbSyncJob( OSPSecret: instance.Spec.Secret, DBPasswordSelector: instance.Spec.PasswordSelectors.Database, UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: getInitVolumeMounts(), + VolumeMounts: initVolumeMounts, } job.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails) diff --git a/pkg/placement/deployment.go b/pkg/placement/deployment.go index 08bb80ed..8ae0438d 100644 --- a/pkg/placement/deployment.go +++ b/pkg/placement/deployment.go @@ -16,9 +16,14 @@ limitations under the License. package placement import ( + "context" + common "github.com/openstack-k8s-operators/lib-common/modules/common" affinity "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" env "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" placementv1 "github.com/openstack-k8s-operators/placement-operator/api/v1beta1" @@ -26,16 +31,24 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" ) // Deployment func func Deployment( + ctx context.Context, + helper *helper.Helper, instance *placementv1.PlacementAPI, configHash string, labels map[string]string, annotations map[string]string, -) *appsv1.Deployment { +) (*appsv1.Deployment, error) { + runAsUser := int64(0) + livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, @@ -64,7 +77,7 @@ func Deployment( }, } } else { - args = append(args, KollaServiceCommand) + args = append(args, ServiceCommand) // // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ // @@ -74,12 +87,49 @@ func Deployment( readinessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(PlacementPublicPort)}, } + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } } envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") envVars["CONFIG_HASH"] = env.SetValue(configHash) + // create Volume and VolumeMounts + volumes := getVolumes(instance.Name) + volumeMounts := getVolumeMounts() + initVolumeMounts := getInitVolumeMounts() + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(getVolumes(instance.Name), instance.Spec.TLS.CreateVolume()) + volumeMounts = append(getVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) + initVolumeMounts = append(getInitVolumeMounts(), instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + initVolumeMounts = append(initVolumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: ServiceName, @@ -97,6 +147,7 @@ func Deployment( }, Spec: corev1.PodSpec{ ServiceAccountName: instance.RbacResourceName(), + Volumes: volumes, Containers: []corev1.Container{ { Name: ServiceName + "-api", @@ -106,10 +157,10 @@ func Deployment( Args: args, Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ - RunAsUser: ptr.To(PlacementUserID), + RunAsUser: &runAsUser, }, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: getVolumeMounts("api"), + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, @@ -120,7 +171,6 @@ func Deployment( }, } - deployment.Spec.Template.Spec.Volumes = getVolumes(instance.Name) // If possible two pods of the same service should not // run on the same worker node. If this is not possible // the get still created on the same worker node. @@ -143,9 +193,9 @@ func Deployment( OSPSecret: instance.Spec.Secret, DBPasswordSelector: instance.Spec.PasswordSelectors.Database, UserPasswordSelector: instance.Spec.PasswordSelectors.Service, - VolumeMounts: getInitVolumeMounts(), + VolumeMounts: initVolumeMounts, } deployment.Spec.Template.Spec.InitContainers = initContainer(initContainerDetails) - return deployment + return deployment, nil } diff --git a/pkg/placement/initcontainer.go b/pkg/placement/initcontainer.go index f62f370b..bc2ed759 100644 --- a/pkg/placement/initcontainer.go +++ b/pkg/placement/initcontainer.go @@ -90,7 +90,7 @@ func initContainer(init APIDetails) []corev1.Container { }, Args: args, Env: envs, - VolumeMounts: getInitVolumeMounts(), + VolumeMounts: init.VolumeMounts, }, } } diff --git a/pkg/placement/volumes.go b/pkg/placement/volumes.go index eec541fc..db907e4d 100644 --- a/pkg/placement/volumes.go +++ b/pkg/placement/volumes.go @@ -79,7 +79,7 @@ func getInitVolumeMounts() []corev1.VolumeMount { } // getVolumeMounts - general VolumeMounts -func getVolumeMounts(serviceName string) []corev1.VolumeMount { +func getVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "scripts", @@ -94,7 +94,7 @@ func getVolumeMounts(serviceName string) []corev1.VolumeMount { { Name: "config-data-merged", MountPath: "/var/lib/kolla/config_files/config.json", - SubPath: "placement-" + serviceName + "-config.json", + SubPath: "placement-api-config.json", ReadOnly: true, }, } diff --git a/templates/placementapi/config/httpd.conf b/templates/placementapi/config/httpd.conf index a42952e2..6b502177 100644 --- a/templates/placementapi/config/httpd.conf +++ b/templates/placementapi/config/httpd.conf @@ -28,14 +28,37 @@ TransferLog /dev/stdout CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration = 2.4> ErrorLogFormat "%M" + ServerName {{ $vhost.ServerName }} + + ## Vhost docroot ErrorLog /dev/stdout SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded CustomLog /dev/stdout combined env=!forwarded CustomLog /dev/stdout proxy env=forwarded + ServerSignature Off + CustomLog /dev/stdout combined + +{{- if $vhost.TLS }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" +{{- end }} + + ## Directories, there should at least be a declaration for /var/www/cgi-bin/placement + + Options -Indexes +FollowSymLinks +MultiViews + AllowOverride None + Require all granted + ## WSGI configuration WSGIProcessGroup placement-api @@ -44,6 +67,7 @@ CustomLog /dev/stdout proxy env=forwarded WSGIDaemonProcess placement-api processes=3 threads=1 user=placement group=placement WSGIScriptAlias / /usr/bin/placement-api +{{ end }} Alias /placement-api /usr/bin/placement-api @@ -52,4 +76,4 @@ Alias /placement-api /usr/bin/placement-api WSGIProcessGroup placement-api WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On - + \ No newline at end of file diff --git a/templates/placementapi/config/placement-api-config.json b/templates/placementapi/config/placement-api-config.json index d7db9837..2585baa3 100644 --- a/templates/placementapi/config/placement-api-config.json +++ b/templates/placementapi/config/placement-api-config.json @@ -19,6 +19,12 @@ "owner": "apache", "perm": "0644" }, + { + "source": "/var/lib/config-data/merged/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "root", + "perm": "0644" + }, { "source": "/var/lib/config-data/merged/logging.conf", "dest": "/etc/placement/logging.conf", diff --git a/templates/placementapi/config/ssl.conf b/templates/placementapi/config/ssl.conf new file mode 100644 index 00000000..865e9f0c --- /dev/null +++ b/templates/placementapi/config/ssl.conf @@ -0,0 +1,22 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + #SSLCACertificateFile "/etc/ipa/ca.crt" + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + \ No newline at end of file diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index bc90e4ef..b5cc0f0c 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -28,19 +28,22 @@ import ( ) type Names struct { - Namespace string - PlacementAPIName types.NamespacedName - ConfigMapName types.NamespacedName - DBSyncJobName types.NamespacedName - MariaDBDatabaseName types.NamespacedName - DeploymentName types.NamespacedName - PublicServiceName types.NamespacedName - InternalServiceName types.NamespacedName - KeystoneServiceName types.NamespacedName - KeystoneEndpointName types.NamespacedName - ServiceAccountName types.NamespacedName - RoleName types.NamespacedName - RoleBindingName types.NamespacedName + Namespace string + PlacementAPIName types.NamespacedName + ConfigMapName types.NamespacedName + DBSyncJobName types.NamespacedName + MariaDBDatabaseName types.NamespacedName + DeploymentName types.NamespacedName + PublicServiceName types.NamespacedName + InternalServiceName types.NamespacedName + KeystoneServiceName types.NamespacedName + KeystoneEndpointName types.NamespacedName + ServiceAccountName types.NamespacedName + RoleName types.NamespacedName + RoleBindingName types.NamespacedName + CaBundleSecretName types.NamespacedName + InternalCertSecretName types.NamespacedName + PublicCertSecretName types.NamespacedName } func CreateNames(placementAPIName types.NamespacedName) Names { @@ -82,6 +85,15 @@ func CreateNames(placementAPIName types.NamespacedName) Names { RoleBindingName: types.NamespacedName{ Namespace: placementAPIName.Namespace, Name: "placement-" + placementAPIName.Name + "-rolebinding"}, + CaBundleSecretName: types.NamespacedName{ + Namespace: placementAPIName.Namespace, + Name: "combined-ca-bundle"}, + InternalCertSecretName: types.NamespacedName{ + Namespace: placementAPIName.Namespace, + Name: "internal-tls-certs"}, + PublicCertSecretName: types.NamespacedName{ + Namespace: placementAPIName.Namespace, + Name: "public-tls-certs"}, } } @@ -92,6 +104,25 @@ func GetDefaultPlacementAPISpec() map[string]interface{} { } } +func GetTLSPlacementAPISpec() map[string]interface{} { + return map[string]interface{}{ + "databaseInstance": "openstack", + "replicas": 1, + "secret": SecretName, + "tls": map[string]interface{}{ + "api": map[string]interface{}{ + "internal": map[string]interface{}{ + "secretName": InternalCertSecretName, + }, + "public": map[string]interface{}{ + "secretName": PublicCertSecretName, + }, + }, + "caBundleSecretName": CABundleSecretName, + }, + } +} + func CreatePlacementAPI(name types.NamespacedName, spec map[string]interface{}) client.Object { raw := map[string]interface{}{ diff --git a/tests/functional/placementapi_controller_test.go b/tests/functional/placementapi_controller_test.go index ca9d6dd6..3bb7050d 100644 --- a/tests/functional/placementapi_controller_test.go +++ b/tests/functional/placementapi_controller_test.go @@ -30,6 +30,7 @@ import ( ) var _ = Describe("PlacementAPI controller", func() { + BeforeEach(func() { // lib-common uses OPERATOR_TEMPLATES env var to locate the "templates" // directory of the operator. We need to set them othervise lib-common @@ -379,7 +380,7 @@ var _ = Describe("PlacementAPI controller", func() { deployment := th.GetDeployment(names.DeploymentName) Expect(int(*deployment.Spec.Replicas)).To(Equal(1)) - Expect(deployment.Spec.Selector.MatchLabels).To(Equal(map[string]string{"service": "placement"})) + Expect(deployment.Spec.Selector.MatchLabels).To(Equal(map[string]string{"service": "placement", "owner": names.PlacementAPIName.Name})) Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal(names.ServiceAccountName.Name)) th.SimulateDeploymentReplicaReady(names.DeploymentName) @@ -682,4 +683,49 @@ var _ = Describe("PlacementAPI controller", func() { }) }) + + When("A PlacementAPI is created with TLS", func() { + BeforeEach(func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(names.CaBundleSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.InternalCertSecretName)) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(names.PublicCertSecretName)) + DeferCleanup( + th.DeleteInstance, + CreatePlacementAPI(names.PlacementAPIName, GetTLSPlacementAPISpec()), + ) + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(namespace)) + DeferCleanup( + k8sClient.Delete, ctx, CreatePlacementAPISecret(namespace, SecretName)) + + spec := GetTLSPlacementAPISpec() + placement := CreatePlacementAPI(names.PlacementAPIName, spec) + + serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}} + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService(namespace, "openstack", serviceSpec), + ) + mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName) + keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName) + keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName) + th.SimulateJobSuccess(names.DBSyncJobName) + DeferCleanup(th.DeleteInstance, placement) + }) + + It("it creates deployment with CA and service certs mounted", func() { + j := th.GetDeployment(names.DeploymentName) + + // CA bundle + th.AssertVolumeExists(names.CaBundleSecretName.Name, j.Spec.Template.Spec.Volumes) + th.AssertVolumeMountExists(names.CaBundleSecretName.Name, "tls-ca-bundle.pem", j.Spec.Template.Spec.Containers[0].VolumeMounts) + + // service certs + th.AssertVolumeExists(names.InternalCertSecretName.Name, j.Spec.Template.Spec.Volumes) + th.AssertVolumeExists(names.PublicCertSecretName.Name, j.Spec.Template.Spec.Volumes) + th.AssertVolumeMountExists(names.PublicCertSecretName.Name, "tls.key", j.Spec.Template.Spec.Containers[0].VolumeMounts) + th.AssertVolumeMountExists(names.PublicCertSecretName.Name, "tls.crt", j.Spec.Template.Spec.Containers[0].VolumeMounts) + th.AssertVolumeMountExists(names.InternalCertSecretName.Name, "tls.key", j.Spec.Template.Spec.Containers[0].VolumeMounts) + th.AssertVolumeMountExists(names.InternalCertSecretName.Name, "tls.crt", j.Spec.Template.Spec.Containers[0].VolumeMounts) + }) + }) }) diff --git a/tests/functional/suite_test.go b/tests/functional/suite_test.go index d7f8508b..677ccfb3 100644 --- a/tests/functional/suite_test.go +++ b/tests/functional/suite_test.go @@ -73,6 +73,12 @@ const ( SecretName = "test-osp-secret" + PublicCertSecretName = "public-tls-certs" + + InternalCertSecretName = "internal-tls-certs" + + CABundleSecretName = "combined-ca-bundle" + interval = time.Millisecond * 200 ) @@ -112,6 +118,8 @@ var _ = BeforeSuite(func() { }, } + logger = ctrl.Log.WithName("---Test---") + // cfg is defined in this file globally. cfg, err = testEnv.Start() Expect(err).NotTo(HaveOccurred())