WIP direct TLS connection to database service

This adds the ability to configure oslo.db/pymysql to connect to the database service over TLS. It requires adding TLS options to bind-mount a CA that can validate the TLS certificate exposed by the database service.
openstack-k8s-operators · Sep 5, 2023 · 05370df · 05370df
1 parent d73d07c
commit 05370df
Show file tree

Hide file tree

Showing 14 changed files with 199 additions and 16 deletions.
diff --git a/api/bases/keystone.openstack.org_keystoneapis.yaml b/api/bases/keystone.openstack.org_keystoneapis.yaml
@@ -249,6 +249,18 @@ spec:
                 description: Secret containing OpenStack password information for
                   keystone KeystoneDatabasePassword, AdminPassword
                 type: string
+              tls:
+                description: TLS certificate and CA for Keystone
+                properties:
+                  caSecretName:
+                    description: Secret in the same namespace containing the CA cert
+                      (ca.crt) for client certificate validation
+                    type: string
+                  secretName:
+                    description: Secret in the same namespace containing the server
+                      private key (tls.key) and public cert (tls.crt) for TLS
+                    type: string
+                type: object
               trustFlushArgs:
                 default: ""
                 description: TrustFlushArgs - Arguments added to keystone-manage trust_flush

diff --git a/api/v1beta1/keystoneapi_types.go b/api/v1beta1/keystoneapi_types.go
@@ -152,8 +152,20 @@ type KeystoneAPISpec struct {
 	// +kubebuilder:validation:Optional
 	// ExternalEndpoints, expose a VIP using a pre-created IPAddressPool
 	ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"`
+
+	// TLS certificate and CA for Keystone
+	TLS TLSSpec `json:"tls,omitempty"`
+}
+
+// TLSSpec defines the TLS options
+type TLSSpec struct {
+	// Secret in the same namespace containing the server private key (tls.key) and public cert (tls.crt) for TLS
+	SecretName string `json:"secretName,omitempty"`
+	// Secret in the same namespace containing the CA cert (ca.crt) for client certificate validation
+	CaSecretName string `json:"caSecretName,omitempty"`
 }
 
+
 // MetalLBConfig to configure the MetalLB loadbalancer service
 type MetalLBConfig struct {
 	// +kubebuilder:validation:Required

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/keystone.openstack.org_keystoneapis.yaml b/config/crd/bases/keystone.openstack.org_keystoneapis.yaml
@@ -249,6 +249,18 @@ spec:
                 description: Secret containing OpenStack password information for
                   keystone KeystoneDatabasePassword, AdminPassword
                 type: string
+              tls:
+                description: TLS certificate and CA for Keystone
+                properties:
+                  caSecretName:
+                    description: Secret in the same namespace containing the CA cert
+                      (ca.crt) for client certificate validation
+                    type: string
+                  secretName:
+                    description: Secret in the same namespace containing the server
+                      private key (tls.key) and public cert (tls.crt) for TLS
+                    type: string
+                type: object
               trustFlushArgs:
                 default: ""
                 description: TrustFlushArgs - Arguments added to keystone-manage trust_flush

diff --git a/controllers/keystoneapi_controller.go b/controllers/keystoneapi_controller.go
@@ -38,6 +38,7 @@ import (
 	nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
 	oko_secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret"
+	secret "github.com/openstack-k8s-operators/lib-common/modules/common/secret"
 	util "github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	database "github.com/openstack-k8s-operators/lib-common/modules/database"
 	mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1"
@@ -405,7 +406,8 @@ func (r *KeystoneAPIReconciler) reconcileInit(
 		return ctrlResult, nil
 	}
 	// update Status.DatabaseHostname, used to bootstrap/config the service
-	instance.Status.DatabaseHostname = db.GetDatabaseHostname()
+	// TODO lib-common should return the FQDN for service
+	instance.Status.DatabaseHostname = db.GetDatabaseHostname() + "." + instance.GetNamespace() + ".svc"
 	instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
 
 	// create service DB - end
@@ -1100,3 +1102,47 @@ func (r *KeystoneAPIReconciler) getKeystoneMemcached(
 	}
 	return memcached, err
 }
+
+// getClientConfigForDatabase - client config flags to access a database
+// (e.g. enforce TLS connections if the database uses TLS)
+//
+// returns a string of mysql config statements, and any error
+// NOTE: currently unused
+func getClientConfigForDatabase(
+	ctx context.Context,
+	h *helper.Helper,
+	databaseName string,
+	namespace string,
+) (string, error) {
+	// . A MariaDBDatabase CR has a label `dbName` that references the
+	// DB server CR. When the server CR is configured to expose TLS, an
+	// optional mysql client config is read by oslo.db to connect via TLS
+	// . To check whether the server is configured with TLS, look for
+	// for a secret CR that has a label `mariadb-ref` that references the
+	// server CR.
+	db := &mariadbv1.MariaDBDatabase{}
+	err := h.GetClient().Get(ctx, types.NamespacedName{Name: databaseName, Namespace: namespace}, db)
+	if err != nil {
+		return "", client.IgnoreNotFound(err)
+	}
+	serverCRName := db.Labels["dbName"]
+
+	selector := map[string]string{
+		"mariadb-ref": serverCRName,
+	}
+	secretList, err := secret.GetSecrets(
+		ctx,
+		h,
+		h.GetBeforeObject().GetNamespace(),
+		selector,
+	)
+	if err != nil || len(secretList.Items) == 0 {
+		return "", fmt.Errorf("Error getting the DB certificate secrets using label %v: %w",
+			selector, err)
+	}
+
+	if len(secretList.Items) > 0 {
+		return "ssl=1\nssl-ca=/etc/ipa/ca.crt'", nil
+	}
+	return "", nil
+}
diff --git a/controllers/keystoneendpoint_controller.go b/controllers/keystoneendpoint_controller.go
@@ -280,7 +280,7 @@ func (r *KeystoneEndpointReconciler) reconcileDelete(
 				return ctrl.Result{}, err
 			}
 		}
-	} else if ! k8s_errors.IsNotFound(err) {
+	} else if !k8s_errors.IsNotFound(err) {
 		return ctrl.Result{}, err
 	}
 

diff --git a/pkg/keystone/bootstrap.go b/pkg/keystone/bootstrap.go
@@ -102,21 +102,22 @@ func BootstrapJob(
 									},
 								},
 							},
-							VolumeMounts: getVolumeMounts(),
+							VolumeMounts: getVolumeMounts(instance),
 						},
 					},
 				},
 			},
 		},
 	}
 	job.Spec.Template.Spec.Containers[0].Env = env.MergeEnvs(job.Spec.Template.Spec.Containers[0].Env, envVars)
-	job.Spec.Template.Spec.Volumes = getVolumes(instance.Name)
+	job.Spec.Template.Spec.Volumes = getVolumes(instance)
 
 	initContainerDetails := APIDetails{
 		ContainerImage:       instance.Spec.ContainerImage,
 		DatabaseHost:         instance.Status.DatabaseHostname,
 		DatabaseUser:         instance.Spec.DatabaseUser,
 		DatabaseName:         DatabaseName,
+		DatabaseUseTLS:       instance.Spec.TLS.CaSecretName != "",
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,

diff --git a/pkg/keystone/cronjob.go b/pkg/keystone/cronjob.go
@@ -79,13 +79,13 @@ func CronJob(
 									},
 									Args:         args,
 									Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-									VolumeMounts: getVolumeMounts(),
+									VolumeMounts: getVolumeMounts(instance),
 									SecurityContext: &corev1.SecurityContext{
 										RunAsUser: &runAsUser,
 									},
 								},
 							},
-							Volumes:            getVolumes(instance.Name),
+							Volumes:            getVolumes(instance),
 							RestartPolicy:      corev1.RestartPolicyNever,
 							ServiceAccountName: instance.RbacResourceName(),
 						},
@@ -103,6 +103,7 @@ func CronJob(
 		DatabaseHost:         instance.Status.DatabaseHostname,
 		DatabaseUser:         instance.Spec.DatabaseUser,
 		DatabaseName:         DatabaseName,
+		DatabaseUseTLS:       instance.Spec.TLS.CaSecretName != "",
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,

diff --git a/pkg/keystone/dbsync.go b/pkg/keystone/dbsync.go
@@ -75,20 +75,21 @@ func DbSyncJob(
 								RunAsUser: &runAsUser,
 							},
 							Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
-							VolumeMounts: getVolumeMounts(),
+							VolumeMounts: getVolumeMounts(instance),
 						},
 					},
 				},
 			},
 		},
 	}
 
-	job.Spec.Template.Spec.Volumes = getVolumes(ServiceName)
+	job.Spec.Template.Spec.Volumes = getVolumes(instance)
 	initContainerDetails := APIDetails{
 		ContainerImage:       instance.Spec.ContainerImage,
 		DatabaseHost:         instance.Status.DatabaseHostname,
 		DatabaseUser:         instance.Spec.DatabaseUser,
 		DatabaseName:         DatabaseName,
+		DatabaseUseTLS:       instance.Spec.TLS.CaSecretName != "",
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,

diff --git a/pkg/keystone/deployment.go b/pkg/keystone/deployment.go
@@ -105,7 +105,7 @@ func Deployment(
 				},
 				Spec: corev1.PodSpec{
 					ServiceAccountName: instance.RbacResourceName(),
-					Volumes:            getVolumes(instance.Name),
+					Volumes:            getVolumes(instance),
 					Containers: []corev1.Container{
 						{
 							Name: ServiceName + "-api",
@@ -118,7 +118,7 @@ func Deployment(
 								RunAsUser: &runAsUser,
 							},
 							Env:            env.MergeEnvs([]corev1.EnvVar{}, envVars),
-							VolumeMounts:   getVolumeMounts(),
+							VolumeMounts:   getVolumeMounts(instance),
 							Resources:      instance.Spec.Resources,
 							ReadinessProbe: readinessProbe,
 							LivenessProbe:  livenessProbe,
@@ -147,6 +147,7 @@ func Deployment(
 		DatabaseHost:         instance.Status.DatabaseHostname,
 		DatabaseUser:         instance.Spec.DatabaseUser,
 		DatabaseName:         DatabaseName,
+		DatabaseUseTLS:       instance.Spec.TLS.CaSecretName != "",
 		OSPSecret:            instance.Spec.Secret,
 		DBPasswordSelector:   instance.Spec.PasswordSelectors.Database,
 		UserPasswordSelector: instance.Spec.PasswordSelectors.Admin,

diff --git a/pkg/keystone/initcontainer.go b/pkg/keystone/initcontainer.go
@@ -27,6 +27,7 @@ type APIDetails struct {
 	DatabaseHost         string
 	DatabaseUser         string
 	DatabaseName         string
+	DatabaseUseTLS       bool
 	OSPSecret            string
 	DBPasswordSelector   string
 	UserPasswordSelector string
@@ -52,6 +53,14 @@ func initContainer(init APIDetails) []corev1.Container {
 	envVars["DatabaseUser"] = env.SetValue(init.DatabaseUser)
 	envVars["DatabaseName"] = env.SetValue(init.DatabaseName)
 
+	var clientConfig string
+	if init.DatabaseUseTLS {
+		clientConfig = "ssl=1\nssl-ca=/etc/ipa/ca.crt'"
+	} else {
+		clientConfig = ""
+	}
+	envVars["DatabaseClientConfig"] = env.SetValue(clientConfig)
+
 	envs := []corev1.EnvVar{
 		{
 			Name: "DatabasePassword",

diff --git a/pkg/keystone/volumes.go b/pkg/keystone/volumes.go
@@ -16,22 +16,24 @@ limitations under the License.
 package keystone
 
 import (
+	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
+
 	corev1 "k8s.io/api/core/v1"
 )
 
 // getVolumes - service volumes
-func getVolumes(name string) []corev1.Volume {
+func getVolumes(instance *keystonev1.KeystoneAPI) []corev1.Volume {
 	var scriptsVolumeDefaultMode int32 = 0755
 	var config0640AccessMode int32 = 0640
 
-	return []corev1.Volume{
+	volumes := []corev1.Volume{
 		{
 			Name: "scripts",
 			VolumeSource: corev1.VolumeSource{
 				ConfigMap: &corev1.ConfigMapVolumeSource{
 					DefaultMode: &scriptsVolumeDefaultMode,
 					LocalObjectReference: corev1.LocalObjectReference{
-						Name: name + "-scripts",
+						Name: instance.Name + "-scripts",
 					},
 				},
 			},
@@ -42,7 +44,18 @@ func getVolumes(name string) []corev1.Volume {
 				ConfigMap: &corev1.ConfigMapVolumeSource{
 					DefaultMode: &config0640AccessMode,
 					LocalObjectReference: corev1.LocalObjectReference{
-						Name: name + "-config-data",
+						Name: instance.Name + "-config-data",
+					},
+				},
+			},
+		},
+		{
+			Name: "mysql-config-data",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					DefaultMode: &config0640AccessMode,
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: "openstack-config-data",
 					},
 				},
 			},
@@ -91,6 +104,26 @@ func getVolumes(name string) []corev1.Volume {
 		},
 	}
 
+	if instance.Spec.TLS.CaSecretName != "" {
+		volumes = append(volumes, []corev1.Volume{
+			{
+				Name: "tls-ca",
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: instance.Spec.TLS.CaSecretName,
+						Items: []corev1.KeyToPath{
+							{
+								Key:  "ca.crt",
+								Path: "ca.crt",
+							},
+						},
+					},
+				},
+			},
+		}...)
+	}
+
+	return volumes
 }
 
 // getInitVolumeMounts - general init task VolumeMounts
@@ -115,8 +148,8 @@ func getInitVolumeMounts() []corev1.VolumeMount {
 }
 
 // getVolumeMounts - general VolumeMounts
-func getVolumeMounts() []corev1.VolumeMount {
-	return []corev1.VolumeMount{
+func getVolumeMounts(instance *keystonev1.KeystoneAPI) []corev1.VolumeMount {
+	volumeMounts := []corev1.VolumeMount{
 		{
 			Name:      "scripts",
 			MountPath: "/usr/local/bin/container-scripts",
@@ -143,5 +176,22 @@ func getVolumeMounts() []corev1.VolumeMount {
 			ReadOnly:  true,
 			Name:      "credential-keys",
 		},
+		{
+			Name:      "mysql-config-data",
+			MountPath: "/var/lib/mysql-config-data",
+			ReadOnly:  true,
+		},
+	}
+
+	if instance.Spec.TLS.CaSecretName != "" {
+		volumeMounts = append(volumeMounts, []corev1.VolumeMount{
+			{
+				MountPath: "/var/lib/tls-ca",
+				ReadOnly:  true,
+				Name:      "tls-ca",
+			},
+		}...)
 	}
+
+	return volumeMounts
 }