Merge pull request #909 from slagle/service-maps

Add DataSources to OpenStackDataPlaneService

api/bases/dataplane.openstack.org_openstackdataplaneservices.yaml

@@ -46,6 +46,29 @@ spec:
                 items:
                   type: string
                 type: array
+              datasources:
+                items:
+                  properties:
+                    configMapRef:
+                      properties:
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    prefix:
+                      type: string
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                  type: object
+                type: array
               deployOnAllNodeSets:
                 type: boolean
               edpmServiceType:

api/v1beta1/common.go

@@ -24,8 +24,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-// AnsibleVarsFromSource represents the source of a set of ConfigMaps/Secrets
-type AnsibleVarsFromSource struct {
+// DataSource represents the source of a set of ConfigMaps/Secrets
+type DataSource struct {
 	// An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.
 	// +optional
 	Prefix string `json:"prefix,omitempty" protobuf:"bytes,1,opt,name=prefix"`
@@ -55,7 +55,7 @@ type AnsibleOpts struct {
 	// AnsibleVarsFrom is a list of sources to populate ansible variables from.
 	// Values defined by an AnsibleVars with a duplicate key take precedence.
 	// +kubebuilder:validation:Optional
-	AnsibleVarsFrom []AnsibleVarsFromSource `json:"ansibleVarsFrom,omitempty"`
+	AnsibleVarsFrom []DataSource `json:"ansibleVarsFrom,omitempty"`
 
 	// AnsiblePort SSH port for Ansible connection
 	// +kubebuilder:validation:Optional

api/v1beta1/openstackdataplaneservice_types.go

@@ -66,6 +66,10 @@ type OpenStackDataPlaneServiceSpec struct {
 	// +kubebuilder:validation:Optional
 	Secrets []string `json:"secrets,omitempty"`
 
+	// DataSources list of DataSource objects to mount as ExtraMounts for the
+	// OpenStackAnsibleEE
+	DataSources []DataSource `json:"datasources,omitempty"`
+
 	// TLSCert tls certs to be generated
 	// +kubebuilder:validation:Optional
 	TLSCert *OpenstackDataPlaneServiceCert `json:"tlsCert,omitempty" yaml:"tlsCert,omitempty"`

config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml

@@ -46,6 +46,29 @@ spec:
                 items:
                   type: string
                 type: array
+              datasources:
+                items:
+                  properties:
+                    configMapRef:
+                      properties:
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    prefix:
+                      type: string
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                  type: object
+                type: array
               deployOnAllNodeSets:
                 type: boolean
               edpmServiceType:

controllers/openstackdataplanenodeset_controller.go

@@ -539,7 +539,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(mgr ctrl.Manager)
 			nodeSet := rawObj.(*dataplanev1.OpenStackDataPlaneNodeSet)
 			configMaps := make([]string, 0)
 
-			appendConfigMaps := func(varsFrom []dataplanev1.AnsibleVarsFromSource) {
+			appendConfigMaps := func(varsFrom []dataplanev1.DataSource) {
 				for _, ref := range varsFrom {
 					if ref.ConfigMapRef != nil {
 						configMaps = append(configMaps, ref.ConfigMapRef.Name)
@@ -566,7 +566,7 @@ func (r *OpenStackDataPlaneNodeSetReconciler) SetupWithManager(mgr ctrl.Manager)
 				secrets = append(secrets, nodeSet.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret)
 			}
 
-			appendSecrets := func(varsFrom []dataplanev1.AnsibleVarsFromSource) {
+			appendSecrets := func(varsFrom []dataplanev1.DataSource) {
 				for _, ref := range varsFrom {
 					if ref.SecretRef != nil {
 						secrets = append(secrets, ref.SecretRef.Name)

docs/assemblies/custom_resources.adoc

@@ -10,7 +10,7 @@
 
 * <<ansibleeespec,AnsibleEESpec>>
 * <<ansibleopts,AnsibleOpts>>
-* <<ansiblevarsfromsource,AnsibleVarsFromSource>>
+* <<datasource,DataSource>>
 * <<nodesection,NodeSection>>
 * <<nodetemplate,NodeTemplate>>
 * <<openstackdataplaneservicelist,OpenStackDataPlaneServiceList>>
@@ -110,7 +110,7 @@ AnsibleOpts defines a logical grouping of Ansible related configuration options.
 
 | ansibleVarsFrom
 | AnsibleVarsFrom is a list of sources to populate ansible variables from. Values defined by an AnsibleVars with a duplicate key take precedence.
-| []<<ansiblevarsfromsource,AnsibleVarsFromSource>>
+| []<<datasource,DataSource>>
 | false
 
 | ansiblePort
@@ -121,10 +121,10 @@ AnsibleOpts defines a logical grouping of Ansible related configuration options.
 
 <<custom-resources,Back to Custom Resources>>
 
-[#ansiblevarsfromsource]
-==== AnsibleVarsFromSource
+[#datasource]
+==== DataSource
 
-AnsibleVarsFromSource represents the source of a set of ConfigMaps/Secrets
+DataSource represents the source of a set of ConfigMaps/Secrets
 
 |===
 | Field | Description | Scheme | Required
@@ -309,6 +309,11 @@ OpenStackDataPlaneServiceSpec defines the desired state of OpenStackDataPlaneSer
 | []string
 | false
 
+| datasources
+| DataSources list of DataSource objects to mount as ExtraMounts for the OpenStackAnsibleEE
+| []<<datasource,DataSource>>
+| false
+
 | tlsCert
 | TLSCert tls certs to be generated
 | *<<openstackdataplaneservicecert,OpenstackDataPlaneServiceCert>>

docs/assemblies/proc_creating-a-custom-service.adoc

@@ -68,7 +68,7 @@ spec:
 
 . Optional: Designate and configure a node set for a Compute feature or workload. For more information, see xref:proc_configuring-a-node-set-for-a-Compute-feature-or-workload_dataplane[Configuring a node set for a Compute feature or workload].
 
-. Optional: Specify the names of `Secret` resources to use to pass secrets into the `OpenStackAnsibleEE` job:
+. Optional: Specify `Secret` resources to use to pass secrets into the `OpenStackAnsibleEE` job. Secrets are specified with a `name` and `required` field. When `required` is false, the service deployment will not fail if the secret doesn't exist.
 +
 ----
 apiVersion: dataplane.openstack.org/v1beta1
@@ -80,8 +80,10 @@ spec:
   play: |
     ...
   secrets:
-    - hello-world-secret-0
-    - hello-world-secret-1
+    - name: hello-world-secret-0
+      required: true
+    - name: hello-world-secret-1
+      required: false
 ----
 +
 A mount is created for each `secret` in the `OpenStackAnsibleEE` pod with a filename that matches the `secret` value. The mounts are created under `/var/lib/openstack/configs/<service name>`.

pkg/deployment/deployment.go

@@ -39,6 +39,7 @@ import (
 	ansibleeev1 "github.com/openstack-k8s-operators/openstack-ansibleee-operator/api/v1beta1"
 	openstackv1 "github.com/openstack-k8s-operators/openstack-operator/apis/core/v1beta1"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 )
 
 // Deployer defines a data structure with all of the relevant objects required for a full deployment.
@@ -361,14 +362,44 @@ func (d *Deployer) addServiceExtraMounts(
 	client := d.Helper.GetClient()
 	baseMountPath := path.Join(ConfigPaths, service.Name)
 
-	for _, cmName := range service.Spec.ConfigMaps {
+	var configMaps []*v1.ConfigMap
+	var secrets []*v1.Secret
 
-		volMounts := storage.VolMounts{}
+	for _, dataSource := range service.Spec.DataSources {
+		_cm, _secret, err := dataplaneutil.GetDataSourceCmSecret(d.Ctx, d.Helper, service.Namespace, dataSource)
+		if err != nil {
+			return nil, err
+		}
+
+		if _cm != nil {
+			configMaps = append(configMaps, _cm)
+		}
+		if _secret != nil {
+			secrets = append(secrets, _secret)
+		}
+	}
+
+	for _, cmName := range service.Spec.ConfigMaps {
 		cm := &corev1.ConfigMap{}
 		err := client.Get(d.Ctx, types.NamespacedName{Name: cmName, Namespace: service.Namespace}, cm)
 		if err != nil {
 			return d.AeeSpec, err
 		}
+		configMaps = append(configMaps, cm)
+	}
+
+	for _, secretName := range service.Spec.Secrets {
+		sec := &corev1.Secret{}
+		err := client.Get(d.Ctx, types.NamespacedName{Name: secretName, Namespace: service.Namespace}, sec)
+		if err != nil {
+			return d.AeeSpec, err
+		}
+		secrets = append(secrets, sec)
+	}
+
+	for _, cm := range configMaps {
+
+		volMounts := storage.VolMounts{}
 
 		keys := []string{}
 		for key := range cm.Data {
@@ -380,13 +411,13 @@ func (d *Deployer) addServiceExtraMounts(
 		sort.Strings(keys)
 
 		for idx, key := range keys {
-			name := fmt.Sprintf("%s-%s", cmName, strconv.Itoa(idx))
+			name := fmt.Sprintf("%s-%s", cm.Name, strconv.Itoa(idx))
 			volume := corev1.Volume{
 				Name: name,
 				VolumeSource: corev1.VolumeSource{
 					ConfigMap: &corev1.ConfigMapVolumeSource{
 						LocalObjectReference: corev1.LocalObjectReference{
-							Name: cmName,
+							Name: cm.Name,
 						},
 						Items: []corev1.KeyToPath{
 							{
@@ -412,28 +443,22 @@ func (d *Deployer) addServiceExtraMounts(
 		d.AeeSpec.ExtraMounts = append(d.AeeSpec.ExtraMounts, volMounts)
 	}
 
-	for _, secretName := range service.Spec.Secrets {
+	for _, sec := range secrets {
 
 		volMounts := storage.VolMounts{}
-		sec := &corev1.Secret{}
-		err := client.Get(d.Ctx, types.NamespacedName{Name: secretName, Namespace: service.Namespace}, sec)
-		if err != nil {
-			return d.AeeSpec, err
-		}
-
 		keys := []string{}
 		for key := range sec.Data {
 			keys = append(keys, key)
 		}
 		sort.Strings(keys)
 
 		for idx, key := range keys {
-			name := fmt.Sprintf("%s-%s", secretName, strconv.Itoa(idx))
+			name := fmt.Sprintf("%s-%s", sec.Name, strconv.Itoa(idx))
 			volume := corev1.Volume{
 				Name: name,
 				VolumeSource: corev1.VolumeSource{
 					Secret: &corev1.SecretVolumeSource{
-						SecretName: secretName,
+						SecretName: sec.Name,
 						Items: []corev1.KeyToPath{
 							{
 								Key:  key,
@@ -457,5 +482,6 @@ func (d *Deployer) addServiceExtraMounts(
 
 		d.AeeSpec.ExtraMounts = append(d.AeeSpec.ExtraMounts, volMounts)
 	}
+
 	return d.AeeSpec, nil
 }

pkg/deployment/hashes.go

@@ -20,6 +20,7 @@ import (
 	"context"
 
 	dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1"
+	dataplaneutil "github.com/openstack-k8s-operators/dataplane-operator/pkg/util"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
@@ -48,6 +49,27 @@ func GetDeploymentHashesForService(
 		helper.GetLogger().Error(err, "Unable to retrieve OpenStackDataPlaneService %v")
 		return err
 	}
+
+	for _, dataSource := range service.Spec.DataSources {
+		cm, sec, err := dataplaneutil.GetDataSourceCmSecret(ctx, helper, namespace, dataSource)
+		if err != nil {
+			return err
+		}
+
+		if cm != nil {
+			configMapHashes[cm.Name], err = configmap.Hash(cm)
+			if err != nil {
+				helper.GetLogger().Error(err, "Unable to hash ConfigMap %v")
+			}
+		}
+		if sec != nil {
+			secretHashes[sec.Name], err = secret.Hash(sec)
+			if err != nil {
+				helper.GetLogger().Error(err, "Unable to hash Secret %v")
+			}
+		}
+	}
+
 	for _, cmName := range service.Spec.ConfigMaps {
 		namespacedName := types.NamespacedName{
 			Name:      cmName,