From 4ffcd00a5be7403b9a20c0ac658bd2a4975d1c90 Mon Sep 17 00:00:00 2001 From: Julien Guitton Date: Thu, 11 Jan 2024 15:52:38 +0100 Subject: [PATCH] Add boostrap-x.conf file management --- api/v1/nificluster_types.go | 60 +++ api/v1/zz_generated.deepcopy.go | 104 +++++ .../nifi.konpyutaika.com_nificlusters.yaml | 232 +++++++++++ ...pyutaika.com_nifinodegroupautoscalers.yaml | 116 ++++++ .../nifi.konpyutaika.com_nificlusters.yaml | 232 +++++++++++ ...pyutaika.com_nifinodegroupautoscalers.yaml | 116 ++++++ pkg/resources/nifi/secretconfig.go | 366 ++++++++++++++++++ .../templates/config/bootstrap_properties.go | 134 +++++++ 8 files changed, 1360 insertions(+) diff --git a/api/v1/nificluster_types.go b/api/v1/nificluster_types.go index 60e916eede..97880ae46d 100644 --- a/api/v1/nificluster_types.go +++ b/api/v1/nificluster_types.go @@ -230,6 +230,14 @@ type ReadOnlyConfig struct { AuthorizerConfig AuthorizerConfig `json:"authorizerConfig,omitempty"` // BootstrapNotificationServices configuration that will be applied to the node. BootstrapNotificationServicesReplaceConfig BootstrapNotificationServicesConfig `json:"bootstrapNotificationServicesConfig,omitempty"` + // BootstrapGCPProperties configuration that will be applied to the node. + BootstrapGCPProperties BootstrapGCPProperties `json:"bootstrapGCPProperties,omitempty"` + // BootstrapAWSProperties configuration that will be applied to the node. + BootstrapAWSProperties BootstrapAWSProperties `json:"bootstrapAWSProperties,omitempty"` + // BootstrapAzureProperties configuration that will be applied to the node. + BootstrapAzureProperties BootstrapAzureProperties `json:"bootstrapAzureProperties,omitempty"` + // BootstrapHashicorpVaultProperties configuration that will be applied to the node. + BootstrapHashicorpVaultProperties BootstrapHashicorpVaultProperties `json:"bootstrapHashicorpVaultProperties,omitempty"` } // Optional configuration for the default authorizers.xml template. @@ -305,6 +313,58 @@ type BootstrapNotificationServicesConfig struct { ReplaceSecretConfig *SecretConfigReference `json:"replaceSecretConfig,omitempty"` } +// BootstrapGCPProperties configuration that will be applied to the node. +type BootstrapGCPProperties struct { + // Additionnals bootstrap-gcp.conf configuration that will override the one produced based on template and + // configuration + OverrideConfigMap *ConfigmapReference `json:"overrideConfigMap,omitempty"` + // Additionnals bootstrap-gcp.conf configuration that will override the one produced based + // on template and configurations. + OverrideConfigs string `json:"overrideConfigs,omitempty"` + // Additionnals bootstrap-gcp.conf configuration that will override the one produced based + // on template, configurations, overrideConfigMap and overrideConfigs. + OverrideSecretConfig *SecretConfigReference `json:"overrideSecretConfig,omitempty"` +} + +// BootstrapAWSProperties configuration that will be applied to the node. +type BootstrapAWSProperties struct { + // Additionnals bootstrap-aws.conf configuration that will override the one produced based on template and + // configuration + OverrideConfigMap *ConfigmapReference `json:"overrideConfigMap,omitempty"` + // Additionnals bootstrap-aws.conf configuration that will override the one produced based + // on template and configurations. + OverrideConfigs string `json:"overrideConfigs,omitempty"` + // Additionnals bootstrap-aws.conf configuration that will override the one produced based + // on template, configurations, overrideConfigMap and overrideConfigs. + OverrideSecretConfig *SecretConfigReference `json:"overrideSecretConfig,omitempty"` +} + +// BootstrapAzureProperties configuration that will be applied to the node. +type BootstrapAzureProperties struct { + // Additionnals bootstrap-azure.conf configuration that will override the one produced based on template and + // configuration + OverrideConfigMap *ConfigmapReference `json:"overrideConfigMap,omitempty"` + // Additionnals bootstrap-azure.conf configuration that will override the one produced based + // on template and configurations. + OverrideConfigs string `json:"overrideConfigs,omitempty"` + // Additionnals bootstrap-azure.conf configuration that will override the one produced based + // on template, configurations, overrideConfigMap and overrideConfigs. + OverrideSecretConfig *SecretConfigReference `json:"overrideSecretConfig,omitempty"` +} + +// BootstrapHashicorpVaultProperties configuration that will be applied to the node. +type BootstrapHashicorpVaultProperties struct { + // Additionnals bootstrap-hashicorp-vault.conf configuration that will override the one produced based on template and + // configuration + OverrideConfigMap *ConfigmapReference `json:"overrideConfigMap,omitempty"` + // Additionnals bootstrap-hashicorp-vault.conf configuration that will override the one produced based + // on template and configurations. + OverrideConfigs string `json:"overrideConfigs,omitempty"` + // Additionnals bootstrap-hashicorp-vault.conf configuration that will override the one produced based + // on template, configurations, overrideConfigMap and overrideConfigs. + OverrideSecretConfig *SecretConfigReference `json:"overrideSecretConfig,omitempty"` +} + // NodeConfig defines the node configuration. type NodeConfig struct { // provenanceStorage allow to specify the maximum amount of data provenance information to store at a time diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 34dad12bdc..428f21bf4a 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -67,6 +67,106 @@ func (in *AuthorizerConfig) DeepCopy() *AuthorizerConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapAWSProperties) DeepCopyInto(out *BootstrapAWSProperties) { + *out = *in + if in.OverrideConfigMap != nil { + in, out := &in.OverrideConfigMap, &out.OverrideConfigMap + *out = new(ConfigmapReference) + **out = **in + } + if in.OverrideSecretConfig != nil { + in, out := &in.OverrideSecretConfig, &out.OverrideSecretConfig + *out = new(SecretConfigReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapAWSProperties. +func (in *BootstrapAWSProperties) DeepCopy() *BootstrapAWSProperties { + if in == nil { + return nil + } + out := new(BootstrapAWSProperties) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapAzureProperties) DeepCopyInto(out *BootstrapAzureProperties) { + *out = *in + if in.OverrideConfigMap != nil { + in, out := &in.OverrideConfigMap, &out.OverrideConfigMap + *out = new(ConfigmapReference) + **out = **in + } + if in.OverrideSecretConfig != nil { + in, out := &in.OverrideSecretConfig, &out.OverrideSecretConfig + *out = new(SecretConfigReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapAzureProperties. +func (in *BootstrapAzureProperties) DeepCopy() *BootstrapAzureProperties { + if in == nil { + return nil + } + out := new(BootstrapAzureProperties) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapGCPProperties) DeepCopyInto(out *BootstrapGCPProperties) { + *out = *in + if in.OverrideConfigMap != nil { + in, out := &in.OverrideConfigMap, &out.OverrideConfigMap + *out = new(ConfigmapReference) + **out = **in + } + if in.OverrideSecretConfig != nil { + in, out := &in.OverrideSecretConfig, &out.OverrideSecretConfig + *out = new(SecretConfigReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapGCPProperties. +func (in *BootstrapGCPProperties) DeepCopy() *BootstrapGCPProperties { + if in == nil { + return nil + } + out := new(BootstrapGCPProperties) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapHashicorpVaultProperties) DeepCopyInto(out *BootstrapHashicorpVaultProperties) { + *out = *in + if in.OverrideConfigMap != nil { + in, out := &in.OverrideConfigMap, &out.OverrideConfigMap + *out = new(ConfigmapReference) + **out = **in + } + if in.OverrideSecretConfig != nil { + in, out := &in.OverrideSecretConfig, &out.OverrideSecretConfig + *out = new(SecretConfigReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapHashicorpVaultProperties. +func (in *BootstrapHashicorpVaultProperties) DeepCopy() *BootstrapHashicorpVaultProperties { + if in == nil { + return nil + } + out := new(BootstrapHashicorpVaultProperties) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapNotificationServicesConfig) DeepCopyInto(out *BootstrapNotificationServicesConfig) { *out = *in @@ -1488,6 +1588,10 @@ func (in *ReadOnlyConfig) DeepCopyInto(out *ReadOnlyConfig) { in.LogbackConfig.DeepCopyInto(&out.LogbackConfig) in.AuthorizerConfig.DeepCopyInto(&out.AuthorizerConfig) in.BootstrapNotificationServicesReplaceConfig.DeepCopyInto(&out.BootstrapNotificationServicesReplaceConfig) + in.BootstrapGCPProperties.DeepCopyInto(&out.BootstrapGCPProperties) + in.BootstrapAWSProperties.DeepCopyInto(&out.BootstrapAWSProperties) + in.BootstrapAzureProperties.DeepCopyInto(&out.BootstrapAzureProperties) + in.BootstrapHashicorpVaultProperties.DeepCopyInto(&out.BootstrapHashicorpVaultProperties) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReadOnlyConfig. diff --git a/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml b/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml index 58bf498acc..c4f945a498 100644 --- a/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml +++ b/config/crd/bases/nifi.konpyutaika.com_nificlusters.yaml @@ -3035,6 +3035,122 @@ spec: - name type: object type: object + bootstrapAWSProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapAzureProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapGCPProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapHashicorpVaultProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object bootstrapNotificationServicesConfig: properties: replaceConfigMap: @@ -3474,6 +3590,122 @@ spec: - name type: object type: object + bootstrapAWSProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapAzureProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapGCPProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapHashicorpVaultProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object bootstrapNotificationServicesConfig: properties: replaceConfigMap: diff --git a/config/crd/bases/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml b/config/crd/bases/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml index e93a2e9832..27277c6eeb 100644 --- a/config/crd/bases/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml +++ b/config/crd/bases/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml @@ -1209,6 +1209,122 @@ spec: - name type: object type: object + bootstrapAWSProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapAzureProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapGCPProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapHashicorpVaultProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object bootstrapNotificationServicesConfig: properties: replaceConfigMap: diff --git a/helm/nifikop/crds/nifi.konpyutaika.com_nificlusters.yaml b/helm/nifikop/crds/nifi.konpyutaika.com_nificlusters.yaml index 58bf498acc..c4f945a498 100644 --- a/helm/nifikop/crds/nifi.konpyutaika.com_nificlusters.yaml +++ b/helm/nifikop/crds/nifi.konpyutaika.com_nificlusters.yaml @@ -3035,6 +3035,122 @@ spec: - name type: object type: object + bootstrapAWSProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapAzureProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapGCPProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapHashicorpVaultProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object bootstrapNotificationServicesConfig: properties: replaceConfigMap: @@ -3474,6 +3590,122 @@ spec: - name type: object type: object + bootstrapAWSProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapAzureProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapGCPProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapHashicorpVaultProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object bootstrapNotificationServicesConfig: properties: replaceConfigMap: diff --git a/helm/nifikop/crds/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml b/helm/nifikop/crds/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml index e93a2e9832..27277c6eeb 100644 --- a/helm/nifikop/crds/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml +++ b/helm/nifikop/crds/nifi.konpyutaika.com_nifinodegroupautoscalers.yaml @@ -1209,6 +1209,122 @@ spec: - name type: object type: object + bootstrapAWSProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapAzureProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapGCPProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object + bootstrapHashicorpVaultProperties: + properties: + overrideConfigMap: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + overrideConfigs: + type: string + overrideSecretConfig: + properties: + data: + type: string + name: + type: string + namespace: + type: string + required: + - data + - name + type: object + type: object bootstrapNotificationServicesConfig: properties: replaceConfigMap: diff --git a/pkg/resources/nifi/secretconfig.go b/pkg/resources/nifi/secretconfig.go index 53fcb4fdc0..4468392b49 100644 --- a/pkg/resources/nifi/secretconfig.go +++ b/pkg/resources/nifi/secretconfig.go @@ -52,6 +52,20 @@ func (r *Reconciler) secretConfig(id int32, nodeConfig *v1.NodeConfig, serverPas if configcommon.UseSSL(r.NifiCluster) { secret.Data["authorizers.xml"] = []byte(r.getAuthorizersConfigString(nodeConfig, id, log)) } + + if boostrapGCPPropertiesNodeConfig := r.generateBootstrapGCPPropertiesNodeConfig(id, nodeConfig, log); boostrapGCPPropertiesNodeConfig != nil { + secret.Data["bootstrap-gcp.conf"] = []byte(*boostrapGCPPropertiesNodeConfig) + } + if boostrapGCPPropertiesNodeConfig := r.generateBootstrapAWSPropertiesNodeConfig(id, nodeConfig, log); boostrapGCPPropertiesNodeConfig != nil { + secret.Data["bootstrap-aws.conf"] = []byte(*boostrapGCPPropertiesNodeConfig) + } + if boostrapGCPPropertiesNodeConfig := r.generateBootstrapAzurePropertiesNodeConfig(id, nodeConfig, log); boostrapGCPPropertiesNodeConfig != nil { + secret.Data["bootstrap-azure.conf"] = []byte(*boostrapGCPPropertiesNodeConfig) + } + if boostrapGCPPropertiesNodeConfig := r.generateBootstrapHashicorpVaultPropertiesNodeConfig(id, nodeConfig, log); boostrapGCPPropertiesNodeConfig != nil { + secret.Data["bootstrap-hashicorp-vault.conf"] = []byte(*boostrapGCPPropertiesNodeConfig) + } + return secret } @@ -699,3 +713,355 @@ func (r Reconciler) generateReadOnlyConfig( zap.Error(err)) } } + +////////////////////////////////////////////// +// Bootstrap GCP properties configuration // +////////////////////////////////////////////// + +func (r Reconciler) generateBootstrapGCPPropertiesNodeConfig(id int32, nodeConfig *v1.NodeConfig, log zap.Logger) *string { + var readOnlyClusterConfig map[string]string + + if &r.NifiCluster.Spec.ReadOnlyConfig != (&v1.ReadOnlyConfig{}) && &r.NifiCluster.Spec.ReadOnlyConfig.BootstrapGCPProperties != (&v1.BootstrapGCPProperties{}) { + r.generateReadOnlyConfig( + &readOnlyClusterConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapGCPProperties.OverrideSecretConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapGCPProperties.OverrideConfigMap, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapGCPProperties.OverrideConfigs, log) + } + + var readOnlyNodeConfig = map[string]string{} + + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapGCPProperties != (&v1.BootstrapGCPProperties{}) { + r.generateReadOnlyConfig( + &readOnlyNodeConfig, + node.ReadOnlyConfig.BootstrapGCPProperties.OverrideSecretConfig, + node.ReadOnlyConfig.BootstrapGCPProperties.OverrideConfigMap, + node.ReadOnlyConfig.BootstrapGCPProperties.OverrideConfigs, log) + break + } + } + + if err := mergo.Merge(&readOnlyNodeConfig, readOnlyClusterConfig); err != nil { + log.Error("error occurred during merging readonly configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + // Generate the Complete Configuration for the Node + completeConfigMap := map[string]string{} + + if err := mergo.Merge(&completeConfigMap, readOnlyNodeConfig); err != nil { + log.Error("error occurred during merging readOnly config to complete configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + if len(completeConfigMap) == 0 { + return nil + } + + if err := mergo.Merge(&completeConfigMap, util.ParsePropertiesFormat(r.getBootstrapGCPPropertiesConfigString(nodeConfig, id, log))); err != nil { + log.Error("error occurred during merging operator generated configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + completeConfig := []string{} + + for key, value := range completeConfigMap { + completeConfig = append(completeConfig, fmt.Sprintf("%s=%s", key, value)) + } + + // We need to sort the config every time to avoid diffs occurred because of ranging through map + sort.Strings(completeConfig) + + output := strings.Join(completeConfig, "\n") + return &output +} + +func (r *Reconciler) getBootstrapGCPPropertiesConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string { + base := r.NifiCluster.Spec.ReadOnlyConfig.BootstrapGCPProperties.DeepCopy() + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapGCPProperties != (&v1.BootstrapGCPProperties{}) { + mergo.Merge(base, node.ReadOnlyConfig.BootstrapGCPProperties, mergo.WithOverride) + } + } + + var out bytes.Buffer + t := template.Must(template.New("nConfig-config").Parse(config.BootstrapGCPPropertiesTemplate)) + if err := t.Execute(&out, map[string]interface{}{}); err != nil { + log.Error("error occurred during parsing the config template", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + return out.String() +} + +////////////////////////////////////////////// +// Bootstrap AWS properties configuration // +////////////////////////////////////////////// + +func (r Reconciler) generateBootstrapAWSPropertiesNodeConfig(id int32, nodeConfig *v1.NodeConfig, log zap.Logger) *string { + var readOnlyClusterConfig map[string]string + + if &r.NifiCluster.Spec.ReadOnlyConfig != (&v1.ReadOnlyConfig{}) && &r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAWSProperties != (&v1.BootstrapAWSProperties{}) { + r.generateReadOnlyConfig( + &readOnlyClusterConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAWSProperties.OverrideSecretConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAWSProperties.OverrideConfigMap, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAWSProperties.OverrideConfigs, log) + } + + var readOnlyNodeConfig = map[string]string{} + + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapAWSProperties != (&v1.BootstrapAWSProperties{}) { + r.generateReadOnlyConfig( + &readOnlyNodeConfig, + node.ReadOnlyConfig.BootstrapAWSProperties.OverrideSecretConfig, + node.ReadOnlyConfig.BootstrapAWSProperties.OverrideConfigMap, + node.ReadOnlyConfig.BootstrapAWSProperties.OverrideConfigs, log) + break + } + } + + if err := mergo.Merge(&readOnlyNodeConfig, readOnlyClusterConfig); err != nil { + log.Error("error occurred during merging readonly configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + // Generate the Complete Configuration for the Node + completeConfigMap := map[string]string{} + + if err := mergo.Merge(&completeConfigMap, readOnlyNodeConfig); err != nil { + log.Error("error occurred during merging readOnly config to complete configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + if len(completeConfigMap) == 0 { + return nil + } + + if err := mergo.Merge(&completeConfigMap, util.ParsePropertiesFormat(r.getBootstrapAWSPropertiesConfigString(nodeConfig, id, log))); err != nil { + log.Error("error occurred during merging operator generated configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + completeConfig := []string{} + + for key, value := range completeConfigMap { + completeConfig = append(completeConfig, fmt.Sprintf("%s=%s", key, value)) + } + + // We need to sort the config every time to avoid diffs occurred because of ranging through map + sort.Strings(completeConfig) + + output := strings.Join(completeConfig, "\n") + return &output +} + +func (r *Reconciler) getBootstrapAWSPropertiesConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string { + base := r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAWSProperties.DeepCopy() + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapAWSProperties != (&v1.BootstrapAWSProperties{}) { + mergo.Merge(base, node.ReadOnlyConfig.BootstrapAWSProperties, mergo.WithOverride) + } + } + + var out bytes.Buffer + t := template.Must(template.New("nConfig-config").Parse(config.BootstrapAWSPropertiesTemplate)) + if err := t.Execute(&out, map[string]interface{}{}); err != nil { + log.Error("error occurred during parsing the config template", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + return out.String() +} + +//////////////////////////////////////////////// +// Bootstrap Azure properties configuration // +//////////////////////////////////////////////// + +func (r Reconciler) generateBootstrapAzurePropertiesNodeConfig(id int32, nodeConfig *v1.NodeConfig, log zap.Logger) *string { + var readOnlyClusterConfig map[string]string + + if &r.NifiCluster.Spec.ReadOnlyConfig != (&v1.ReadOnlyConfig{}) && &r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAzureProperties != (&v1.BootstrapAzureProperties{}) { + r.generateReadOnlyConfig( + &readOnlyClusterConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAzureProperties.OverrideSecretConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAzureProperties.OverrideConfigMap, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAzureProperties.OverrideConfigs, log) + } + + var readOnlyNodeConfig = map[string]string{} + + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapAzureProperties != (&v1.BootstrapAzureProperties{}) { + r.generateReadOnlyConfig( + &readOnlyNodeConfig, + node.ReadOnlyConfig.BootstrapAzureProperties.OverrideSecretConfig, + node.ReadOnlyConfig.BootstrapAzureProperties.OverrideConfigMap, + node.ReadOnlyConfig.BootstrapAzureProperties.OverrideConfigs, log) + break + } + } + + if err := mergo.Merge(&readOnlyNodeConfig, readOnlyClusterConfig); err != nil { + log.Error("error occurred during merging readonly configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + // Generate the Complete Configuration for the Node + completeConfigMap := map[string]string{} + + if err := mergo.Merge(&completeConfigMap, readOnlyNodeConfig); err != nil { + log.Error("error occurred during merging readOnly config to complete configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + if len(completeConfigMap) == 0 { + return nil + } + + if err := mergo.Merge(&completeConfigMap, util.ParsePropertiesFormat(r.getBootstrapAzurePropertiesConfigString(nodeConfig, id, log))); err != nil { + log.Error("error occurred during merging operator generated configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + completeConfig := []string{} + + for key, value := range completeConfigMap { + completeConfig = append(completeConfig, fmt.Sprintf("%s=%s", key, value)) + } + + // We need to sort the config every time to avoid diffs occurred because of ranging through map + sort.Strings(completeConfig) + + output := strings.Join(completeConfig, "\n") + return &output +} + +func (r *Reconciler) getBootstrapAzurePropertiesConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string { + base := r.NifiCluster.Spec.ReadOnlyConfig.BootstrapAzureProperties.DeepCopy() + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapAzureProperties != (&v1.BootstrapAzureProperties{}) { + mergo.Merge(base, node.ReadOnlyConfig.BootstrapAzureProperties, mergo.WithOverride) + } + } + + var out bytes.Buffer + t := template.Must(template.New("nConfig-config").Parse(config.BootstrapAzurePropertiesTemplate)) + if err := t.Execute(&out, map[string]interface{}{}); err != nil { + log.Error("error occurred during parsing the config template", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + return out.String() +} + +///////////////////////////////////////////////////////// +// Bootstrap HashicorpVault properties configuration // +///////////////////////////////////////////////////////// + +func (r Reconciler) generateBootstrapHashicorpVaultPropertiesNodeConfig(id int32, nodeConfig *v1.NodeConfig, log zap.Logger) *string { + var readOnlyClusterConfig map[string]string + + if &r.NifiCluster.Spec.ReadOnlyConfig != (&v1.ReadOnlyConfig{}) && &r.NifiCluster.Spec.ReadOnlyConfig.BootstrapHashicorpVaultProperties != (&v1.BootstrapHashicorpVaultProperties{}) { + r.generateReadOnlyConfig( + &readOnlyClusterConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapHashicorpVaultProperties.OverrideSecretConfig, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapHashicorpVaultProperties.OverrideConfigMap, + r.NifiCluster.Spec.ReadOnlyConfig.BootstrapHashicorpVaultProperties.OverrideConfigs, log) + } + + var readOnlyNodeConfig = map[string]string{} + + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapHashicorpVaultProperties != (&v1.BootstrapHashicorpVaultProperties{}) { + r.generateReadOnlyConfig( + &readOnlyNodeConfig, + node.ReadOnlyConfig.BootstrapHashicorpVaultProperties.OverrideSecretConfig, + node.ReadOnlyConfig.BootstrapHashicorpVaultProperties.OverrideConfigMap, + node.ReadOnlyConfig.BootstrapHashicorpVaultProperties.OverrideConfigs, log) + break + } + } + + if err := mergo.Merge(&readOnlyNodeConfig, readOnlyClusterConfig); err != nil { + log.Error("error occurred during merging readonly configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + // Generate the Complete Configuration for the Node + completeConfigMap := map[string]string{} + + if err := mergo.Merge(&completeConfigMap, readOnlyNodeConfig); err != nil { + log.Error("error occurred during merging readOnly config to complete configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + if len(completeConfigMap) == 0 { + return nil + } + + if err := mergo.Merge(&completeConfigMap, util.ParsePropertiesFormat(r.getBootstrapHashicorpVaultPropertiesConfigString(nodeConfig, id, log))); err != nil { + log.Error("error occurred during merging operator generated configs", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + + completeConfig := []string{} + + for key, value := range completeConfigMap { + completeConfig = append(completeConfig, fmt.Sprintf("%s=%s", key, value)) + } + + // We need to sort the config every time to avoid diffs occurred because of ranging through map + sort.Strings(completeConfig) + + output := strings.Join(completeConfig, "\n") + return &output +} + +func (r *Reconciler) getBootstrapHashicorpVaultPropertiesConfigString(nConfig *v1.NodeConfig, id int32, log zap.Logger) string { + base := r.NifiCluster.Spec.ReadOnlyConfig.BootstrapHashicorpVaultProperties.DeepCopy() + for _, node := range r.NifiCluster.Spec.Nodes { + if node.Id == id && node.ReadOnlyConfig != nil && &node.ReadOnlyConfig.BootstrapHashicorpVaultProperties != (&v1.BootstrapHashicorpVaultProperties{}) { + mergo.Merge(base, node.ReadOnlyConfig.BootstrapHashicorpVaultProperties, mergo.WithOverride) + } + } + + var out bytes.Buffer + t := template.Must(template.New("nConfig-config").Parse(config.BootstrapHashicorpVaultPropertiesTemplate)) + if err := t.Execute(&out, map[string]interface{}{}); err != nil { + log.Error("error occurred during parsing the config template", + zap.String("clusterName", r.NifiCluster.Name), + zap.Int32("nodeId", id), + zap.Error(err)) + } + return out.String() +} diff --git a/pkg/resources/templates/config/bootstrap_properties.go b/pkg/resources/templates/config/bootstrap_properties.go index 1717cf94f0..c2fa75148a 100644 --- a/pkg/resources/templates/config/bootstrap_properties.go +++ b/pkg/resources/templates/config/bootstrap_properties.go @@ -78,3 +78,137 @@ notification.max.attempts=5 # Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? #nifi.dead.notification.services=email-notification ` + +var BootstrapGCPPropertiesTemplate = `# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# These GCP KMS settings must all be configured in order to use the GCP KMS Sensitive Property Provider +gcp.kms.project= +gcp.kms.location= +gcp.kms.keyring= +gcp.kms.key= +` + +var BootstrapAWSPropertiesTemplate = `# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# AWS KMS Key ID is required to be configured for AWS KMS Sensitive Property Provider +aws.kms.key.id= + +# NiFi uses the following properties when authentication to AWS when all values are provided. +# NiFi uses the default AWS credentials provider chain when one or more or the following properties are blank +# AWS SDK documentation describes the default credential retrieval order: +# https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html#credentials-chain +aws.access.key.id= +aws.secret.access.key= +aws.region= +` + +var BootstrapAzurePropertiesTemplate = `# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Key Identifier for Azure Key Vault Key Sensitive Property Provider +azure.keyvault.key.id= +# Encryption Algorithm for Azure Key Vault Key Sensitive Property Provider +azure.keyvault.encryption.algorithm= + +# Vault URI for Azure Key Vault Secret Sensitive Property Provider +azure.keyvault.uri= +` + +var BootstrapHashicorpVaultPropertiesTemplate = `# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive Properties Provider +vault.uri= + +# Transit Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/transit/{path}' +vault.transit.path= + +# Key/Value Path is required to enable the Sensitive Properties Provider Protection Scheme 'hashicorp/vault/kv/{path}' +vault.kv.path= +# Key/Value Secrets Engine version may be 1 or 2, and defaults to 1 +# vault.kv.version=1 + +# Token Authentication example properties +# vault.authentication=TOKEN +# vault.token= + +# Optional file supports authentication properties described in the Spring Vault Environment Configuration +# https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration +# +# All authentication properties must be included in bootstrap-hashicorp-vault.conf when this property is not specified. +# Properties in bootstrap-hashicorp-vault.conf take precedence when the same values are defined in both files. +# Token Authentication is the default when the 'vault.authentication' property is not specified. +vault.authentication.properties.file= + +# Optional Timeout properties +vault.connection.timeout=5 secs +vault.read.timeout=15 secs + +# Optional TLS properties +vault.ssl.enabledCipherSuites= +vault.ssl.enabledProtocols= +vault.ssl.key-store= +vault.ssl.key-store-type= +vault.ssl.key-store-password= +vault.ssl.trust-store= +vault.ssl.trust-store-type= +vault.ssl.trust-store-password= +`