From fe5ef441b2a2841c850a4d41d8685bc5fb8a1a06 Mon Sep 17 00:00:00 2001 From: Anthony TREUILLIER Date: Fri, 8 Dec 2023 08:43:49 +0100 Subject: [PATCH] fix: Add SSL/TLS settings feature for fluentd output Elasticsearch #418 Signed-off-by: Anthony TREUILLIER --- apis/fluentd/v1alpha1/plugins/output/es.go | 10 +++ apis/fluentd/v1alpha1/plugins/output/types.go | 24 +++++ .../fluentd.fluent.io_clusteroutputs.yaml | 45 ++++++++++ .../crds/fluentd.fluent.io_outputs.yaml | 45 ++++++++++ .../fluentd.fluent.io_clusteroutputs.yaml | 45 ++++++++++ .../crd/bases/fluentd.fluent.io_outputs.yaml | 45 ++++++++++ docs/plugins/fluentd/output/es.md | 5 ++ manifests/setup/fluent-operator-crd.yaml | 90 +++++++++++++++++++ manifests/setup/setup.yaml | 90 +++++++++++++++++++ 9 files changed, 399 insertions(+) diff --git a/apis/fluentd/v1alpha1/plugins/output/es.go b/apis/fluentd/v1alpha1/plugins/output/es.go index a7990993e..369a8f150 100644 --- a/apis/fluentd/v1alpha1/plugins/output/es.go +++ b/apis/fluentd/v1alpha1/plugins/output/es.go @@ -26,4 +26,14 @@ type Elasticsearch struct { User *plugins.Secret `json:"user,omitempty"` // Optional, The login credentials to connect to Elasticsearch Password *plugins.Secret `json:"password,omitempty"` + // Optional, Force certificate validation + SslVerify *bool `json:"sslVerify,omitempty"` + // Optional, Absolute path to CA certificate file + CAFile *string `json:"caFile,omitempty"` + // Optional, Absolute path to client Certificate file + ClientCert *string `json:"clientCert,omitempty"` + // Optional, Absolute path to client private Key file + ClientKey *string `json:"clientKey,omitempty"` + // Optional, password for ClientKey file + ClientKeyPassword *plugins.Secret `json:"clientKeyPassword,omitempty"` } diff --git a/apis/fluentd/v1alpha1/plugins/output/types.go b/apis/fluentd/v1alpha1/plugins/output/types.go index 1cf579b32..fd0b25e0a 100644 --- a/apis/fluentd/v1alpha1/plugins/output/types.go +++ b/apis/fluentd/v1alpha1/plugins/output/types.go @@ -405,6 +405,30 @@ func (o *Output) elasticsearchPlugin(parent *params.PluginStore, loader plugins. parent.InsertPairs("password", pwd) } + if o.Elasticsearch.SslVerify != nil { + parent.InsertPairs("ssl_verify", fmt.Sprint(*o.Elasticsearch.SslVerify)) + } + + if o.Elasticsearch.CAFile != nil { + parent.InsertPairs("ca_file", fmt.Sprint(*o.Elasticsearch.CAFile)) + } + + if o.Elasticsearch.ClientCert != nil { + parent.InsertPairs("client_cert", fmt.Sprint(*o.Elasticsearch.ClientCert)) + } + + if o.Elasticsearch.ClientKey != nil { + parent.InsertPairs("client_key", fmt.Sprint(*o.Elasticsearch.ClientKey)) + } + + if o.Elasticsearch.ClientKeyPassword != nil { + pwd, err := loader.LoadSecret(*o.Elasticsearch.ClientKeyPassword) + if err != nil { + return nil, err + } + parent.InsertPairs("client_key_pass", pwd) + } + if o.Elasticsearch.Scheme != nil { parent.InsertPairs("scheme", fmt.Sprint(*o.Elasticsearch.Scheme)) } diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml index c984fa41c..489513939 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml @@ -539,6 +539,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -608,6 +650,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml index b9949ca8e..0f9f66db0 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml @@ -539,6 +539,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -608,6 +650,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch diff --git a/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml index c984fa41c..489513939 100644 --- a/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml @@ -539,6 +539,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -608,6 +650,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch diff --git a/config/crd/bases/fluentd.fluent.io_outputs.yaml b/config/crd/bases/fluentd.fluent.io_outputs.yaml index b9949ca8e..0f9f66db0 100644 --- a/config/crd/bases/fluentd.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentd.fluent.io_outputs.yaml @@ -539,6 +539,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -608,6 +650,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch diff --git a/docs/plugins/fluentd/output/es.md b/docs/plugins/fluentd/output/es.md index 72272819c..f9736751f 100644 --- a/docs/plugins/fluentd/output/es.md +++ b/docs/plugins/fluentd/output/es.md @@ -15,3 +15,8 @@ Elasticsearch defines the parameters for out_es output plugin | logstashPrefix | LogstashPrefix defines the logstash prefix index name to write events when logstash_format is true (default: logstash). | *string | | user | Optional, The login credentials to connect to Elasticsearch | *[plugins.Secret](../secret.md) | | password | Optional, The login credentials to connect to Elasticsearch | *[plugins.Secret](../secret.md) | +| sslVerify | Optional, Force certificate validation | *bool | +| caFile | Optional, Absolute path to CA certificate file | *string | +| clientCert | Optional, Absolute path to client Certificate file | *string | +| clientKey | Optional, Absolute path to client private Key file | *string | +| clientKeyPassword | Optional, password for ClientKey file | *[plugins.Secret](../secret.md) | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 533eb1a2d..268b63f04 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -6252,6 +6252,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -6321,6 +6363,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch @@ -29464,6 +29509,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -29533,6 +29620,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 079f47d49..9ec5755b5 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -6252,6 +6252,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -6321,6 +6363,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch @@ -29464,6 +29509,48 @@ spec: elasticsearch: description: out_es plugin properties: + caFile: + description: Optional, Absolute path to CA certificate file + type: string + clientCert: + description: Optional, Absolute path to client Certificate + file + type: string + clientKey: + description: Optional, Absolute path to client private Key + file + type: string + clientKeyPassword: + description: Optional, password for ClientKey file + properties: + valueFrom: + description: ValueSource defines how to find a value's + key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object host: description: 'The hostname of your Elasticsearch node (default: localhost).' @@ -29533,6 +29620,9 @@ spec: description: 'Specify https if your Elasticsearch endpoint supports SSL (default: http).' type: string + sslVerify: + description: Optional, Force certificate validation + type: boolean user: description: Optional, The login credentials to connect to Elasticsearch