From bb019e1a4d9b0406d93b15027e7125d59a812250 Mon Sep 17 00:00:00 2001 From: rene Date: Fri, 28 Jun 2024 14:43:24 +0200 Subject: [PATCH 1/3] add bearer token parameter to fluent-bit (cluster) output Signed-off-by: rene --- .../v1alpha2/plugins/output/loki_types.go | 15 ++++ .../plugins/output/zz_generated.deepcopy.go | 5 ++ .../fluentbit.fluent.io_clusteroutputs.yaml | 35 ++++++++++ .../crds/fluentbit.fluent.io_outputs.yaml | 35 ++++++++++ .../fluentbit.fluent.io_clusteroutputs.yaml | 35 ++++++++++ .../bases/fluentbit.fluent.io_outputs.yaml | 35 ++++++++++ docs/plugins/fluentbit/output/loki.md | 2 + manifests/setup/fluent-operator-crd.yaml | 70 +++++++++++++++++++ manifests/setup/setup.yaml | 70 +++++++++++++++++++ 9 files changed, 302 insertions(+) diff --git a/apis/fluentbit/v1alpha2/plugins/output/loki_types.go b/apis/fluentbit/v1alpha2/plugins/output/loki_types.go index 1253c7cc4..d91ec4f50 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/loki_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/loki_types.go @@ -19,11 +19,16 @@ type Loki struct { // +kubebuilder:validation:Minimum:=1 // +kubebuilder:validation:Maximum:=65535 Port *int32 `json:"port,omitempty"` + // Specify a custom HTTP URI. It must start with forward slash. + Uri string `json:"uri,omitempty"` // Set HTTP basic authentication user name. HTTPUser *plugins.Secret `json:"httpUser,omitempty"` // Password for user defined in HTTP_User // Set HTTP basic authentication password HTTPPasswd *plugins.Secret `json:"httpPassword,omitempty"` + // Set bearer token authentication token value. + // Can be used as alterntative to HTTP basic authentication + BearerToken *plugins.Secret `json:"bearerToken,omitempty"` // Tenant ID used by default to push logs to Loki. // If omitted or empty it assumes Loki is running in single-tenant mode and no X-Scope-OrgID header is sent. TenantID *plugins.Secret `json:"tenantID,omitempty"` @@ -70,6 +75,9 @@ func (l *Loki) Params(sl plugins.SecretLoader) (*params.KVs, error) { if l.Port != nil { kvs.Insert("port", fmt.Sprint(*l.Port)) } + if l.Uri != "" { + kvs.Insert("uri", l.Uri) + } if l.HTTPUser != nil { u, err := sl.LoadSecret(*l.HTTPUser) if err != nil { @@ -84,6 +92,13 @@ func (l *Loki) Params(sl plugins.SecretLoader) (*params.KVs, error) { } kvs.Insert("http_passwd", pwd) } + if l.BearerToken != nil { + bearerToken, err := sl.LoadSecret(*l.BearerToken) + if err != nil { + return nil, err + } + kvs.Insert("bearer_token", bearerToken) + } if l.TenantID != nil { id, err := sl.LoadSecret(*l.TenantID) if err != nil { diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go index bde893e27..04acb1130 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go @@ -576,6 +576,11 @@ func (in *Loki) DeepCopyInto(out *Loki) { *out = new(plugins.Secret) (*in).DeepCopyInto(*out) } + if in.BearerToken != nil { + in, out := &in.BearerToken, &out.BearerToken + *out = new(plugins.Secret) + (*in).DeepCopyInto(*out) + } if in.TenantID != nil { in, out := &in.TenantID, &out.TenantID *out = new(plugins.Secret) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index 58ac71953..e4e95949c 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -1979,6 +1979,37 @@ spec: - "on" - "off" type: string + bearerToken: + description: |- + Set bearer token authentication token value. + Can be used as alterntative to HTTP basic authentication + properties: + valueFrom: + description: ValueSource defines how to find a value's key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object dropSingleKey: description: If set to true and after extracting labels only a single key remains, the log line sent to Loki will be the value @@ -2249,6 +2280,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uri: + description: Specify a custom HTTP URI. It must start with forward + slash. + type: string required: - host type: object diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index fe54c285e..1a0218985 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -1979,6 +1979,37 @@ spec: - "on" - "off" type: string + bearerToken: + description: |- + Set bearer token authentication token value. + Can be used as alterntative to HTTP basic authentication + properties: + valueFrom: + description: ValueSource defines how to find a value's key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object dropSingleKey: description: If set to true and after extracting labels only a single key remains, the log line sent to Loki will be the value @@ -2249,6 +2280,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uri: + description: Specify a custom HTTP URI. It must start with forward + slash. + type: string required: - host type: object diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index 58ac71953..e4e95949c 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -1979,6 +1979,37 @@ spec: - "on" - "off" type: string + bearerToken: + description: |- + Set bearer token authentication token value. + Can be used as alterntative to HTTP basic authentication + properties: + valueFrom: + description: ValueSource defines how to find a value's key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object dropSingleKey: description: If set to true and after extracting labels only a single key remains, the log line sent to Loki will be the value @@ -2249,6 +2280,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uri: + description: Specify a custom HTTP URI. It must start with forward + slash. + type: string required: - host type: object diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index fe54c285e..1a0218985 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -1979,6 +1979,37 @@ spec: - "on" - "off" type: string + bearerToken: + description: |- + Set bearer token authentication token value. + Can be used as alterntative to HTTP basic authentication + properties: + valueFrom: + description: ValueSource defines how to find a value's key. + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object dropSingleKey: description: If set to true and after extracting labels only a single key remains, the log line sent to Loki will be the value @@ -2249,6 +2280,10 @@ spec: description: Hostname to be used for TLS SNI extension type: string type: object + uri: + description: Specify a custom HTTP URI. It must start with forward + slash. + type: string required: - host type: object diff --git a/docs/plugins/fluentbit/output/loki.md b/docs/plugins/fluentbit/output/loki.md index c165b8903..80b93b30a 100644 --- a/docs/plugins/fluentbit/output/loki.md +++ b/docs/plugins/fluentbit/output/loki.md @@ -7,8 +7,10 @@ The loki output plugin, allows to ingest your records into a Loki service.
Date: Fri, 28 Jun 2024 22:05:50 +0200 Subject: [PATCH 2/3] add bearer token file parameter to fluentd (cluster) output Signed-off-by: rene --- apis/fluentd/v1alpha1/plugins/output/loki.go | 3 +++ apis/fluentd/v1alpha1/plugins/output/types.go | 3 +++ .../crds/fluentd.fluent.io_clusteroutputs.yaml | 5 +++++ .../fluentd-crds/crds/fluentd.fluent.io_outputs.yaml | 5 +++++ config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml | 5 +++++ config/crd/bases/fluentd.fluent.io_outputs.yaml | 5 +++++ docs/plugins/fluentd/output/loki.md | 1 + manifests/setup/fluent-operator-crd.yaml | 10 ++++++++++ manifests/setup/setup.yaml | 10 ++++++++++ 9 files changed, 47 insertions(+) diff --git a/apis/fluentd/v1alpha1/plugins/output/loki.go b/apis/fluentd/v1alpha1/plugins/output/loki.go index dfca351d3..3a447afbb 100644 --- a/apis/fluentd/v1alpha1/plugins/output/loki.go +++ b/apis/fluentd/v1alpha1/plugins/output/loki.go @@ -13,6 +13,9 @@ type Loki struct { // Password for user defined in HTTP_User // Set HTTP basic authentication password HTTPPasswd *plugins.Secret `json:"httpPassword,omitempty"` + // Set path to file with bearer authentication token + // Can be used as alterntative to HTTP basic authentication + BearerTokenFile *string `json:"bearerTokenFile,omitempty"` // Tenant ID used by default to push logs to Loki. // If omitted or empty it assumes Loki is running in single-tenant mode and no X-Scope-OrgID header is sent. TenantID *plugins.Secret `json:"tenantID,omitempty"` diff --git a/apis/fluentd/v1alpha1/plugins/output/types.go b/apis/fluentd/v1alpha1/plugins/output/types.go index 009f14a1d..2facbef47 100644 --- a/apis/fluentd/v1alpha1/plugins/output/types.go +++ b/apis/fluentd/v1alpha1/plugins/output/types.go @@ -714,6 +714,9 @@ func (o *Output) lokiPlugin(parent *params.PluginStore, loader plugins.SecretLoa } parent.InsertPairs("password", passwd) } + if o.Loki.BearerTokenFile != nil { + parent.InsertPairs("bearer_token_file", fmt.Sprint(*o.Loki.BearerTokenFile)) + } if o.Loki.TenantID != nil { id, err := loader.LoadSecret(*o.Loki.TenantID) if err != nil { diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml index fade897cf..2dd2236c4 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml @@ -1914,6 +1914,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml index d6b155b3e..a574ad8b7 100644 --- a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml @@ -1914,6 +1914,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. diff --git a/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml index fade897cf..2dd2236c4 100644 --- a/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml @@ -1914,6 +1914,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. diff --git a/config/crd/bases/fluentd.fluent.io_outputs.yaml b/config/crd/bases/fluentd.fluent.io_outputs.yaml index 0da25d5f9..29fc3337e 100644 --- a/config/crd/bases/fluentd.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentd.fluent.io_outputs.yaml @@ -1914,6 +1914,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. diff --git a/docs/plugins/fluentd/output/loki.md b/docs/plugins/fluentd/output/loki.md index 45a2548d8..0b2bfed7e 100644 --- a/docs/plugins/fluentd/output/loki.md +++ b/docs/plugins/fluentd/output/loki.md @@ -8,6 +8,7 @@ The loki output plugin, allows to ingest your records into a Loki service. | url | Loki URL. | *string | | httpUser | Set HTTP basic authentication user name. | *[plugins.Secret](../secret.md) | | httpPassword | Password for user defined in HTTP_User Set HTTP basic authentication password | *[plugins.Secret](../secret.md) | +| bearerTokenFile | Set path to file with bearer authentication token Can be used as alterntative to HTTP basic authentication | *string | | tenantID | Tenant ID used by default to push logs to Loki. If omitted or empty it assumes Loki is running in single-tenant mode and no X-Scope-OrgID header is sent. | *[plugins.Secret](../secret.md) | | labels | Stream labels for API request. It can be multiple comma separated of strings specifying key=value pairs. In addition to fixed parameters, it also allows to add custom record keys (similar to label_keys property). | []string | | labelKeys | Optional list of record keys that will be placed as stream labels. This configuration property is for records key only. | []string | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index e97d78ac2..ed1bd70a4 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -9752,6 +9752,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. @@ -35880,6 +35885,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 2b98ba51e..a076ee29c 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -9752,6 +9752,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. @@ -35880,6 +35885,11 @@ spec: loki: description: out_loki plugin properties: + bearerTokenFile: + description: |- + Set path to file with bearer authentication token + Can be used as alterntative to HTTP basic authentication + type: string dropSingleKey: description: If a record only has 1 key, then just set the log line to the value and discard the key. From f11d4c1867d4d9f0f380258835e92e8c7c372407 Mon Sep 17 00:00:00 2001 From: rene Date: Tue, 2 Jul 2024 12:16:23 +0200 Subject: [PATCH 3/3] add bearer token support in the helm chart for fluentbit loki output Signed-off-by: rene --- .../templates/fluentbit-output-loki.yaml | 18 ++++++++++++++++-- charts/fluent-operator/values.yaml | 11 ++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/charts/fluent-operator/templates/fluentbit-output-loki.yaml b/charts/fluent-operator/templates/fluentbit-output-loki.yaml index 87c342690..2c2bc9073 100644 --- a/charts/fluent-operator/templates/fluentbit-output-loki.yaml +++ b/charts/fluent-operator/templates/fluentbit-output-loki.yaml @@ -5,14 +5,15 @@ {{ with .Values.fluentbit.output.loki -}} {{/* -When http{User,Password} or tenantID is a string, make a secret for them +When http{User,Password}, bearerToken, or tenantID is a string, make a secret for them When these keys are objects, they specify a secret to use generated elsewhere, assumed to exist in the k8s cluster */}} {{ $userSecret := "loki-http-auth" -}} {{ $passSecret := "loki-http-pass" -}} +{{ $bearerTokenSecret := "loki-bearer-token" -}} {{ $tenantIDSecret := "loki-tenant-id" -}} -{{ range $k, $v := dict $userSecret .httpUser $passSecret .httpPassword $tenantIDSecret .tenantID -}} +{{ range $k, $v := dict $userSecret .httpUser $passSecret .httpPassword $tenantIDSecret .tenantID $bearerTokenSecret .bearerToken -}} {{ if kindIs "string" $v -}} --- apiVersion: v1 @@ -81,6 +82,19 @@ spec: {{- end }} {{- end }} + {{- if .bearerToken }} + bearerToken: + {{- if kindIs "string" .bearerToken }} + valueFrom: + secretKeyRef: + key: 'value' + name: {{ $bearerTokenSecret }} + optional: false + {{- else }} +{{ .bearerToken | toYaml | indent 6 }} + {{- end }} + {{- end }} + {{- if .tenantID }} tenantID: {{- if kindIs "string" .tenantID }} diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index c3018be96..fa3bc1ef6 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -278,7 +278,7 @@ fluentbit: # See https://docs.fluentbit.io/manual/pipeline/outputs/loki loki: # Switch for generation of fluentbit loki ClusterOutput (and loki basic auth http user and pass secrets if required) - enable: false # Bool + enable: false # Bool host: 127.0.0.1 # String port: 3100 # Int # Either, give http{User,Password},tenantID string values specifying them directly @@ -305,6 +305,15 @@ fluentbit: # name: tenantsecret # optional: true # + # To use bearer token auth instead of http basic auth + #bearerToken: ey.... + # or with existing secret + #bearerToken: + # valueFrom: + # secretKeyRef: + # key: value + # name: bearerTokenSecret + # optional: true #labels: [] # String list of = #labelKeys: [] # String list of #removeKeys: [] # String list of