From b2fb860d6551cf6f1c0ab732b220b2b97d74fc4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Wed, 17 Jul 2024 15:26:59 +0200 Subject: [PATCH 1/2] Add missing fluent-bit config parameters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added fluent-bit config parameters which are not yet exposed in the CRD. Signed-off-by: Zoltán Reegn --- .../plugins/filter/kubernetes_types.go | 29 +++++++++++++++++++ .../plugins/filter/zz_generated.deepcopy.go | 20 +++++++++++++ docs/plugins/fluentbit/filter/kubernetes.md | 5 ++++ 3 files changed, 54 insertions(+) diff --git a/apis/fluentbit/v1alpha2/plugins/filter/kubernetes_types.go b/apis/fluentbit/v1alpha2/plugins/filter/kubernetes_types.go index 99493a34f..4ee7961e4 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/kubernetes_types.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/kubernetes_types.go @@ -89,6 +89,20 @@ type Kubernetes struct { // configurable 'time to live' for the K8s token. By default, it is set to 600 seconds. // After this time, the token is reloaded from Kube_Token_File or the Kube_Token_Command. KubeTokenTTL string `json:"kubeTokenTTL,omitempty"` + // Command to get Kubernetes authorization token. + // By default, it will be NULL and we will use token file to get token. + KubeTokenCommand string `json:"kubeTokenCommand,omitempty"` + // Configurable TTL for K8s cached namespace metadata. + // By default, it is set to 900 which means a 15min TTL for namespace cache entries. + // Setting this to 0 will mean entries are evicted at random once the cache is full. + KubeMetaNamespaceCacheTTL *int32 `json:"kubeMetaNamespaceCacheTTL,omitempty"` + // Include Kubernetes namespace resource labels in the extra metadata. + NamespaceLabels *bool `json:"namespaceLabels,omitempty"` + // Include Kubernetes namespace resource annotations in the extra metadata. + NamespaceAnnotations *bool `json:"namespaceAnnotations,omitempty"` + // Include Kubernetes namespace metadata only and no pod metadata. + // If this is set, the values of Labels and Annotations are ignored. + NamespaceMetadataOnly *bool `json:"namespaceMetadataOnly,omitempty"` } func (_ *Kubernetes) Name() string { @@ -188,6 +202,21 @@ func (k *Kubernetes) Params(_ plugins.SecretLoader) (*params.KVs, error) { if k.KubeTokenTTL != "" { kvs.Insert("Kube_Token_TTL", k.KubeTokenTTL) } + if k.KubeTokenCommand != "" { + kvs.Insert("Kube_Token_Command", fmt.Sprint(k.KubeTokenCommand)) + } + if k.KubeMetaNamespaceCacheTTL != nil { + kvs.Insert("Kube_Meta_Namespace_Cache_TTL", fmt.Sprint(*k.KubeMetaNamespaceCacheTTL)) + } + if k.NamespaceLabels != nil { + kvs.Insert("Namespace_Labels", fmt.Sprint(*k.NamespaceLabels)) + } + if k.NamespaceAnnotations != nil { + kvs.Insert("Namespace_Annotations", fmt.Sprint(*k.NamespaceAnnotations)) + } + if k.NamespaceMetadataOnly != nil { + kvs.Insert("Namespace_Metadata_Only", fmt.Sprint(*k.NamespaceMetadataOnly)) + } return kvs, nil } diff --git a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go index c569a9c6f..511f984c7 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/zz_generated.deepcopy.go @@ -242,6 +242,26 @@ func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { *out = new(int32) **out = **in } + if in.KubeMetaNamespaceCacheTTL != nil { + in, out := &in.KubeMetaNamespaceCacheTTL, &out.KubeMetaNamespaceCacheTTL + *out = new(int32) + **out = **in + } + if in.NamespaceLabels != nil { + in, out := &in.NamespaceLabels, &out.NamespaceLabels + *out = new(bool) + **out = **in + } + if in.NamespaceAnnotations != nil { + in, out := &in.NamespaceAnnotations, &out.NamespaceAnnotations + *out = new(bool) + **out = **in + } + if in.NamespaceMetadataOnly != nil { + in, out := &in.NamespaceMetadataOnly, &out.NamespaceMetadataOnly + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kubernetes. diff --git a/docs/plugins/fluentbit/filter/kubernetes.md b/docs/plugins/fluentbit/filter/kubernetes.md index 772245b8e..32ed7ad21 100644 --- a/docs/plugins/fluentbit/filter/kubernetes.md +++ b/docs/plugins/fluentbit/filter/kubernetes.md @@ -34,3 +34,8 @@ Kubernetes filter allows to enrich your log files with Kubernetes metadata.
Date: Wed, 17 Jul 2024 15:40:58 +0200 Subject: [PATCH 2/2] update crds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zoltán Reegn --- .../fluentbit.fluent.io_clusterfilters.yaml | 25 ++++++++++ .../crds/fluentbit.fluent.io_filters.yaml | 25 ++++++++++ .../fluentbit.fluent.io_clusterfilters.yaml | 25 ++++++++++ .../bases/fluentbit.fluent.io_filters.yaml | 25 ++++++++++ manifests/setup/fluent-operator-crd.yaml | 50 +++++++++++++++++++ manifests/setup/setup.yaml | 50 +++++++++++++++++++ 6 files changed, 200 insertions(+) diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index 19826e857..f17e3270f 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -200,6 +200,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -210,6 +217,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -254,6 +266,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index a5dc151d6..2f8e6c0b2 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -200,6 +200,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -210,6 +217,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -254,6 +266,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index 19826e857..f17e3270f 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -200,6 +200,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -210,6 +217,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -254,6 +266,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index a5dc151d6..2f8e6c0b2 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -200,6 +200,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -210,6 +217,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -254,6 +266,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 8671b15c6..1309ba3d7 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -199,6 +199,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -209,6 +216,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -253,6 +265,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. @@ -13683,6 +13708,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -13693,6 +13725,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -13737,6 +13774,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index a076ee29c..84a696f80 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -199,6 +199,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -209,6 +216,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -253,6 +265,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id. @@ -13683,6 +13708,13 @@ spec: when capacity is reached. In order to enable this option, you should set the number to a time interval. For example, set this value to 60 or 60s and cache entries which have been created more than 60s will be evicted. type: string + kubeMetaNamespaceCacheTTL: + description: |- + Configurable TTL for K8s cached namespace metadata. + By default, it is set to 900 which means a 15min TTL for namespace cache entries. + Setting this to 0 will mean entries are evicted at random once the cache is full. + format: int32 + type: integer kubeMetaPreloadCacheDir: description: |- If set, Kubernetes meta-data can be cached/pre-loaded from files in JSON format in this directory, @@ -13693,6 +13725,11 @@ spec: When the source records comes from Tail input plugin, this option allows to specify what's the prefix used in Tail configuration. type: string + kubeTokenCommand: + description: |- + Command to get Kubernetes authorization token. + By default, it will be NULL and we will use token file to get token. + type: string kubeTokenFile: description: Token file type: string @@ -13737,6 +13774,19 @@ spec: the data contained in the log key. Recommended use is for developers or testing only. type: string + namespaceAnnotations: + description: Include Kubernetes namespace resource annotations + in the extra metadata. + type: boolean + namespaceLabels: + description: Include Kubernetes namespace resource labels + in the extra metadata. + type: boolean + namespaceMetadataOnly: + description: |- + Include Kubernetes namespace metadata only and no pod metadata. + If this is set, the values of Labels and Annotations are ignored. + type: boolean regexParser: description: |- Set an alternative Parser to process record Tag and extract pod_name, namespace_name, container_name and docker_id.