From cc8615dbe6c91de3245415e5ede78fcf4ea7d811 Mon Sep 17 00:00:00 2001 From: juicer Date: Sun, 15 Dec 2024 17:32:03 -0800 Subject: [PATCH] support offsetKey Signed-off-by: juicer --- apis/fluentbit/v1alpha2/plugins/input/tail_types.go | 6 ++++++ .../crds/fluentbit.fluent.io_clusterinputs.yaml | 5 +++++ config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml | 5 +++++ docs/plugins/fluentbit/input/tail.md | 1 + manifests/setup/fluent-operator-crd.yaml | 5 +++++ manifests/setup/setup.yaml | 5 +++++ 6 files changed, 27 insertions(+) diff --git a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go index 33dd67071..446ef4138 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go +++ b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go @@ -33,6 +33,9 @@ type Tail struct { // Set one or multiple shell patterns separated by commas to exclude files matching a certain criteria, // e.g: exclude_path=*.gz,*.zip ExcludePath string `json:"excludePath,omitempty"` + // If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. + // The value assigned becomes the key in the map + OffsetKey string `json:"offsetKey,omitempty"` // For new discovered files on start (without a database offset/position), // read the content from the head of the file, not tail. ReadFromHead *bool `json:"readFromHead,omitempty"` @@ -124,6 +127,9 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) { if t.ExcludePath != "" { kvs.Insert("Exclude_Path", t.ExcludePath) } + if t.OffsetKey != "" { + kvs.Insert("Offset_Key", t.OffsetKey) + } if t.ReadFromHead != nil { kvs.Insert("Read_from_Head", fmt.Sprint(*t.ReadFromHead)) } diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index aacf75b01..f3ce7b5c1 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -761,6 +761,11 @@ spec: This will help to reassembly multiline messages originally split by Docker or CRI Specify one or Multiline Parser definition to apply to the content. type: string + offsetKey: + description: |- + If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. + The value assigned becomes the key in the map + type: string parser: description: Specify the name of a parser to interpret the entry as a structured message. diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index aacf75b01..f3ce7b5c1 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -761,6 +761,11 @@ spec: This will help to reassembly multiline messages originally split by Docker or CRI Specify one or Multiline Parser definition to apply to the content. type: string + offsetKey: + description: |- + If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. + The value assigned becomes the key in the map + type: string parser: description: Specify the name of a parser to interpret the entry as a structured message. diff --git a/docs/plugins/fluentbit/input/tail.md b/docs/plugins/fluentbit/input/tail.md index 3e024d94b..3be8c53be 100644 --- a/docs/plugins/fluentbit/input/tail.md +++ b/docs/plugins/fluentbit/input/tail.md @@ -10,6 +10,7 @@ The Tail input plugin allows to monitor one or several text files.
It has | path | Pattern specifying a specific log files or multiple ones through the use of common wildcards. | string | | pathKey | If enabled, it appends the name of the monitored file as part of the record. The value assigned becomes the key in the map. | string | | excludePath | Set one or multiple shell patterns separated by commas to exclude files matching a certain criteria, e.g: exclude_path=*.gz,*.zip | string | +| offsetKey | If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. The value assigned becomes the key in the map | string | | readFromHead | For new discovered files on start (without a database offset/position), read the content from the head of the file, not tail. | *bool | | refreshIntervalSeconds | The interval of refreshing the list of watched files in seconds. | *int64 | | rotateWaitSeconds | Specify the number of extra time in seconds to monitor a file once is rotated in case some pending data is flushed. | *int64 | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 16b0c26ed..74771bd5e 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -2877,6 +2877,11 @@ spec: This will help to reassembly multiline messages originally split by Docker or CRI Specify one or Multiline Parser definition to apply to the content. type: string + offsetKey: + description: |- + If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. + The value assigned becomes the key in the map + type: string parser: description: Specify the name of a parser to interpret the entry as a structured message. diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 774dd97ef..c12ae2949 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -2877,6 +2877,11 @@ spec: This will help to reassembly multiline messages originally split by Docker or CRI Specify one or Multiline Parser definition to apply to the content. type: string + offsetKey: + description: |- + If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. + The value assigned becomes the key in the map + type: string parser: description: Specify the name of a parser to interpret the entry as a structured message.