support offsetKey

Signed-off-by: juicer <[email protected]>
fluent · Dec 16, 2024 · cc8615d · cc8615d
1 parent cc23366
commit cc8615d
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 0 deletions.
diff --git a/apis/fluentbit/v1alpha2/plugins/input/tail_types.go b/apis/fluentbit/v1alpha2/plugins/input/tail_types.go
@@ -33,6 +33,9 @@ type Tail struct {
 	// Set one or multiple shell patterns separated by commas to exclude files matching a certain criteria,
 	// e.g: exclude_path=*.gz,*.zip
 	ExcludePath string `json:"excludePath,omitempty"`
+	// If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+	// The value assigned becomes the key in the map
+	OffsetKey string `json:"offsetKey,omitempty"`
 	// For new discovered files on start (without a database offset/position),
 	// read the content from the head of the file, not tail.
 	ReadFromHead *bool `json:"readFromHead,omitempty"`
@@ -124,6 +127,9 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 	if t.ExcludePath != "" {
 		kvs.Insert("Exclude_Path", t.ExcludePath)
 	}
+	if t.OffsetKey != "" {
+		kvs.Insert("Offset_Key", t.OffsetKey)
+	}
 	if t.ReadFromHead != nil {
 		kvs.Insert("Read_from_Head", fmt.Sprint(*t.ReadFromHead))
 	}

diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml
@@ -761,6 +761,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml
@@ -761,6 +761,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

diff --git a/docs/plugins/fluentbit/input/tail.md b/docs/plugins/fluentbit/input/tail.md
@@ -10,6 +10,7 @@ The Tail input plugin allows to monitor one or several text files. <br /> It has
 | path | Pattern specifying a specific log files or multiple ones through the use of common wildcards. | string |
 | pathKey | If enabled, it appends the name of the monitored file as part of the record. The value assigned becomes the key in the map. | string |
 | excludePath | Set one or multiple shell patterns separated by commas to exclude files matching a certain criteria, e.g: exclude_path=*.gz,*.zip | string |
+| offsetKey | If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. The value assigned becomes the key in the map | string |
 | readFromHead | For new discovered files on start (without a database offset/position), read the content from the head of the file, not tail. | *bool |
 | refreshIntervalSeconds | The interval of refreshing the list of watched files in seconds. | *int64 |
 | rotateWaitSeconds | Specify the number of extra time in seconds to monitor a file once is rotated in case some pending data is flushed. | *int64 |

diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml
@@ -2877,6 +2877,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml
@@ -2877,6 +2877,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.