Merge pull request #936 from sjliu1/tcp

[summerospp]add fluentbit tcp plugin
fluent · Oct 7, 2023 · 5eff68d · 5eff68d
2 parents aa780d0 + bb0251e
commit 5eff68d
Show file tree

Hide file tree

Showing 10 changed files with 258 additions and 0 deletions.
diff --git a/apis/fluentbit/v1alpha2/clusterinput_types.go b/apis/fluentbit/v1alpha2/clusterinput_types.go
@@ -69,6 +69,8 @@ type InputSpec struct {
 	Nginx *input.Nginx `json:"nginx,omitempty"`
 	// Syslog defines the Syslog input plugin configuration
 	Syslog *input.Syslog `json:"syslog,omitempty"`
+	// TCP defines the TCP input plugin configuration
+	TCP *input.TCP `json:"tcp,omitempty"`
 }
 
 // +kubebuilder:object:root=true

diff --git a/apis/fluentbit/v1alpha2/plugins/input/tcp.go b/apis/fluentbit/v1alpha2/plugins/input/tcp.go
@@ -0,0 +1,61 @@
+package input
+
+import (
+	"fmt"
+
+	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins"
+	"github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params"
+)
+
+// +kubebuilder:object:generate:=true
+
+// The tcp input plugin allows to retrieve structured JSON or raw messages over a TCP network interface (TCP port).
+// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/tcp**
+type TCP struct {
+	// Listener network interface,default 0.0.0.0
+	Listen string `json:"listen,omitempty"`
+	// TCP port where listening for connections,default 5170
+	// +kubebuilder:validation:Minimum:=1
+	// +kubebuilder:validation:Maximum:=65535
+	Port *int32 `json:"port,omitempty"`
+	// Specify the maximum buffer size in KB to receive a JSON message. If not set, the default size will be the value of Chunk_Size.
+	// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
+	BufferSize string `json:"bufferSize,omitempty"`
+	// By default the buffer to store the incoming JSON messages, do not allocate the maximum memory allowed, instead it allocate memory when is required.
+	//The rounds of allocations are set by Chunk_Size in KB. If not set, Chunk_Size is equal to 32 (32KB).
+	// +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$"
+	ChunkSize string `json:"chunkSize,omitempty"`
+	// Specify the expected payload format. It support the options json and none.
+	// When using json, it expects JSON maps, when is set to none, it will split every record using the defined Separator (option below).
+	Format string `json:"format,omitempty"`
+	// When the expected Format is set to none, Fluent Bit needs a separator string to split the records. By default it uses the breakline character (LF or 0x10).
+	Separator string `json:"separator,omitempty"`
+}
+
+func (_ *TCP) Name() string {
+	return "tcp"
+}
+
+// Params implement Section() method
+func (t *TCP) Params(sl plugins.SecretLoader) (*params.KVs, error) {
+	kvs := params.NewKVs()
+	if t.Listen != "" {
+		kvs.Insert("listen", t.Listen)
+	}
+	if t.Port != nil {
+		kvs.Insert("port", fmt.Sprint(*t.Port))
+	}
+	if t.BufferSize != "" {
+		kvs.Insert("Buffer_Size", t.BufferSize)
+	}
+	if t.ChunkSize != "" {
+		kvs.Insert("Chunk_Size", t.ChunkSize)
+	}
+	if t.Format != "" {
+		kvs.Insert("Format", t.Format)
+	}
+	if t.Separator != "" {
+		kvs.Insert("Separator", t.Separator)
+	}
+	return kvs, nil
+}
diff --git a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go
diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml
@@ -702,6 +702,45 @@ spec:
                     description: Set a regex to exctract fields from the file
                     type: string
                 type: object
+              tcp:
+                description: TCP defines the TCP input plugin configuration
+                properties:
+                  bufferSize:
+                    description: Specify the maximum buffer size in KB to receive
+                      a JSON message. If not set, the default size will be the value
+                      of Chunk_Size.
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  chunkSize:
+                    description: By default the buffer to store the incoming JSON
+                      messages, do not allocate the maximum memory allowed, instead
+                      it allocate memory when is required. The rounds of allocations
+                      are set by Chunk_Size in KB. If not set, Chunk_Size is equal
+                      to 32 (32KB).
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  format:
+                    description: Specify the expected payload format. It support the
+                      options json and none. When using json, it expects JSON maps,
+                      when is set to none, it will split every record using the defined
+                      Separator (option below).
+                    type: string
+                  listen:
+                    description: Listener network interface,default 0.0.0.0
+                    type: string
+                  port:
+                    description: TCP port where listening for connections,default
+                      5170
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                  separator:
+                    description: When the expected Format is set to none, Fluent Bit
+                      needs a separator string to split the records. By default it
+                      uses the breakline character (LF or 0x10).
+                    type: string
+                type: object
             type: object
         type: object
     served: true

diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml
@@ -702,6 +702,45 @@ spec:
                     description: Set a regex to exctract fields from the file
                     type: string
                 type: object
+              tcp:
+                description: TCP defines the TCP input plugin configuration
+                properties:
+                  bufferSize:
+                    description: Specify the maximum buffer size in KB to receive
+                      a JSON message. If not set, the default size will be the value
+                      of Chunk_Size.
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  chunkSize:
+                    description: By default the buffer to store the incoming JSON
+                      messages, do not allocate the maximum memory allowed, instead
+                      it allocate memory when is required. The rounds of allocations
+                      are set by Chunk_Size in KB. If not set, Chunk_Size is equal
+                      to 32 (32KB).
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  format:
+                    description: Specify the expected payload format. It support the
+                      options json and none. When using json, it expects JSON maps,
+                      when is set to none, it will split every record using the defined
+                      Separator (option below).
+                    type: string
+                  listen:
+                    description: Listener network interface,default 0.0.0.0
+                    type: string
+                  port:
+                    description: TCP port where listening for connections,default
+                      5170
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                  separator:
+                    description: When the expected Format is set to none, Fluent Bit
+                      needs a separator string to split the records. By default it
+                      uses the breakline character (LF or 0x10).
+                    type: string
+                type: object
             type: object
         type: object
     served: true

diff --git a/docs/fluentbit.md b/docs/fluentbit.md
@@ -427,6 +427,7 @@ InputSpec defines the desired state of ClusterInput
 | statsd | StatsD defines the StatsD input plugin configuration | *[input.StatsD](plugins/input/statsd.md) |
 | nginx | Nginx defines the Nginx input plugin configuration | *[input.Nginx](plugins/input/nginx.md) |
 | syslog | Syslog defines the Syslog input plugin configuration | *[input.Syslog](plugins/input/syslog.md) |
+| tcp | TCP defines the TCP input plugin configuration | *[input.TCP](plugins/input/tcp.md) |
 
 [Back to TOC](#table-of-contents)
 # NamespacedFluentBitCfgSpec

diff --git a/docs/plugins/fluentbit/input/tcp.md b/docs/plugins/fluentbit/input/tcp.md
@@ -0,0 +1,13 @@
+# TCP
+
+The tcp input plugin allows to retrieve structured JSON or raw messages over a TCP network interface (TCP port). **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/tcp**
+
+
+| Field | Description | Scheme |
+| ----- | ----------- | ------ |
+| listen | Listener network interface,default 0.0.0.0 | string |
+| port | TCP port where listening for connections,default 5170 | *int32 |
+| bufferSize | Specify the maximum buffer size in KB to receive a JSON message. If not set, the default size will be the value of Chunk_Size. | string |
+| chunkSize | By default the buffer to store the incoming JSON messages, do not allocate the maximum memory allowed, instead it allocate memory when is required. The rounds of allocations are set by Chunk_Size in KB. If not set, Chunk_Size is equal to 32 (32KB). | string |
+| format | Specify the expected payload format. It support the options json and none. When using json, it expects JSON maps, when is set to none, it will split every record using the defined Separator (option below). | string |
+| separator | When the expected Format is set to none, Fluent Bit needs a separator string to split the records. By default it uses the breakline character (LF or 0x10). | string |
diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml
@@ -2451,6 +2451,45 @@ spec:
                     description: Set a regex to exctract fields from the file
                     type: string
                 type: object
+              tcp:
+                description: TCP defines the TCP input plugin configuration
+                properties:
+                  bufferSize:
+                    description: Specify the maximum buffer size in KB to receive
+                      a JSON message. If not set, the default size will be the value
+                      of Chunk_Size.
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  chunkSize:
+                    description: By default the buffer to store the incoming JSON
+                      messages, do not allocate the maximum memory allowed, instead
+                      it allocate memory when is required. The rounds of allocations
+                      are set by Chunk_Size in KB. If not set, Chunk_Size is equal
+                      to 32 (32KB).
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  format:
+                    description: Specify the expected payload format. It support the
+                      options json and none. When using json, it expects JSON maps,
+                      when is set to none, it will split every record using the defined
+                      Separator (option below).
+                    type: string
+                  listen:
+                    description: Listener network interface,default 0.0.0.0
+                    type: string
+                  port:
+                    description: TCP port where listening for connections,default
+                      5170
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                  separator:
+                    description: When the expected Format is set to none, Fluent Bit
+                      needs a separator string to split the records. By default it
+                      uses the breakline character (LF or 0x10).
+                    type: string
+                type: object
             type: object
         type: object
     served: true

diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml
@@ -2451,6 +2451,45 @@ spec:
                     description: Set a regex to exctract fields from the file
                     type: string
                 type: object
+              tcp:
+                description: TCP defines the TCP input plugin configuration
+                properties:
+                  bufferSize:
+                    description: Specify the maximum buffer size in KB to receive
+                      a JSON message. If not set, the default size will be the value
+                      of Chunk_Size.
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  chunkSize:
+                    description: By default the buffer to store the incoming JSON
+                      messages, do not allocate the maximum memory allowed, instead
+                      it allocate memory when is required. The rounds of allocations
+                      are set by Chunk_Size in KB. If not set, Chunk_Size is equal
+                      to 32 (32KB).
+                    pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$
+                    type: string
+                  format:
+                    description: Specify the expected payload format. It support the
+                      options json and none. When using json, it expects JSON maps,
+                      when is set to none, it will split every record using the defined
+                      Separator (option below).
+                    type: string
+                  listen:
+                    description: Listener network interface,default 0.0.0.0
+                    type: string
+                  port:
+                    description: TCP port where listening for connections,default
+                      5170
+                    format: int32
+                    maximum: 65535
+                    minimum: 1
+                    type: integer
+                  separator:
+                    description: When the expected Format is set to none, Fluent Bit
+                      needs a separator string to split the records. By default it
+                      uses the breakline character (LF or 0x10).
+                    type: string
+                type: object
             type: object
         type: object
     served: true