From bb0251e4a7f98af4cf2f0ba61e2003fa6d5e9c41 Mon Sep 17 00:00:00 2001 From: sjliu1 Date: Sat, 23 Sep 2023 14:54:13 +0800 Subject: [PATCH] [summerospp]add fluentbit tcp plugin Signed-off-by: sjliu1 --- apis/fluentbit/v1alpha2/clusterinput_types.go | 2 + apis/fluentbit/v1alpha2/plugins/input/tcp.go | 61 +++++++++++++++++++ .../plugins/input/zz_generated.deepcopy.go | 20 ++++++ .../v1alpha2/zz_generated.deepcopy.go | 5 ++ .../fluentbit.fluent.io_clusterinputs.yaml | 39 ++++++++++++ .../fluentbit.fluent.io_clusterinputs.yaml | 39 ++++++++++++ docs/fluentbit.md | 1 + docs/plugins/fluentbit/input/tcp.md | 13 ++++ manifests/setup/fluent-operator-crd.yaml | 39 ++++++++++++ manifests/setup/setup.yaml | 39 ++++++++++++ 10 files changed, 258 insertions(+) create mode 100644 apis/fluentbit/v1alpha2/plugins/input/tcp.go create mode 100644 docs/plugins/fluentbit/input/tcp.md diff --git a/apis/fluentbit/v1alpha2/clusterinput_types.go b/apis/fluentbit/v1alpha2/clusterinput_types.go index 450e9d625..bef1ed954 100644 --- a/apis/fluentbit/v1alpha2/clusterinput_types.go +++ b/apis/fluentbit/v1alpha2/clusterinput_types.go @@ -69,6 +69,8 @@ type InputSpec struct { Nginx *input.Nginx `json:"nginx,omitempty"` // Syslog defines the Syslog input plugin configuration Syslog *input.Syslog `json:"syslog,omitempty"` + // TCP defines the TCP input plugin configuration + TCP *input.TCP `json:"tcp,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/fluentbit/v1alpha2/plugins/input/tcp.go b/apis/fluentbit/v1alpha2/plugins/input/tcp.go new file mode 100644 index 000000000..5aec983cc --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/input/tcp.go @@ -0,0 +1,61 @@ +package input + +import ( + "fmt" + + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" +) + +// +kubebuilder:object:generate:=true + +// The tcp input plugin allows to retrieve structured JSON or raw messages over a TCP network interface (TCP port). +// **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/tcp** +type TCP struct { + // Listener network interface,default 0.0.0.0 + Listen string `json:"listen,omitempty"` + // TCP port where listening for connections,default 5170 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=65535 + Port *int32 `json:"port,omitempty"` + // Specify the maximum buffer size in KB to receive a JSON message. If not set, the default size will be the value of Chunk_Size. + // +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$" + BufferSize string `json:"bufferSize,omitempty"` + // By default the buffer to store the incoming JSON messages, do not allocate the maximum memory allowed, instead it allocate memory when is required. + //The rounds of allocations are set by Chunk_Size in KB. If not set, Chunk_Size is equal to 32 (32KB). + // +kubebuilder:validation:Pattern:="^\\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$" + ChunkSize string `json:"chunkSize,omitempty"` + // Specify the expected payload format. It support the options json and none. + // When using json, it expects JSON maps, when is set to none, it will split every record using the defined Separator (option below). + Format string `json:"format,omitempty"` + // When the expected Format is set to none, Fluent Bit needs a separator string to split the records. By default it uses the breakline character (LF or 0x10). + Separator string `json:"separator,omitempty"` +} + +func (_ *TCP) Name() string { + return "tcp" +} + +// Params implement Section() method +func (t *TCP) Params(sl plugins.SecretLoader) (*params.KVs, error) { + kvs := params.NewKVs() + if t.Listen != "" { + kvs.Insert("listen", t.Listen) + } + if t.Port != nil { + kvs.Insert("port", fmt.Sprint(*t.Port)) + } + if t.BufferSize != "" { + kvs.Insert("Buffer_Size", t.BufferSize) + } + if t.ChunkSize != "" { + kvs.Insert("Chunk_Size", t.ChunkSize) + } + if t.Format != "" { + kvs.Insert("Format", t.Format) + } + if t.Separator != "" { + kvs.Insert("Separator", t.Separator) + } + return kvs, nil +} diff --git a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go index 8342ce0e7..0f134424b 100644 --- a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go @@ -320,6 +320,26 @@ func (in *Systemd) DeepCopy() *Systemd { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TCP) DeepCopyInto(out *TCP) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TCP. +func (in *TCP) DeepCopy() *TCP { + if in == nil { + return nil + } + out := new(TCP) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Tail) DeepCopyInto(out *Tail) { *out = *in diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index caf372a94..c699b3eb9 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1113,6 +1113,11 @@ func (in *InputSpec) DeepCopyInto(out *InputSpec) { *out = new(input.Syslog) (*in).DeepCopyInto(*out) } + if in.TCP != nil { + in, out := &in.TCP, &out.TCP + *out = new(input.TCP) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InputSpec. diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml index 5759ea7b2..a552a3ef0 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml @@ -702,6 +702,45 @@ spec: description: Set a regex to exctract fields from the file type: string type: object + tcp: + description: TCP defines the TCP input plugin configuration + properties: + bufferSize: + description: Specify the maximum buffer size in KB to receive + a JSON message. If not set, the default size will be the value + of Chunk_Size. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + chunkSize: + description: By default the buffer to store the incoming JSON + messages, do not allocate the maximum memory allowed, instead + it allocate memory when is required. The rounds of allocations + are set by Chunk_Size in KB. If not set, Chunk_Size is equal + to 32 (32KB). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + format: + description: Specify the expected payload format. It support the + options json and none. When using json, it expects JSON maps, + when is set to none, it will split every record using the defined + Separator (option below). + type: string + listen: + description: Listener network interface,default 0.0.0.0 + type: string + port: + description: TCP port where listening for connections,default + 5170 + format: int32 + maximum: 65535 + minimum: 1 + type: integer + separator: + description: When the expected Format is set to none, Fluent Bit + needs a separator string to split the records. By default it + uses the breakline character (LF or 0x10). + type: string + type: object type: object type: object served: true diff --git a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml index 5759ea7b2..a552a3ef0 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml @@ -702,6 +702,45 @@ spec: description: Set a regex to exctract fields from the file type: string type: object + tcp: + description: TCP defines the TCP input plugin configuration + properties: + bufferSize: + description: Specify the maximum buffer size in KB to receive + a JSON message. If not set, the default size will be the value + of Chunk_Size. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + chunkSize: + description: By default the buffer to store the incoming JSON + messages, do not allocate the maximum memory allowed, instead + it allocate memory when is required. The rounds of allocations + are set by Chunk_Size in KB. If not set, Chunk_Size is equal + to 32 (32KB). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + format: + description: Specify the expected payload format. It support the + options json and none. When using json, it expects JSON maps, + when is set to none, it will split every record using the defined + Separator (option below). + type: string + listen: + description: Listener network interface,default 0.0.0.0 + type: string + port: + description: TCP port where listening for connections,default + 5170 + format: int32 + maximum: 65535 + minimum: 1 + type: integer + separator: + description: When the expected Format is set to none, Fluent Bit + needs a separator string to split the records. By default it + uses the breakline character (LF or 0x10). + type: string + type: object type: object type: object served: true diff --git a/docs/fluentbit.md b/docs/fluentbit.md index cf6a37991..70d0f1ef4 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -427,6 +427,7 @@ InputSpec defines the desired state of ClusterInput | statsd | StatsD defines the StatsD input plugin configuration | *[input.StatsD](plugins/input/statsd.md) | | nginx | Nginx defines the Nginx input plugin configuration | *[input.Nginx](plugins/input/nginx.md) | | syslog | Syslog defines the Syslog input plugin configuration | *[input.Syslog](plugins/input/syslog.md) | +| tcp | TCP defines the TCP input plugin configuration | *[input.TCP](plugins/input/tcp.md) | [Back to TOC](#table-of-contents) # NamespacedFluentBitCfgSpec diff --git a/docs/plugins/fluentbit/input/tcp.md b/docs/plugins/fluentbit/input/tcp.md new file mode 100644 index 000000000..e00536fb9 --- /dev/null +++ b/docs/plugins/fluentbit/input/tcp.md @@ -0,0 +1,13 @@ +# TCP + +The tcp input plugin allows to retrieve structured JSON or raw messages over a TCP network interface (TCP port). **For full documentation, refer to https://docs.fluentbit.io/manual/pipeline/inputs/tcp** + + +| Field | Description | Scheme | +| ----- | ----------- | ------ | +| listen | Listener network interface,default 0.0.0.0 | string | +| port | TCP port where listening for connections,default 5170 | *int32 | +| bufferSize | Specify the maximum buffer size in KB to receive a JSON message. If not set, the default size will be the value of Chunk_Size. | string | +| chunkSize | By default the buffer to store the incoming JSON messages, do not allocate the maximum memory allowed, instead it allocate memory when is required. The rounds of allocations are set by Chunk_Size in KB. If not set, Chunk_Size is equal to 32 (32KB). | string | +| format | Specify the expected payload format. It support the options json and none. When using json, it expects JSON maps, when is set to none, it will split every record using the defined Separator (option below). | string | +| separator | When the expected Format is set to none, Fluent Bit needs a separator string to split the records. By default it uses the breakline character (LF or 0x10). | string | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 251d3efb1..4e91d6ed1 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -2451,6 +2451,45 @@ spec: description: Set a regex to exctract fields from the file type: string type: object + tcp: + description: TCP defines the TCP input plugin configuration + properties: + bufferSize: + description: Specify the maximum buffer size in KB to receive + a JSON message. If not set, the default size will be the value + of Chunk_Size. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + chunkSize: + description: By default the buffer to store the incoming JSON + messages, do not allocate the maximum memory allowed, instead + it allocate memory when is required. The rounds of allocations + are set by Chunk_Size in KB. If not set, Chunk_Size is equal + to 32 (32KB). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + format: + description: Specify the expected payload format. It support the + options json and none. When using json, it expects JSON maps, + when is set to none, it will split every record using the defined + Separator (option below). + type: string + listen: + description: Listener network interface,default 0.0.0.0 + type: string + port: + description: TCP port where listening for connections,default + 5170 + format: int32 + maximum: 65535 + minimum: 1 + type: integer + separator: + description: When the expected Format is set to none, Fluent Bit + needs a separator string to split the records. By default it + uses the breakline character (LF or 0x10). + type: string + type: object type: object type: object served: true diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index ad965f276..73abca8f9 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -2451,6 +2451,45 @@ spec: description: Set a regex to exctract fields from the file type: string type: object + tcp: + description: TCP defines the TCP input plugin configuration + properties: + bufferSize: + description: Specify the maximum buffer size in KB to receive + a JSON message. If not set, the default size will be the value + of Chunk_Size. + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + chunkSize: + description: By default the buffer to store the incoming JSON + messages, do not allocate the maximum memory allowed, instead + it allocate memory when is required. The rounds of allocations + are set by Chunk_Size in KB. If not set, Chunk_Size is equal + to 32 (32KB). + pattern: ^\d+(k|K|KB|kb|m|M|MB|mb|g|G|GB|gb)?$ + type: string + format: + description: Specify the expected payload format. It support the + options json and none. When using json, it expects JSON maps, + when is set to none, it will split every record using the defined + Separator (option below). + type: string + listen: + description: Listener network interface,default 0.0.0.0 + type: string + port: + description: TCP port where listening for connections,default + 5170 + format: int32 + maximum: 65535 + minimum: 1 + type: integer + separator: + description: When the expected Format is set to none, Fluent Bit + needs a separator string to split the records. By default it + uses the breakline character (LF or 0x10). + type: string + type: object type: object type: object served: true