diff --git a/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go b/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go index b64e839a5..0911ada38 100644 --- a/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go +++ b/apis/fluentbit/v1alpha2/plugins/filter/lua_types.go @@ -1,9 +1,9 @@ package filter import ( + "regexp" "strconv" "strings" - "regexp" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins" "github.com/fluent/fluent-operator/v2/apis/fluentbit/v1alpha2/plugins/params" @@ -17,12 +17,12 @@ import ( type Lua struct { plugins.CommonParams `json:",inline"` // Path to the Lua script that will be used. - Script v1.ConfigMapKeySelector `json:"script"` + Script v1.ConfigMapKeySelector `json:"script,omitempty"` // Lua function name that will be triggered to do filtering. // It's assumed that the function is declared inside the Script defined above. Call string `json:"call"` // Inline LUA code instead of loading from a path via script. - Code string `json:"code"` + Code string `json:"code,omitempty"` // If these keys are matched, the fields are converted to integer. // If more than one key, delimit by space. // Note that starting from Fluent Bit v1.6 integer data types are preserved diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index 5bc319a7b..e327e3227 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -345,8 +345,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index dd166771a..1a0145ba8 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -345,8 +345,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. diff --git a/charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml b/charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml new file mode 100644 index 000000000..091f5d402 --- /dev/null +++ b/charts/fluent-operator/templates/fluentbit-clusterfilter-multiline.yaml @@ -0,0 +1,19 @@ +{{- if .Values.Kubernetes -}} +{{- if .Values.fluentbit.enable -}} +{{- if .Values.fluentbit.filter.multiline.enable -}} +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterFilter +metadata: + name: multiline + labels: + fluentbit.fluent.io/enabled: "true" + fluentbit.fluent.io/component: logging +spec: + match: kube.* + filters: + - multiline: + keyContent: {{ .Values.fluentbit.filter.multiline.keyContent | quote }} + parser: "{{- join "," .Values.fluentbit.filter.multiline.parsers -}}" +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml index 466aa6434..8f703224c 100644 --- a/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml +++ b/charts/fluent-operator/templates/fluentbit-clusterinput-tail.yaml @@ -20,6 +20,9 @@ spec: {{- else if eq .Values.containerRuntime "crio" }} parser: cri {{- end }} + {{- if .Values.fluentbit.input.tail.multilineParser }} + multilineParser: {{ .Values.fluentbit.input.tail.multilineParser | quote }} + {{- end }} refreshIntervalSeconds: {{ .Values.fluentbit.input.tail.refreshIntervalSeconds }} memBufLimit: {{ .Values.fluentbit.input.tail.memBufLimit }} {{- if .Values.fluentbit.input.tail.bufferMaxSize }} diff --git a/charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml b/charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml new file mode 100644 index 000000000..2cd02efa2 --- /dev/null +++ b/charts/fluent-operator/templates/fluentbit-multilineParser-javaMultiline.yaml @@ -0,0 +1,24 @@ +{{- if .Values.Kubernetes -}} +{{- if .Values.fluentbit.enable -}} +{{- if .Values.fluentbit.parsers.javaMultiline.enable -}} +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterMultilineParser +metadata: + name: java-multiline + labels: + fluentbit.fluent.io/enabled: "true" + fluentbit.fluent.io/component: logging +spec: + type: "regex" + flushTimeout: 1000 + keyContent: "log" + rules: + - start: "start_state" + regex: '/\[?(\d+\-\d+\-\d+ \d+\:\d+\:\d+(\.\d+)?)\]? /' + next: "cont" + - start: "cont" + regex: '/^(?!\[?\d+\-\d+\-\d+).*/' + next: "cont" +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml index 112da8304..2bf58c70c 100644 --- a/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml +++ b/charts/fluent-operator/templates/fluentbitconfig-fluentBitConfig.yaml @@ -8,7 +8,9 @@ metadata: app.kubernetes.io/name: fluent-bit spec: service: - parsersFile: parsers.conf + parsersFiles: + - /fluent-bit/config/parsers.conf + - /fluent-bit/config/parsers_multiline.conf httpServer: true {{- if .Values.fluentbit.logLevel }} logLevel: {{ .Values.fluentbit.logLevel }} @@ -29,5 +31,8 @@ spec: outputSelector: matchLabels: fluentbit.fluent.io/enabled: "true" + multilineParserSelector: + matchLabels: + fluentbit.fluent.io/enabled: "true" {{- end }} {{- end }} diff --git a/charts/fluent-operator/values.yaml b/charts/fluent-operator/values.yaml index 67e93f861..0e6cdc7bc 100644 --- a/charts/fluent-operator/values.yaml +++ b/charts/fluent-operator/values.yaml @@ -183,6 +183,8 @@ fluentbit: # Use storageType as "filesystem" if you want to use filesystem as the buffering mechanism for tail input. storageType: memory pauseOnChunksOverlimit: "off" + # multiline.parser + # multilineParser: "docker, cri" systemd: enable: true systemdFilter: @@ -319,6 +321,15 @@ fluentbit: # Configure the default filters in FluentBit. # The `filter` will filter and parse the collected log information and output the logs into a uniform format. You can choose whether to turn this on or not. filter: + multiline: + enable: false + keyContent: log + parsers: + - go + - python + - java + # use custom multiline parser need set .Values.parsers.javaMultiline.enable = true + # - java-multiline kubernetes: enable: true labels: false @@ -342,6 +353,11 @@ fluentbit: # removes the hostPath mounts for varlibcontainers, varlogs and systemd. disableLogVolumes: false + parsers: + javaMultiline: + # use in filter for parser generic springboot multiline log format + enable: false + fluentd: # Installs a sub chart carrying the CRDs for the fluentd controller. The sub chart is enabled by default. crdsEnable: true diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index 5bc319a7b..e327e3227 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -345,8 +345,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index dd166771a..1a0145ba8 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -345,8 +345,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. diff --git a/config/samples/fluentbit_v1alpha2_clustermultilineparser.yaml b/config/samples/fluentbit_v1alpha2_clustermultilineparser.yaml new file mode 100644 index 000000000..bed6ff16e --- /dev/null +++ b/config/samples/fluentbit_v1alpha2_clustermultilineparser.yaml @@ -0,0 +1,18 @@ +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterMultilineParser +metadata: + name: java-multiline + labels: + fluentbit.fluent.io/enabled: "true" + fluentbit.fluent.io/component: logging +spec: + type: "regex" + flushTimeout: 1000 + keyContent: "log" + rules: + - start: "start_state" + regex: '/\[?(\d+\-\d+\-\d+ \d+\:\d+\:\d+(\.\d+)?)\]? /' + next: "cont" + - start: "cont" + regex: '/^(?!\[?\d+\-\d+\-\d+).*/' + next: "cont" \ No newline at end of file diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 69259298c..9a9c61a10 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -343,8 +343,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. @@ -11438,8 +11436,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 845c0f7fd..999f40b50 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -343,8 +343,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration. @@ -11438,8 +11436,6 @@ spec: type: array required: - call - - code - - script type: object modify: description: Modify defines Modify Filter configuration.