diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml index 056e23fff..202878b05 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_clusterflows.yaml @@ -965,6 +965,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: @@ -1992,6 +2003,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_flows.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_flows.yaml index 13cfcdbdd..c30b4ea59 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_flows.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_flows.yaml @@ -965,6 +965,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: @@ -1988,6 +1999,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml index 0f7503156..29628b155 100644 --- a/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml +++ b/charts/logging-operator/crds/logging.banzaicloud.io_loggings.yaml @@ -1011,6 +1011,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: @@ -7032,6 +7043,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array loggingRef: diff --git a/config/crd/bases/logging.banzaicloud.io_clusterflows.yaml b/config/crd/bases/logging.banzaicloud.io_clusterflows.yaml index 056e23fff..202878b05 100644 --- a/config/crd/bases/logging.banzaicloud.io_clusterflows.yaml +++ b/config/crd/bases/logging.banzaicloud.io_clusterflows.yaml @@ -965,6 +965,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: @@ -1992,6 +2003,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: diff --git a/config/crd/bases/logging.banzaicloud.io_flows.yaml b/config/crd/bases/logging.banzaicloud.io_flows.yaml index 13cfcdbdd..c30b4ea59 100644 --- a/config/crd/bases/logging.banzaicloud.io_flows.yaml +++ b/config/crd/bases/logging.banzaicloud.io_flows.yaml @@ -965,6 +965,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: @@ -1988,6 +1999,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: diff --git a/config/crd/bases/logging.banzaicloud.io_loggings.yaml b/config/crd/bases/logging.banzaicloud.io_loggings.yaml index 0f7503156..29628b155 100644 --- a/config/crd/bases/logging.banzaicloud.io_loggings.yaml +++ b/config/crd/bases/logging.banzaicloud.io_loggings.yaml @@ -1011,6 +1011,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array flowLabel: @@ -7032,6 +7043,17 @@ spec: group_warning_delay_s: type: integer type: object + useragent: + properties: + delete_key: + type: boolean + flatten: + type: boolean + key_name: + type: string + out_key: + type: string + type: object type: object type: array loggingRef: diff --git a/docs/configuration/crds/v1beta1/flow_types.md b/docs/configuration/crds/v1beta1/flow_types.md index 500a888a9..315742ed2 100644 --- a/docs/configuration/crds/v1beta1/flow_types.md +++ b/docs/configuration/crds/v1beta1/flow_types.md @@ -125,6 +125,9 @@ Filter definition for FlowSpec ### throttle (*filter.Throttle, optional) {#filter-throttle} +### useragent (*filter.UserAgent, optional) {#filter-useragent} + + ## FlowStatus diff --git a/docs/configuration/plugins/_index.md b/docs/configuration/plugins/_index.md index a819d9cd1..adf4ac752 100644 --- a/docs/configuration/plugins/_index.md +++ b/docs/configuration/plugins/_index.md @@ -26,6 +26,7 @@ For more information please click on the plugin name | **[SumoLogic](filters/sumologic/)** | filters | Sumo Logic collection solution for Kubernetes | GA | [2.3.1](https://github.com/SumoLogic/sumologic-kubernetes-collection) | | **[Tag Normaliser](filters/tagnormaliser/)** | filters | Re-tag based on log metadata | GA | [0.1.1](https://github.com/kube-logging/fluent-plugin-tag-normaliser) | | **[Throttle](filters/throttle/)** | filters | A sentry plugin to throttle logs. Logs are grouped by a configurable key. When a group exceeds a configuration rate, logs are dropped for this group. | GA | [0.0.5](https://github.com/rubrikinc/fluent-plugin-throttle) | +| **[UserAgent](filters/useragent/)** | filters | Fluentd UserAgent filter | GA | [1.2.0](https://github.com/bungoume/fluent-plugin-ua-parser) | | **[Amazon Elasticsearch](outputs/aws_elasticsearch/)** | outputs | Fluent plugin for Amazon Elasticsearch | Testing | [2.4.1](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service) | | **[Azure Storage](outputs/azurestore/)** | outputs | Store logs in Azure Storage | GA | [0.2.1](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) | | **[Buffer](outputs/buffer/)** | outputs | Fluentd event buffer | GA | [mode info](https://docs.fluentd.org/configuration/buffer-section) | diff --git a/docs/configuration/plugins/filters/useragent.md b/docs/configuration/plugins/filters/useragent.md new file mode 100644 index 000000000..d1b142e37 --- /dev/null +++ b/docs/configuration/plugins/filters/useragent.md @@ -0,0 +1,72 @@ +--- +title: User Agent +weight: 200 +generated_file: true +--- + +# Fluentd UserAgent filter +## Overview + Fluentd Filter plugin to parse user-agent + More information at https://github.com/bungoume/fluent-plugin-ua-parser + +## Configuration +## UserAgent + +### delete_key (bool, optional) {#useragent-delete_key} + +Delete input key + +Default: false + +### flatten (bool, optional) {#useragent-flatten} + +Join hashed data by '_' + +Default: false + +### key_name (string, optional) {#useragent-key_name} + +Target key name + +Default: user_agent + +### out_key (string, optional) {#useragent-out_key} + +Output prefix key name + +Default: ua + + + ## Example `UserAgent` filter configurations + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - useragent: + key_name: my_agent + delete_key: true + out_key: ua_fields + flatten: true + selectors: {} + localOutputRefs: + - demo-output +{{}} + +## Fluentd Config Result +{{< highlight xml >}} + + @type ua_parser + @id test_useragent + key_name my_agent + delete_key true + out_key ua_fields + flatten true + +{{}} + + +--- diff --git a/pkg/sdk/logging/api/v1beta1/flow_types.go b/pkg/sdk/logging/api/v1beta1/flow_types.go index 138619a60..7103a2e3d 100644 --- a/pkg/sdk/logging/api/v1beta1/flow_types.go +++ b/pkg/sdk/logging/api/v1beta1/flow_types.go @@ -70,6 +70,7 @@ type Filter struct { RecordTransformer *filter.RecordTransformer `json:"record_transformer,omitempty"` RecordModifier *filter.RecordModifier `json:"record_modifier,omitempty"` GeoIP *filter.GeoIP `json:"geoip,omitempty"` + UserAgent *filter.UserAgent `json:"useragent,omitempty"` Concat *filter.Concat `json:"concat,omitempty"` DetectExceptions *filter.DetectExceptions `json:"detectExceptions,omitempty"` Grep *filter.GrepConfig `json:"grep,omitempty"` diff --git a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go index dcda008c4..d44b9784c 100644 --- a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go @@ -507,6 +507,11 @@ func (in *Filter) DeepCopyInto(out *Filter) { *out = new(filter.GeoIP) (*in).DeepCopyInto(*out) } + if in.UserAgent != nil { + in, out := &in.UserAgent, &out.UserAgent + *out = new(filter.UserAgent) + **out = **in + } if in.Concat != nil { in, out := &in.Concat, &out.Concat *out = new(filter.Concat) diff --git a/pkg/sdk/logging/model/filter/useragent.go b/pkg/sdk/logging/model/filter/useragent.go new file mode 100644 index 000000000..661d5c98e --- /dev/null +++ b/pkg/sdk/logging/model/filter/useragent.go @@ -0,0 +1,100 @@ +// Copyright © 2023 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filter + +import ( + "github.com/cisco-open/operator-tools/pkg/secret" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/types" +) + +// +name:"User Agent" +// +weight:"200" +type _hugoUserAgent interface{} //nolint:deadcode,unused + +// +docName:"Fluentd UserAgent filter" +// Fluentd Filter plugin to parse user-agent +// More information at https://github.com/bungoume/fluent-plugin-ua-parser +type _docUserAgent interface{} //nolint:deadcode,unused + +// +name:"UserAgent" +// +url:"https://github.com/bungoume/fluent-plugin-ua-parser" +// +version:"1.2.0" +// +description:"Fluentd UserAgent filter" +// +status:"GA" +type _metaUserAgent interface{} //nolint:deadcode,unused + +// +kubebuilder:object:generate=true +type UserAgent struct { + // Target key name (default: user_agent) + KeyName string `json:"key_name,omitempty"` + // Delete input key (default: false) + DeleteKey bool `json:"delete_key,omitempty"` + // Output prefix key name (default: ua) + OutKey string `json:"out_key,omitempty"` + // Join hashed data by '_' (default: false) + Flatten bool `json:"flatten,omitempty"` +} + +// ## Example `UserAgent` filter configurations +/* +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: demo-flow +spec: + filters: + - useragent: + key_name: my_agent + delete_key: true + out_key: ua_fields + flatten: true + selectors: {} + localOutputRefs: + - demo-output +{{}} + +## Fluentd Config Result +{{< highlight xml >}} + + @type ua_parser + @id test_useragent + key_name my_agent + delete_key true + out_key ua_fields + flatten true + +{{}} +*/ +type _expUserAgent interface{} //nolint:deadcode,unused + +func (g *UserAgent) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) { + const pluginType = "ua_parser" + userAgent := &types.GenericDirective{ + PluginMeta: types.PluginMeta{ + Type: pluginType, + Directive: "filter", + Tag: "**", + Id: id, + }, + } + if params, err := types.NewStructToStringMapper(secretLoader).StringsMap(g); err != nil { + return nil, err + } else { + userAgent.Params = params + } + return userAgent, nil +} diff --git a/pkg/sdk/logging/model/filter/useragent_test.go b/pkg/sdk/logging/model/filter/useragent_test.go new file mode 100644 index 000000000..658bd2092 --- /dev/null +++ b/pkg/sdk/logging/model/filter/useragent_test.go @@ -0,0 +1,48 @@ +// Copyright © 2023 Kube logging authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package filter_test + +import ( + "testing" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/require" + + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/filter" + "github.com/kube-logging/logging-operator/pkg/sdk/logging/model/render" +) + +func TestUserAgent(t *testing.T) { + CONFIG := []byte(` +key_name: ua_string +delete_key: true +out_key: ua +flatten: true +`) + expected := ` + + @type ua_parser + @id test + delete_key true + flatten true + key_name ua_string + out_key ua + +` + parser := &filter.UserAgent{} + require.NoError(t, yaml.Unmarshal(CONFIG, parser)) + test := render.NewOutputPluginTest(t, parser) + test.DiffResult(expected) +} diff --git a/pkg/sdk/logging/model/filter/zz_generated.deepcopy.go b/pkg/sdk/logging/model/filter/zz_generated.deepcopy.go index e5cfa1dcc..a7f2c57ed 100644 --- a/pkg/sdk/logging/model/filter/zz_generated.deepcopy.go +++ b/pkg/sdk/logging/model/filter/zz_generated.deepcopy.go @@ -594,3 +594,18 @@ func (in *Throttle) DeepCopy() *Throttle { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserAgent) DeepCopyInto(out *UserAgent) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserAgent. +func (in *UserAgent) DeepCopy() *UserAgent { + if in == nil { + return nil + } + out := new(UserAgent) + in.DeepCopyInto(out) + return out +}