From cc23366ea3edfe3d84d946a9d1625086c2205901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Fri, 29 Nov 2024 03:56:01 +0100 Subject: [PATCH] feat: filter ordinals (#1386) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: filter ordinals Implements #1384 Signed-off-by: Zoltán Reegn * Fix tests Signed-off-by: Zoltán Reegn --------- Signed-off-by: Zoltán Reegn --- .../fluentbit/v1alpha2/clusterfilter_types.go | 63 ++----- .../v1alpha2/clusterfilter_types_test.go | 115 ++++++++++++- .../v1alpha2/clusterfluentbitconfig_types.go | 1 - apis/fluentbit/v1alpha2/filter_types.go | 22 ++- apis/fluentbit/v1alpha2/filter_types_test.go | 154 ++++++++++++------ .../v1alpha2/zz_generated.deepcopy.go | 10 +- .../fluentbit.fluent.io_clusterfilters.yaml | 4 + .../crds/fluentbit.fluent.io_filters.yaml | 4 + .../fluentbit.fluent.io_clusterfilters.yaml | 4 + .../bases/fluentbit.fluent.io_filters.yaml | 4 + docs/fluentbit.md | 1 + manifests/setup/fluent-operator-crd.yaml | 8 + manifests/setup/setup.yaml | 8 + 13 files changed, 288 insertions(+), 110 deletions(-) diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types.go b/apis/fluentbit/v1alpha2/clusterfilter_types.go index 95878b6aa..c7ae407a6 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types.go @@ -44,6 +44,8 @@ type FilterSpec struct { LogLevel string `json:"logLevel,omitempty"` // A set of filter plugins in order. FilterItems []FilterItem `json:"filters,omitempty"` + // An ordinal to influence filter ordering + Ordinal int32 `json:"ordinal,omitempty"` } type FilterItem struct { @@ -101,17 +103,25 @@ type ClusterFilterList struct { // +kubebuilder:object:generate:=false -// FilterByName implements sort.Interface for []ClusterFilter based on the Name field. -type FilterByName []ClusterFilter - -func (a FilterByName) Len() int { return len(a) } -func (a FilterByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a FilterByName) Less(i, j int) bool { return a[i].Name < a[j].Name } +// FilterByOrdinalAndName implements sort.Interface for []ClusterFilter based on the Ordinal and Name field. +type FilterByOrdinalAndName []ClusterFilter + +func (a FilterByOrdinalAndName) Len() int { return len(a) } +func (a FilterByOrdinalAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a FilterByOrdinalAndName) Less(i, j int) bool { + if a[i].Spec.Ordinal < a[j].Spec.Ordinal { + return true + } else if a[i].Spec.Ordinal == a[j].Spec.Ordinal { + return a[i].Name < a[j].Name + } else { + return false + } +} func (list ClusterFilterList) Load(sl plugins.SecretLoader) (string, error) { var buf bytes.Buffer - sort.Sort(FilterByName(list.Items)) + sort.Sort(FilterByOrdinalAndName(list.Items)) for _, item := range list.Items { merge := func(p plugins.Plugin) error { @@ -156,7 +166,7 @@ func (list ClusterFilterList) Load(sl plugins.SecretLoader) (string, error) { func (list ClusterFilterList) LoadAsYaml(sl plugins.SecretLoader, depth int) (string, error) { var buf bytes.Buffer - sort.Sort(FilterByName(list.Items)) + sort.Sort(FilterByOrdinalAndName(list.Items)) if len(list.Items) == 0 { return "", nil } @@ -202,43 +212,6 @@ func (list ClusterFilterList) LoadAsYaml(sl plugins.SecretLoader, depth int) (st return buf.String(), nil } -func (clusterFilter ClusterFilter) LoadAsYaml(sl plugins.SecretLoader, depth int) (string, error) { - var buf bytes.Buffer - padding := utils.YamlIndent(depth + 2) - merge := func(p plugins.Plugin) error { - if p == nil || reflect.ValueOf(p).IsNil() { - return nil - } - - if p.Name() != "" { - buf.WriteString(fmt.Sprintf("%s- name: %s\n", utils.YamlIndent(depth+1), p.Name())) - } - if clusterFilter.Spec.LogLevel != "" { - buf.WriteString(fmt.Sprintf("%slog_level: %s\n", padding, clusterFilter.Spec.LogLevel)) - } - if clusterFilter.Spec.Match != "" { - buf.WriteString(fmt.Sprintf("%smatch: \"%s\"\n", padding, clusterFilter.Spec.Match)) - } - if clusterFilter.Spec.MatchRegex != "" { - buf.WriteString(fmt.Sprintf("%smatch_regex: %s\n", padding, clusterFilter.Spec.MatchRegex)) - } - kvs, err := p.Params(sl) - if err != nil { - return err - } - buf.WriteString(kvs.YamlString(depth + 2)) - return nil - } - for _, elem := range clusterFilter.Spec.FilterItems { - for i := 0; i < reflect.ValueOf(elem).NumField(); i++ { - p, _ := reflect.ValueOf(elem).Field(i).Interface().(plugins.Plugin) - if err := merge(p); err != nil { - return "", err - } - } - } - return buf.String(), nil -} func init() { SchemeBuilder.Register(&ClusterFilter{}, &ClusterFilterList{}) } diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go index 809fc426f..f551e6531 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go @@ -10,7 +10,7 @@ import ( ) func TestClusterFilterList_Load(t *testing.T) { - var filtersExpected = `[Filter] + filtersExpected := `[Filter] Name modify Match logs.foo.bar Condition Key_value_equals kve0 kvev0 @@ -185,8 +185,115 @@ func TestClusterFilterList_Load(t *testing.T) { } } +func TestClusterFilterList_Load_With_Ordinals(t *testing.T) { + filtersExpected := `[Filter] + Name grep + Match * + Alias first + Regex ^.*$ +[Filter] + Name grep + Match * + Alias second + Regex ^.*$ +[Filter] + Name grep + Match * + Alias third + Regex ^.*$ +` + + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + + filterObj1 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "first", + }, + Spec: FilterSpec{ + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "second", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filterObj2 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "second", + }, + Spec: FilterSpec{ + Ordinal: 10, + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "third", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filterObj3 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "third", + }, + Spec: FilterSpec{ + Ordinal: -10, + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "first", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filters := ClusterFilterList{ + Items: []ClusterFilter{*filterObj1, *filterObj2, *filterObj3}, + } + + i := 0 + for i < 5 { + clusterFilters, err := filters.Load(sl) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(clusterFilters).To(Equal(filtersExpected)) + + i++ + } +} + func TestClusterFilter_RecordModifier_Generated(t *testing.T) { - var filtersExpected = `[Filter] + filtersExpected := `[Filter] Name record_modifier Match logs.foo.bar Record hostname ${HOSTNAME} @@ -260,7 +367,7 @@ func TestClusterFilter_RecordModifier_Generated(t *testing.T) { } func TestClusterFilterList_Load_As_Yaml(t *testing.T) { - var filtersExpected = `filters: + filtersExpected := `filters: - name: modify match: "logs.foo.bar" condition: @@ -438,7 +545,7 @@ func TestClusterFilterList_Load_As_Yaml(t *testing.T) { } func TestClusterFilter_RecordModifier_Generated_Load_As_Yaml(t *testing.T) { - var filtersExpected = `filters: + filtersExpected := `filters: - name: record_modifier match: "logs.foo.bar" record: diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index 17751a919..361ad9939 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -20,7 +20,6 @@ import ( "bytes" "crypto/md5" "fmt" - "sort" "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" diff --git a/apis/fluentbit/v1alpha2/filter_types.go b/apis/fluentbit/v1alpha2/filter_types.go index d63392c20..4804e0a02 100644 --- a/apis/fluentbit/v1alpha2/filter_types.go +++ b/apis/fluentbit/v1alpha2/filter_types.go @@ -51,16 +51,24 @@ type FilterList struct { Items []Filter `json:"items"` } -type NSFilterByName []Filter - -func (a NSFilterByName) Len() int { return len(a) } -func (a NSFilterByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a NSFilterByName) Less(i, j int) bool { return a[i].Name < a[j].Name } +type NSFilterByOrdinalAndName []Filter + +func (a NSFilterByOrdinalAndName) Len() int { return len(a) } +func (a NSFilterByOrdinalAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a NSFilterByOrdinalAndName) Less(i, j int) bool { + if a[i].Spec.Ordinal < a[j].Spec.Ordinal { + return true + } else if a[i].Spec.Ordinal == a[j].Spec.Ordinal { + return a[i].Name < a[j].Name + } else { + return false + } +} func (list FilterList) Load(sl plugins.SecretLoader) (string, error) { var buf bytes.Buffer - sort.Sort(NSFilterByName(list.Items)) + sort.Sort(NSFilterByOrdinalAndName(list.Items)) for _, item := range list.Items { merge := func(p plugins.Plugin) error { @@ -108,7 +116,7 @@ func (list FilterList) Load(sl plugins.SecretLoader) (string, error) { func (list FilterList) LoadAsYaml(sl plugins.SecretLoader, depth int) (string, error) { var buf bytes.Buffer - sort.Sort(NSFilterByName(list.Items)) + sort.Sort(NSFilterByOrdinalAndName(list.Items)) padding := utils.YamlIndent(depth + 2) for _, item := range list.Items { merge := func(p plugins.Plugin) error { diff --git a/apis/fluentbit/v1alpha2/filter_types_test.go b/apis/fluentbit/v1alpha2/filter_types_test.go index 2fa193978..5a4c1d84a 100644 --- a/apis/fluentbit/v1alpha2/filter_types_test.go +++ b/apis/fluentbit/v1alpha2/filter_types_test.go @@ -13,18 +13,20 @@ func ptr[T any](v T) *T { return &v } func TestFilterList_Load(t *testing.T) { testcases := []struct { name string - input Filter + input []Filter expected string }{ { name: "a single filteritem", - input: Filter{ - Spec: FilterSpec{ - FilterItems: []FilterItem{ - FilterItem{ - Parser: &filter.Parser{ - KeyName: "log", - Parser: "first-parser", + input: []Filter{ + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Parser: &filter.Parser{ + KeyName: "log", + Parser: "first-parser", + }, }, }, }, @@ -38,16 +40,18 @@ func TestFilterList_Load(t *testing.T) { }, { name: "a single filteritem, with multiple plugins", - input: Filter{ - Spec: FilterSpec{ - FilterItems: []FilterItem{ - FilterItem{ - Kubernetes: &filter.Kubernetes{ - KubeTagPrefix: "custom-tag", - }, - Parser: &filter.Parser{ - KeyName: "log", - Parser: "first-parser", + input: []Filter{ + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Kubernetes: &filter.Kubernetes{ + KubeTagPrefix: "custom-tag", + }, + Parser: &filter.Parser{ + KeyName: "log", + Parser: "first-parser", + }, }, }, }, @@ -64,30 +68,32 @@ func TestFilterList_Load(t *testing.T) { }, { name: "multiple filteritems", - input: Filter{ - Spec: FilterSpec{ - FilterItems: []FilterItem{ - FilterItem{ - Kubernetes: &filter.Kubernetes{ - KubeTagPrefix: "custom-tag", + input: []Filter{ + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Kubernetes: &filter.Kubernetes{ + KubeTagPrefix: "custom-tag", + }, + Parser: &filter.Parser{ + KeyName: "log", + Parser: "first-parser", + }, }, - Parser: &filter.Parser{ - KeyName: "log", - Parser: "first-parser", + { + Parser: &filter.Parser{ + KeyName: "msg", + Parser: "second-parser", + ReserveData: ptr(true), + }, }, - }, - FilterItem{ - Parser: &filter.Parser{ - KeyName: "msg", - Parser: "second-parser", - ReserveData: ptr(true), - }, - }, - FilterItem{ - Parser: &filter.Parser{ - KeyName: "msg", - Parser: "third-parser", - ReserveData: ptr(true), + { + Parser: &filter.Parser{ + KeyName: "msg", + Parser: "third-parser", + ReserveData: ptr(true), + }, }, }, }, @@ -110,6 +116,60 @@ func TestFilterList_Load(t *testing.T) { Key_Name msg Parser third-parser-d41d8cd98f00b204e9800998ecf8427e Reserve_Data true +`, + }, + { + name: "ordinal-based sorting", + input: []Filter{ + { + Spec: FilterSpec{ + Ordinal: 10, + FilterItems: []FilterItem{ + { + Parser: &filter.Parser{ + KeyName: "msg", + Parser: "parser-two", + }, + }, + }, + }, + }, + { + Spec: FilterSpec{ + Ordinal: -10, + FilterItems: []FilterItem{ + { + Kubernetes: &filter.Kubernetes{ + KubeTagPrefix: "custom-tag", + }, + }, + }, + }, + }, + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Parser: &filter.Parser{ + KeyName: "log", + Parser: "parser-one", + }, + }, + }, + }, + }, + }, + expected: `[Filter] + Name kubernetes + Kube_Tag_Prefix d41d8cd98f00b204e9800998ecf8427e.custom-tag +[Filter] + Name parser + Key_Name log + Parser parser-one-d41d8cd98f00b204e9800998ecf8427e +[Filter] + Name parser + Key_Name msg + Parser parser-two-d41d8cd98f00b204e9800998ecf8427e `, }, } @@ -121,10 +181,8 @@ func TestFilterList_Load(t *testing.T) { sl := plugins.NewSecretLoader(nil, "testnamespace") fl := FilterList{ - Items: make([]Filter, 1), + Items: tc.input, } - fl.Items[0] = tc.input - renderedFilterList, err := fl.Load(sl) g.Expect(err).NotTo(HaveOccurred()) g.Expect(renderedFilterList).To(Equal(tc.expected)) @@ -143,7 +201,7 @@ func TestFilterList_LoadAsYaml(t *testing.T) { input: Filter{ Spec: FilterSpec{ FilterItems: []FilterItem{ - FilterItem{ + { Parser: &filter.Parser{ KeyName: "log", Parser: "first-parser", @@ -162,7 +220,7 @@ func TestFilterList_LoadAsYaml(t *testing.T) { input: Filter{ Spec: FilterSpec{ FilterItems: []FilterItem{ - FilterItem{ + { Kubernetes: &filter.Kubernetes{ KubeTagPrefix: "custom-tag", }, @@ -186,7 +244,7 @@ func TestFilterList_LoadAsYaml(t *testing.T) { input: Filter{ Spec: FilterSpec{ FilterItems: []FilterItem{ - FilterItem{ + { Kubernetes: &filter.Kubernetes{ KubeTagPrefix: "custom-tag", }, @@ -195,14 +253,14 @@ func TestFilterList_LoadAsYaml(t *testing.T) { Parser: "first-parser", }, }, - FilterItem{ + { Parser: &filter.Parser{ KeyName: "msg", Parser: "second-parser", ReserveData: ptr(true), }, }, - FilterItem{ + { Parser: &filter.Parser{ KeyName: "msg", Parser: "third-parser", diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index f0b14564d..128000df0 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1314,22 +1314,22 @@ func (in *MultilineParserSpec) DeepCopy() *MultilineParserSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in NSFilterByName) DeepCopyInto(out *NSFilterByName) { +func (in NSFilterByOrdinalAndName) DeepCopyInto(out *NSFilterByOrdinalAndName) { { in := &in - *out = make(NSFilterByName, len(*in)) + *out = make(NSFilterByOrdinalAndName, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NSFilterByName. -func (in NSFilterByName) DeepCopy() NSFilterByName { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NSFilterByOrdinalAndName. +func (in NSFilterByOrdinalAndName) DeepCopy() NSFilterByOrdinalAndName { if in == nil { return nil } - out := new(NSFilterByName) + out := new(NSFilterByOrdinalAndName) in.DeepCopyInto(out) return *out } diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index d98d80449..de485ef13 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index 20427043d..10128cf2f 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index d98d80449..de485ef13 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index 20427043d..10128cf2f 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/docs/fluentbit.md b/docs/fluentbit.md index 12380bdce..b60dedb89 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -311,6 +311,7 @@ FilterSpec defines the desired state of ClusterFilter | matchRegex | A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. | string | | logLevel | | string | | filters | A set of filter plugins in order. | [][FilterItem](#filteritem) | +| ordinal | An ordinal to influence filter ordering | int32 | [Back to TOC](#table-of-contents) # FluentBit diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index 31ab9a705..16b0c26ed 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -946,6 +946,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true @@ -15619,6 +15623,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 23f509b90..774dd97ef 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -946,6 +946,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true @@ -15619,6 +15623,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true