From fc4ea21cacd61d9cde6b97aa4db33e4c964a043b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Fri, 25 Oct 2024 12:50:54 +0200 Subject: [PATCH 1/2] feat: filter ordinals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements #1384 Signed-off-by: Zoltán Reegn --- .../fluentbit/v1alpha2/clusterfilter_types.go | 63 ++---- .../v1alpha2/clusterfilter_types_test.go | 206 +++++++++++++++++- .../v1alpha2/clusterfluentbitconfig_types.go | 1 - apis/fluentbit/v1alpha2/filter_types.go | 22 +- apis/fluentbit/v1alpha2/filter_types_test.go | 154 +++++++++---- .../v1alpha2/zz_generated.deepcopy.go | 10 +- .../fluentbit.fluent.io_clusterfilters.yaml | 4 + .../crds/fluentbit.fluent.io_filters.yaml | 4 + .../fluentbit.fluent.io_clusterfilters.yaml | 4 + .../bases/fluentbit.fluent.io_filters.yaml | 4 + docs/fluentbit.md | 1 + go.mod | 4 + go.sum | 6 + manifests/setup/fluent-operator-crd.yaml | 8 + manifests/setup/setup.yaml | 8 + 15 files changed, 389 insertions(+), 110 deletions(-) diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types.go b/apis/fluentbit/v1alpha2/clusterfilter_types.go index 95878b6aa..c7ae407a6 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types.go @@ -44,6 +44,8 @@ type FilterSpec struct { LogLevel string `json:"logLevel,omitempty"` // A set of filter plugins in order. FilterItems []FilterItem `json:"filters,omitempty"` + // An ordinal to influence filter ordering + Ordinal int32 `json:"ordinal,omitempty"` } type FilterItem struct { @@ -101,17 +103,25 @@ type ClusterFilterList struct { // +kubebuilder:object:generate:=false -// FilterByName implements sort.Interface for []ClusterFilter based on the Name field. -type FilterByName []ClusterFilter - -func (a FilterByName) Len() int { return len(a) } -func (a FilterByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a FilterByName) Less(i, j int) bool { return a[i].Name < a[j].Name } +// FilterByOrdinalAndName implements sort.Interface for []ClusterFilter based on the Ordinal and Name field. +type FilterByOrdinalAndName []ClusterFilter + +func (a FilterByOrdinalAndName) Len() int { return len(a) } +func (a FilterByOrdinalAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a FilterByOrdinalAndName) Less(i, j int) bool { + if a[i].Spec.Ordinal < a[j].Spec.Ordinal { + return true + } else if a[i].Spec.Ordinal == a[j].Spec.Ordinal { + return a[i].Name < a[j].Name + } else { + return false + } +} func (list ClusterFilterList) Load(sl plugins.SecretLoader) (string, error) { var buf bytes.Buffer - sort.Sort(FilterByName(list.Items)) + sort.Sort(FilterByOrdinalAndName(list.Items)) for _, item := range list.Items { merge := func(p plugins.Plugin) error { @@ -156,7 +166,7 @@ func (list ClusterFilterList) Load(sl plugins.SecretLoader) (string, error) { func (list ClusterFilterList) LoadAsYaml(sl plugins.SecretLoader, depth int) (string, error) { var buf bytes.Buffer - sort.Sort(FilterByName(list.Items)) + sort.Sort(FilterByOrdinalAndName(list.Items)) if len(list.Items) == 0 { return "", nil } @@ -202,43 +212,6 @@ func (list ClusterFilterList) LoadAsYaml(sl plugins.SecretLoader, depth int) (st return buf.String(), nil } -func (clusterFilter ClusterFilter) LoadAsYaml(sl plugins.SecretLoader, depth int) (string, error) { - var buf bytes.Buffer - padding := utils.YamlIndent(depth + 2) - merge := func(p plugins.Plugin) error { - if p == nil || reflect.ValueOf(p).IsNil() { - return nil - } - - if p.Name() != "" { - buf.WriteString(fmt.Sprintf("%s- name: %s\n", utils.YamlIndent(depth+1), p.Name())) - } - if clusterFilter.Spec.LogLevel != "" { - buf.WriteString(fmt.Sprintf("%slog_level: %s\n", padding, clusterFilter.Spec.LogLevel)) - } - if clusterFilter.Spec.Match != "" { - buf.WriteString(fmt.Sprintf("%smatch: \"%s\"\n", padding, clusterFilter.Spec.Match)) - } - if clusterFilter.Spec.MatchRegex != "" { - buf.WriteString(fmt.Sprintf("%smatch_regex: %s\n", padding, clusterFilter.Spec.MatchRegex)) - } - kvs, err := p.Params(sl) - if err != nil { - return err - } - buf.WriteString(kvs.YamlString(depth + 2)) - return nil - } - for _, elem := range clusterFilter.Spec.FilterItems { - for i := 0; i < reflect.ValueOf(elem).NumField(); i++ { - p, _ := reflect.ValueOf(elem).Field(i).Interface().(plugins.Plugin) - if err := merge(p); err != nil { - return "", err - } - } - } - return buf.String(), nil -} func init() { SchemeBuilder.Register(&ClusterFilter{}, &ClusterFilterList{}) } diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go index 809fc426f..875e4caac 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go @@ -10,7 +10,7 @@ import ( ) func TestClusterFilterList_Load(t *testing.T) { - var filtersExpected = `[Filter] + filtersExpected := `[Filter] Name modify Match logs.foo.bar Condition Key_value_equals kve0 kvev0 @@ -185,8 +185,206 @@ func TestClusterFilterList_Load(t *testing.T) { } } +func TestClusterFilterList_Load_Before(t *testing.T) { + filtersExpected := `[Filter] + Name grep + Match * + Alias third + Regex ^.*$ +[Filter] + Name grep + Match * + Alias first + Regex ^.*$ +` + + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + + filterObj1 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "first", + }, + Spec: FilterSpec{ + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "first", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filterObj2 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "second", + }, + Spec: FilterSpec{ + Ordinal: 10, + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "second", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filterObj3 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "third", + }, + Spec: FilterSpec{ + Ordinal: -10, + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "third", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filters := ClusterFilterList{ + Items: []ClusterFilter{*filterObj1, *filterObj2, *filterObj3}, + } + + i := 0 + for i < 5 { + clusterFilters, err := filters.LoadBefore(sl) + g.Expect(err).NotTo(HaveOccurred()) + + g.Expect(clusterFilters).To(Equal(filtersExpected)) + + i++ + } +} + +func TestClusterFilterList_Load_After(t *testing.T) { + filtersExpected := `[Filter] + Name grep + Match * + Alias second + Regex ^.*$ +` + + g := NewGomegaWithT(t) + sl := plugins.NewSecretLoader(nil, "testnamespace") + + filterObj1 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "first", + }, + Spec: FilterSpec{ + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "first", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filterObj2 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "second", + }, + Spec: FilterSpec{ + Ordinal: 10, + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "second", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filterObj3 := &ClusterFilter{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "fluentbit.fluent.io/v1alpha2", + Kind: "ClusterFilter", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "third", + }, + Spec: FilterSpec{ + Ordinal: -10, + Match: "*", + FilterItems: []FilterItem{ + { + Grep: &filter.Grep{ + CommonParams: plugins.CommonParams{ + Alias: "third", + }, + Regex: "^.*$", + }, + }, + }, + }, + } + + filters := ClusterFilterList{ + Items: []ClusterFilter{*filterObj1, *filterObj2, *filterObj3}, + } + + i := 0 + for i < 5 { + clusterFilters, err := filters.LoadAfter(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(clusterFilters).To(Equal(filtersExpected)) + + i++ + } +} + func TestClusterFilter_RecordModifier_Generated(t *testing.T) { - var filtersExpected = `[Filter] + filtersExpected := `[Filter] Name record_modifier Match logs.foo.bar Record hostname ${HOSTNAME} @@ -260,7 +458,7 @@ func TestClusterFilter_RecordModifier_Generated(t *testing.T) { } func TestClusterFilterList_Load_As_Yaml(t *testing.T) { - var filtersExpected = `filters: + filtersExpected := `filters: - name: modify match: "logs.foo.bar" condition: @@ -438,7 +636,7 @@ func TestClusterFilterList_Load_As_Yaml(t *testing.T) { } func TestClusterFilter_RecordModifier_Generated_Load_As_Yaml(t *testing.T) { - var filtersExpected = `filters: + filtersExpected := `filters: - name: record_modifier match: "logs.foo.bar" record: diff --git a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go index 06661f557..617808a4a 100644 --- a/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go +++ b/apis/fluentbit/v1alpha2/clusterfluentbitconfig_types.go @@ -20,7 +20,6 @@ import ( "bytes" "crypto/md5" "fmt" - "sort" "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" diff --git a/apis/fluentbit/v1alpha2/filter_types.go b/apis/fluentbit/v1alpha2/filter_types.go index d63392c20..4804e0a02 100644 --- a/apis/fluentbit/v1alpha2/filter_types.go +++ b/apis/fluentbit/v1alpha2/filter_types.go @@ -51,16 +51,24 @@ type FilterList struct { Items []Filter `json:"items"` } -type NSFilterByName []Filter - -func (a NSFilterByName) Len() int { return len(a) } -func (a NSFilterByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a NSFilterByName) Less(i, j int) bool { return a[i].Name < a[j].Name } +type NSFilterByOrdinalAndName []Filter + +func (a NSFilterByOrdinalAndName) Len() int { return len(a) } +func (a NSFilterByOrdinalAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a NSFilterByOrdinalAndName) Less(i, j int) bool { + if a[i].Spec.Ordinal < a[j].Spec.Ordinal { + return true + } else if a[i].Spec.Ordinal == a[j].Spec.Ordinal { + return a[i].Name < a[j].Name + } else { + return false + } +} func (list FilterList) Load(sl plugins.SecretLoader) (string, error) { var buf bytes.Buffer - sort.Sort(NSFilterByName(list.Items)) + sort.Sort(NSFilterByOrdinalAndName(list.Items)) for _, item := range list.Items { merge := func(p plugins.Plugin) error { @@ -108,7 +116,7 @@ func (list FilterList) Load(sl plugins.SecretLoader) (string, error) { func (list FilterList) LoadAsYaml(sl plugins.SecretLoader, depth int) (string, error) { var buf bytes.Buffer - sort.Sort(NSFilterByName(list.Items)) + sort.Sort(NSFilterByOrdinalAndName(list.Items)) padding := utils.YamlIndent(depth + 2) for _, item := range list.Items { merge := func(p plugins.Plugin) error { diff --git a/apis/fluentbit/v1alpha2/filter_types_test.go b/apis/fluentbit/v1alpha2/filter_types_test.go index 2fa193978..5a4c1d84a 100644 --- a/apis/fluentbit/v1alpha2/filter_types_test.go +++ b/apis/fluentbit/v1alpha2/filter_types_test.go @@ -13,18 +13,20 @@ func ptr[T any](v T) *T { return &v } func TestFilterList_Load(t *testing.T) { testcases := []struct { name string - input Filter + input []Filter expected string }{ { name: "a single filteritem", - input: Filter{ - Spec: FilterSpec{ - FilterItems: []FilterItem{ - FilterItem{ - Parser: &filter.Parser{ - KeyName: "log", - Parser: "first-parser", + input: []Filter{ + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Parser: &filter.Parser{ + KeyName: "log", + Parser: "first-parser", + }, }, }, }, @@ -38,16 +40,18 @@ func TestFilterList_Load(t *testing.T) { }, { name: "a single filteritem, with multiple plugins", - input: Filter{ - Spec: FilterSpec{ - FilterItems: []FilterItem{ - FilterItem{ - Kubernetes: &filter.Kubernetes{ - KubeTagPrefix: "custom-tag", - }, - Parser: &filter.Parser{ - KeyName: "log", - Parser: "first-parser", + input: []Filter{ + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Kubernetes: &filter.Kubernetes{ + KubeTagPrefix: "custom-tag", + }, + Parser: &filter.Parser{ + KeyName: "log", + Parser: "first-parser", + }, }, }, }, @@ -64,30 +68,32 @@ func TestFilterList_Load(t *testing.T) { }, { name: "multiple filteritems", - input: Filter{ - Spec: FilterSpec{ - FilterItems: []FilterItem{ - FilterItem{ - Kubernetes: &filter.Kubernetes{ - KubeTagPrefix: "custom-tag", + input: []Filter{ + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Kubernetes: &filter.Kubernetes{ + KubeTagPrefix: "custom-tag", + }, + Parser: &filter.Parser{ + KeyName: "log", + Parser: "first-parser", + }, }, - Parser: &filter.Parser{ - KeyName: "log", - Parser: "first-parser", + { + Parser: &filter.Parser{ + KeyName: "msg", + Parser: "second-parser", + ReserveData: ptr(true), + }, }, - }, - FilterItem{ - Parser: &filter.Parser{ - KeyName: "msg", - Parser: "second-parser", - ReserveData: ptr(true), - }, - }, - FilterItem{ - Parser: &filter.Parser{ - KeyName: "msg", - Parser: "third-parser", - ReserveData: ptr(true), + { + Parser: &filter.Parser{ + KeyName: "msg", + Parser: "third-parser", + ReserveData: ptr(true), + }, }, }, }, @@ -110,6 +116,60 @@ func TestFilterList_Load(t *testing.T) { Key_Name msg Parser third-parser-d41d8cd98f00b204e9800998ecf8427e Reserve_Data true +`, + }, + { + name: "ordinal-based sorting", + input: []Filter{ + { + Spec: FilterSpec{ + Ordinal: 10, + FilterItems: []FilterItem{ + { + Parser: &filter.Parser{ + KeyName: "msg", + Parser: "parser-two", + }, + }, + }, + }, + }, + { + Spec: FilterSpec{ + Ordinal: -10, + FilterItems: []FilterItem{ + { + Kubernetes: &filter.Kubernetes{ + KubeTagPrefix: "custom-tag", + }, + }, + }, + }, + }, + { + Spec: FilterSpec{ + FilterItems: []FilterItem{ + { + Parser: &filter.Parser{ + KeyName: "log", + Parser: "parser-one", + }, + }, + }, + }, + }, + }, + expected: `[Filter] + Name kubernetes + Kube_Tag_Prefix d41d8cd98f00b204e9800998ecf8427e.custom-tag +[Filter] + Name parser + Key_Name log + Parser parser-one-d41d8cd98f00b204e9800998ecf8427e +[Filter] + Name parser + Key_Name msg + Parser parser-two-d41d8cd98f00b204e9800998ecf8427e `, }, } @@ -121,10 +181,8 @@ func TestFilterList_Load(t *testing.T) { sl := plugins.NewSecretLoader(nil, "testnamespace") fl := FilterList{ - Items: make([]Filter, 1), + Items: tc.input, } - fl.Items[0] = tc.input - renderedFilterList, err := fl.Load(sl) g.Expect(err).NotTo(HaveOccurred()) g.Expect(renderedFilterList).To(Equal(tc.expected)) @@ -143,7 +201,7 @@ func TestFilterList_LoadAsYaml(t *testing.T) { input: Filter{ Spec: FilterSpec{ FilterItems: []FilterItem{ - FilterItem{ + { Parser: &filter.Parser{ KeyName: "log", Parser: "first-parser", @@ -162,7 +220,7 @@ func TestFilterList_LoadAsYaml(t *testing.T) { input: Filter{ Spec: FilterSpec{ FilterItems: []FilterItem{ - FilterItem{ + { Kubernetes: &filter.Kubernetes{ KubeTagPrefix: "custom-tag", }, @@ -186,7 +244,7 @@ func TestFilterList_LoadAsYaml(t *testing.T) { input: Filter{ Spec: FilterSpec{ FilterItems: []FilterItem{ - FilterItem{ + { Kubernetes: &filter.Kubernetes{ KubeTagPrefix: "custom-tag", }, @@ -195,14 +253,14 @@ func TestFilterList_LoadAsYaml(t *testing.T) { Parser: "first-parser", }, }, - FilterItem{ + { Parser: &filter.Parser{ KeyName: "msg", Parser: "second-parser", ReserveData: ptr(true), }, }, - FilterItem{ + { Parser: &filter.Parser{ KeyName: "msg", Parser: "third-parser", diff --git a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go index 2e1bf58ae..54af9495e 100644 --- a/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go +++ b/apis/fluentbit/v1alpha2/zz_generated.deepcopy.go @@ -1307,22 +1307,22 @@ func (in *MultilineParserSpec) DeepCopy() *MultilineParserSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in NSFilterByName) DeepCopyInto(out *NSFilterByName) { +func (in NSFilterByOrdinalAndName) DeepCopyInto(out *NSFilterByOrdinalAndName) { { in := &in - *out = make(NSFilterByName, len(*in)) + *out = make(NSFilterByOrdinalAndName, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NSFilterByName. -func (in NSFilterByName) DeepCopy() NSFilterByName { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NSFilterByOrdinalAndName. +func (in NSFilterByOrdinalAndName) DeepCopy() NSFilterByOrdinalAndName { if in == nil { return nil } - out := new(NSFilterByName) + out := new(NSFilterByOrdinalAndName) in.DeepCopyInto(out) return *out } diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml index d98d80449..de485ef13 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterfilters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml index 20427043d..10128cf2f 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_filters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml index d98d80449..de485ef13 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusterfilters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/config/crd/bases/fluentbit.fluent.io_filters.yaml b/config/crd/bases/fluentbit.fluent.io_filters.yaml index 20427043d..10128cf2f 100644 --- a/config/crd/bases/fluentbit.fluent.io_filters.yaml +++ b/config/crd/bases/fluentbit.fluent.io_filters.yaml @@ -947,6 +947,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/docs/fluentbit.md b/docs/fluentbit.md index 450dc7fc9..0a90d310f 100644 --- a/docs/fluentbit.md +++ b/docs/fluentbit.md @@ -311,6 +311,7 @@ FilterSpec defines the desired state of ClusterFilter | matchRegex | A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. | string | | logLevel | | string | | filters | A set of filter plugins in order. | [][FilterItem](#filteritem) | +| ordinal | An ordinal to influence filter ordering | int32 | [Back to TOC](#table-of-contents) # FluentBit diff --git a/go.mod b/go.mod index 2f63cff37..47d44a03e 100644 --- a/go.mod +++ b/go.mod @@ -60,12 +60,14 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.34.1 // indirect @@ -74,6 +76,8 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/code-generator v0.30.3 // indirect + k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index ac8b993f2..40f0d6f9d 100644 --- a/go.sum +++ b/go.sum @@ -152,6 +152,8 @@ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= @@ -238,6 +240,10 @@ k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= +k8s.io/code-generator v0.30.3 h1:bmtnLJKagDS5f5uOEpLyJiDfIMKXGMKgOLBdde+w0Mc= +k8s.io/code-generator v0.30.3/go.mod h1:PFgBiv+miFV7TZYp+RXgROkhA+sWYZ+mtpbMLofMke8= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= +k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index b1fe030cd..535f91170 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -946,6 +946,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true @@ -15616,6 +15620,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 3a111c138..019f3a9a2 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -946,6 +946,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true @@ -15616,6 +15620,10 @@ spec: A regular expression to match against the tags of incoming records. Use this option if you want to use the full regex syntax. type: string + ordinal: + description: An ordinal to influence filter ordering + format: int32 + type: integer type: object type: object served: true From 86652be28a148a5878151f5f3159a9da190d0911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 28 Nov 2024 11:31:59 +0100 Subject: [PATCH 2/2] Fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zoltán Reegn --- .../v1alpha2/clusterfilter_types_test.go | 111 ++---------------- 1 file changed, 10 insertions(+), 101 deletions(-) diff --git a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go index 875e4caac..f551e6531 100644 --- a/apis/fluentbit/v1alpha2/clusterfilter_types_test.go +++ b/apis/fluentbit/v1alpha2/clusterfilter_types_test.go @@ -185,113 +185,21 @@ func TestClusterFilterList_Load(t *testing.T) { } } -func TestClusterFilterList_Load_Before(t *testing.T) { +func TestClusterFilterList_Load_With_Ordinals(t *testing.T) { filtersExpected := `[Filter] Name grep Match * - Alias third + Alias first Regex ^.*$ [Filter] Name grep Match * - Alias first + Alias second Regex ^.*$ -` - - g := NewGomegaWithT(t) - sl := plugins.NewSecretLoader(nil, "testnamespace") - - filterObj1 := &ClusterFilter{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "fluentbit.fluent.io/v1alpha2", - Kind: "ClusterFilter", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "first", - }, - Spec: FilterSpec{ - Match: "*", - FilterItems: []FilterItem{ - { - Grep: &filter.Grep{ - CommonParams: plugins.CommonParams{ - Alias: "first", - }, - Regex: "^.*$", - }, - }, - }, - }, - } - - filterObj2 := &ClusterFilter{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "fluentbit.fluent.io/v1alpha2", - Kind: "ClusterFilter", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "second", - }, - Spec: FilterSpec{ - Ordinal: 10, - Match: "*", - FilterItems: []FilterItem{ - { - Grep: &filter.Grep{ - CommonParams: plugins.CommonParams{ - Alias: "second", - }, - Regex: "^.*$", - }, - }, - }, - }, - } - - filterObj3 := &ClusterFilter{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "fluentbit.fluent.io/v1alpha2", - Kind: "ClusterFilter", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "third", - }, - Spec: FilterSpec{ - Ordinal: -10, - Match: "*", - FilterItems: []FilterItem{ - { - Grep: &filter.Grep{ - CommonParams: plugins.CommonParams{ - Alias: "third", - }, - Regex: "^.*$", - }, - }, - }, - }, - } - - filters := ClusterFilterList{ - Items: []ClusterFilter{*filterObj1, *filterObj2, *filterObj3}, - } - - i := 0 - for i < 5 { - clusterFilters, err := filters.LoadBefore(sl) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(clusterFilters).To(Equal(filtersExpected)) - - i++ - } -} - -func TestClusterFilterList_Load_After(t *testing.T) { - filtersExpected := `[Filter] +[Filter] Name grep Match * - Alias second + Alias third Regex ^.*$ ` @@ -312,7 +220,7 @@ func TestClusterFilterList_Load_After(t *testing.T) { { Grep: &filter.Grep{ CommonParams: plugins.CommonParams{ - Alias: "first", + Alias: "second", }, Regex: "^.*$", }, @@ -336,7 +244,7 @@ func TestClusterFilterList_Load_After(t *testing.T) { { Grep: &filter.Grep{ CommonParams: plugins.CommonParams{ - Alias: "second", + Alias: "third", }, Regex: "^.*$", }, @@ -360,7 +268,7 @@ func TestClusterFilterList_Load_After(t *testing.T) { { Grep: &filter.Grep{ CommonParams: plugins.CommonParams{ - Alias: "third", + Alias: "first", }, Regex: "^.*$", }, @@ -375,8 +283,9 @@ func TestClusterFilterList_Load_After(t *testing.T) { i := 0 for i < 5 { - clusterFilters, err := filters.LoadAfter(sl) + clusterFilters, err := filters.Load(sl) g.Expect(err).NotTo(HaveOccurred()) + g.Expect(clusterFilters).To(Equal(filtersExpected)) i++