Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix fd record transformer parameters #960

Merged
merged 2 commits into from
Oct 23, 2023
Merged

fix: fix fd record transformer parameters #960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cf3b539
fix autoTypeCast & renewRecord
cw-Guo Oct 22, 2023
16b883d
add tests
cw-Guo Oct 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion apis/fluentd/v1alpha1/plugins/filter/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,10 @@ func (f *Filter) recordTransformerPlugin(parent *params.PluginStore, loader plug
parent.InsertPairs("enable_ruby", fmt.Sprint(*f.RecordTransformer.EnableRuby))
}
if f.RecordTransformer.AutoTypeCast != nil {
parent.InsertPairs("renew_record", fmt.Sprint(*f.RecordTransformer.AutoTypeCast))
parent.InsertPairs("auto_typecast", fmt.Sprint(*f.RecordTransformer.AutoTypeCast))
}
if f.RecordTransformer.RenewRecord != nil {
parent.InsertPairs("renew_record", fmt.Sprint(*f.RecordTransformer.RenewRecord))
}
if f.RecordTransformer.RenewTimeKey != nil {
parent.InsertPairs("renew_time_key", fmt.Sprint(*f.RecordTransformer.RenewTimeKey))
Expand Down
35 changes: 35 additions & 0 deletions apis/fluentd/v1alpha1/tests/expected/fluentd-cluster-cfg-filter-recordTransformer.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<source>
@type forward
bind 0.0.0.0
port 24224
</source>
<match **>
@id main
@type label_router
<route>
@label @a2170d34e9940ec56d328100e375c43e
<match>
namespaces default,kube-system
</match>
</route>
</match>
<label @a2170d34e9940ec56d328100e375c43e>
<filter **>
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusterfilter::fluentd-filter-0
@type record_transformer
auto_typecast true
enable_ruby true
renew_record true
<record>
kubernetes_ns ${record["kubernetes"]["namespace_name"]}
</record>
</filter>
<match **>
@id ClusterFluentdConfig-cluster-fluentd-config::cluster::clusteroutput::fluentd-output-cluster-0
@type elasticsearch
host elasticsearch-logging-data.kubesphere-logging-system.svc
logstash_format true
logstash_prefix ks-logstash-log
port 9200
</match>
</label>
27 changes: 27 additions & 0 deletions apis/fluentd/v1alpha1/tests/helper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,3 +607,30 @@ func Test_DuplicateRemovalCRSpecs(t *testing.T) {
i++
}
}

func Test_RecordTransformer(t *testing.T) {
g := NewGomegaWithT(t)
sl := plugins.NewSecretLoader(nil, Fluentd.Namespace, logr.Logger{})

psr := fluentdv1alpha1.NewGlobalPluginResources("main")
psr.CombineGlobalInputsPlugins(sl, Fluentd.Spec.GlobalInputs)

clustercfgRouter, err := psr.BuildCfgRouter(&FluentdClusterFluentdConfig1)
g.Expect(err).NotTo(HaveOccurred())
clusterFilters := []fluentdv1alpha1.ClusterFilter{FluentdClusterRecordTransformerFilter}
clusterOutputs := []fluentdv1alpha1.ClusterOutput{FluentdClusterOutputCluster}
clustercfgResources, _ := psr.PatchAndFilterClusterLevelResources(sl, FluentdClusterFluentdConfig1.GetCfgId(), clusterFilters, clusterOutputs)
err = psr.WithCfgResources(*clustercfgRouter.Label, clustercfgResources)
g.Expect(err).NotTo(HaveOccurred())

// we should not see any permutations in serialized config
i := 0
for i < maxRuntimes {
config, errs := psr.RenderMainConfig(false)
// fmt.Println(config)
g.Expect(errs).NotTo(HaveOccurred())
g.Expect(string(getExpectedCfg("./expected/fluentd-cluster-cfg-filter-recordTransformer.cfg"))).To(Equal(config))

i++
}
}
20 changes: 20 additions & 0 deletions apis/fluentd/v1alpha1/tests/tools.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,25 @@ spec:
value: ${record["kubernetes"]["namespace_name"]}
`

FluentdClusterRecordTransformerFilter fluentdv1alpha1.ClusterFilter
FluentdClusterRecordTransformerRaw = `
apiVersion: fluentd.fluent.io/v1alpha1
kind: ClusterFilter
metadata:
name: fluentd-filter
labels:
filter.fluentd.fluent.io/enabled: "true"
spec:
filters:
- recordTransformer:
enableRuby: true
autoTypeCast: true
renewRecord: true
records:
- key: kubernetes_ns
value: ${record["kubernetes"]["namespace_name"]}
`

FluentdClusterOutputBuffer fluentdv1alpha1.ClusterOutput
FluentdClusterOutputBufferRaw = `
apiVersion: fluentd.fluent.io/v1alpha1
Expand Down Expand Up @@ -562,6 +581,7 @@ func init() {
ParseIntoObject(FluentdConfigUser1Raw, &FluentdConfigUser1)
ParseIntoObject(FluentdConfig1Raw, &FluentdConfig1)
ParseIntoObject(FluentdClusterFilter1Raw, &FluentdClusterFilter1)
ParseIntoObject(FluentdClusterRecordTransformerRaw, &FluentdClusterRecordTransformerFilter)
ParseIntoObject(FluentdClusterOutputClusterRaw, &FluentdClusterOutputCluster)
ParseIntoObject(FluentdClusterOutputLogOperatorRaw, &FluentdClusterOutputLogOperator)
ParseIntoObject(FluentdClusterOutputBufferRaw, &FluentdClusterOutputBuffer)
Expand Down
Loading