From 4b2aa28783dd8171bafa4f199e90318c75a3e11b Mon Sep 17 00:00:00 2001 From: yilmazo Date: Fri, 15 Nov 2024 14:41:58 +0300 Subject: [PATCH] feat: add support for logs_body_key parameter on Opentelemetry output plugin (https://github.com/fluent/fluent-operator/issues/1410) Signed-off-by: yilmazo --- .../plugins/output/open_telemetry_types.go | 7 +- .../output/open_telemetry_types_test.go | 64 +++++++++++++++++++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 ++ .../crds/fluentbit.fluent.io_outputs.yaml | 4 ++ .../fluentbit.fluent.io_clusteroutputs.yaml | 4 ++ .../bases/fluentbit.fluent.io_outputs.yaml | 4 ++ .../fluentbit/output/open_telemetry.md | 1 + manifests/setup/fluent-operator-crd.yaml | 8 +++ manifests/setup/setup.yaml | 8 +++ 9 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types_test.go diff --git a/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types.go b/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types.go index de25b7be7..2f6361898 100644 --- a/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types.go +++ b/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types.go @@ -39,7 +39,9 @@ type OpenTelemetry struct { AddLabel map[string]string `json:"addLabel,omitempty"` // If true, remaining unmatched keys are added as attributes. LogsBodyKeyAttributes *bool `json:"logsBodyKeyAttributes,omitempty"` - *plugins.TLS `json:"tls,omitempty"` + // The log body key to look up in the log events body/message. Sets the Body field of the opentelemtry logs data model. + LogsBodyKey string `json:"logsBodyKey,omitempty"` + *plugins.TLS `json:"tls,omitempty"` // Include fluentbit networking options for this output-plugin *plugins.Networking `json:"networking,omitempty"` } @@ -96,6 +98,9 @@ func (o *OpenTelemetry) Params(sl plugins.SecretLoader) (*params.KVs, error) { if o.LogsBodyKeyAttributes != nil { kvs.Insert("logs_body_key_attributes", fmt.Sprint(*o.LogsBodyKeyAttributes)) } + if o.LogsBodyKey != "" { + kvs.Insert("logs_body_key", o.LogsBodyKey) + } if o.TLS != nil { tls, err := o.TLS.Params(sl) if err != nil { diff --git a/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types_test.go b/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types_test.go new file mode 100644 index 000000000..80902d14f --- /dev/null +++ b/apis/fluentbit/v1alpha2/plugins/output/open_telemetry_types_test.go @@ -0,0 +1,64 @@ +package output + +import ( + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins" + "github.com/fluent/fluent-operator/v3/apis/fluentbit/v1alpha2/plugins/params" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestOpenTelemetry_Params(t *testing.T) { + g := NewGomegaWithT(t) + fcb := fake.ClientBuilder{} + fc := fcb.WithObjects(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "test_namespace", Name: "http_secret"}, + Data: map[string][]byte{ + "http_user": []byte("expected_http_user"), + "http_passwd": []byte("expected_http_passwd"), + }, + }).Build() + + sl := plugins.NewSecretLoader(fc, "test_namespace") + ot := OpenTelemetry{ + Host: "otlp-collector.example.com", + Port: ptrAny(int32(443)), + HTTPUser: &plugins.Secret{ValueFrom: plugins.ValueSource{SecretKeyRef: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "http_secret"}, Key: "http_user"}}}, + HTTPPasswd: &plugins.Secret{ValueFrom: plugins.ValueSource{SecretKeyRef: v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: "http_secret"}, Key: "http_passwd"}}}, + Proxy: "expected_proxy", + MetricsUri: "expected_metrics_uri", + LogsUri: "expected_logs_uri", + TracesUri: "expected_traces_uri", + Header: map[string]string{"custom_header_key": "custom_header_val"}, + LogResponsePayload: ptrBool(true), + AddLabel: map[string]string{"add_label_key": "add_label_val"}, + LogsBodyKeyAttributes: ptrBool(true), + LogsBodyKey: "expected_logs_body_key", + TLS: &plugins.TLS{Verify: ptrBool(false)}, + Networking: &plugins.Networking{SourceAddress: ptrAny("expected_source_address")}, + } + + expected := params.NewKVs() + expected.Insert("host", "otlp-collector.example.com") + expected.Insert("port", "443") + expected.Insert("http_user", "expected_http_user") + expected.Insert("http_passwd", "expected_http_passwd") + expected.Insert("proxy", "expected_proxy") + expected.Insert("metrics_uri", "expected_metrics_uri") + expected.Insert("logs_uri", "expected_logs_uri") + expected.Insert("traces_uri", "expected_traces_uri") + expected.Insert("header", " custom_header_key custom_header_val") + expected.Insert("log_response_payload", "true") + expected.Insert("add_label", " add_label_key add_label_val") + expected.Insert("logs_body_key_attributes", "true") + expected.Insert("logs_body_key", "expected_logs_body_key") + expected.Insert("tls", "On") + expected.Insert("tls.verify", "false") + expected.Insert("net.source_address", "expected_source_address") + + kvs, err := ot.Params(sl) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(kvs).To(Equal(expected)) +} diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml index b434232bd..54e6eef18 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusteroutputs.yaml @@ -2930,6 +2930,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean diff --git a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml index b3b11db98..c992e204f 100644 --- a/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml +++ b/charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_outputs.yaml @@ -2930,6 +2930,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean diff --git a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml index b434232bd..54e6eef18 100644 --- a/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_clusteroutputs.yaml @@ -2930,6 +2930,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean diff --git a/config/crd/bases/fluentbit.fluent.io_outputs.yaml b/config/crd/bases/fluentbit.fluent.io_outputs.yaml index b3b11db98..c992e204f 100644 --- a/config/crd/bases/fluentbit.fluent.io_outputs.yaml +++ b/config/crd/bases/fluentbit.fluent.io_outputs.yaml @@ -2930,6 +2930,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean diff --git a/docs/plugins/fluentbit/output/open_telemetry.md b/docs/plugins/fluentbit/output/open_telemetry.md index 064271ed3..d372c73be 100644 --- a/docs/plugins/fluentbit/output/open_telemetry.md +++ b/docs/plugins/fluentbit/output/open_telemetry.md @@ -17,5 +17,6 @@ The OpenTelemetry plugin allows you to take logs, metrics, and traces from Fluen | logResponsePayload | Log the response payload within the Fluent Bit log. | *bool | | addLabel | This allows you to add custom labels to all metrics exposed through the OpenTelemetry exporter. You may have multiple of these fields. | map[string]string | | logsBodyKeyAttributes | If true, remaining unmatched keys are added as attributes. | *bool | +| logsBodyKey | The log body key to look up in the log events body/message. Sets the Body field of the opentelemtry logs data model. | string | | tls | | *[plugins.TLS](../tls.md) | | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking | diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml index c7723c253..4774f086a 100644 --- a/manifests/setup/fluent-operator-crd.yaml +++ b/manifests/setup/fluent-operator-crd.yaml @@ -6966,6 +6966,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean @@ -35656,6 +35660,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean diff --git a/manifests/setup/setup.yaml b/manifests/setup/setup.yaml index 9453cb6ac..404333627 100644 --- a/manifests/setup/setup.yaml +++ b/manifests/setup/setup.yaml @@ -6966,6 +6966,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean @@ -35656,6 +35660,10 @@ spec: logResponsePayload: description: Log the response payload within the Fluent Bit log. type: boolean + logsBodyKey: + description: The log body key to look up in the log events body/message. + Sets the Body field of the opentelemtry logs data model. + type: string logsBodyKeyAttributes: description: If true, remaining unmatched keys are added as attributes. type: boolean