From bd93e6179f4ca911650158be89a74e2609040ad8 Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Wed, 4 Dec 2024 14:19:16 +0100 Subject: [PATCH] [chore] use config sample in e2e tests (#389) Signed-off-by: Florian Bacher Co-authored-by: odubajDT <93584209+odubajDT@users.noreply.github.com> --- config_examples/k8s_attribute_redaction.yaml | 70 ++++++++++++++++++ config_examples/k8s_enrichment.yaml | 65 +++++++++++++++++ config_examples/prometheus.yaml | 33 +++++++++ config_examples/zipkin.yaml | 4 ++ .../integration/k8senrichment/e2e_test.go | 13 +++- .../testdata/collector/configmap.yaml | 66 +---------------- .../integration/prometheus/e2e_test.go | 9 ++- .../testdata/collector/configmap.yaml | 35 +-------- .../testbed/integration/redaction/e2e_test.go | 9 ++- .../testdata/collector/configmap.yaml | 71 +------------------ .../testbed/integration/zipkin/e2e_test.go | 9 ++- .../zipkin/testdata/collector/configmap.yaml | 22 +----- internal/testcommon/k8stest/collector.go | 2 + 13 files changed, 214 insertions(+), 194 deletions(-) create mode 100644 config_examples/k8s_attribute_redaction.yaml create mode 100644 config_examples/k8s_enrichment.yaml create mode 100644 config_examples/prometheus.yaml diff --git a/config_examples/k8s_attribute_redaction.yaml b/config_examples/k8s_attribute_redaction.yaml new file mode 100644 index 00000000..b89102d0 --- /dev/null +++ b/config_examples/k8s_attribute_redaction.yaml @@ -0,0 +1,70 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 +receivers: + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 +processors: + k8sattributes: + extract: + metadata: + - k8s.pod.name + - k8s.pod.uid + - k8s.deployment.name + - k8s.statefulset.name + - k8s.daemonset.name + - k8s.job.name + - k8s.cronjob.name + - k8s.namespace.name + - k8s.node.name + - k8s.cluster.uid + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.name + - from: resource_attribute + name: k8s.namespace.name + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: connection + redaction: + allow_all_keys: false + allowed_keys: + - "k8s.pod.uid" + - "k8s.namespace.name" + - "k8s.cluster.uid" + - "k8s.job.name" + - "k8s.statefulset.name" + - "k8s.deployment.name" + - "k8s.daemonset.name" + ignored_keys: + - "k8s.pod.name" + - "service.name" + blocked_values: + - "^[a-f0-9]{32}$" + - "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" + summary: info +exporters: + otlphttp: + endpoint: ${env:DT_ENDPOINT} + headers: + Authorization: "Api-Token ${env:DT_API_TOKEN}" +service: + extensions: + - health_check + pipelines: + traces: + receivers: + - otlp + processors: + - k8sattributes + - redaction + exporters: + - otlphttp diff --git a/config_examples/k8s_enrichment.yaml b/config_examples/k8s_enrichment.yaml new file mode 100644 index 00000000..4bf61f5a --- /dev/null +++ b/config_examples/k8s_enrichment.yaml @@ -0,0 +1,65 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 +receivers: + otlp: + protocols: + grpc: + endpoint: ${env:MY_POD_IP}:4317 +processors: + k8sattributes: + extract: + metadata: + - k8s.pod.name + - k8s.pod.uid + - k8s.deployment.name + - k8s.statefulset.name + - k8s.daemonset.name + - k8s.job.name + - k8s.cronjob.name + - k8s.namespace.name + - k8s.node.name + - k8s.cluster.uid + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.name + - from: resource_attribute + name: k8s.namespace.name + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: connection + transform: + error_mode: ignore + trace_statements: + - context: resource + statements: + - set(attributes["dt.kubernetes.workload.kind"], "statefulset") where IsString(attributes["k8s.statefulset.name"]) + - set(attributes["dt.kubernetes.workload.name"], attributes["k8s.statefulset.name"]) where IsString(attributes["k8s.statefulset.name"]) + - set(attributes["dt.kubernetes.workload.kind"], "deployment") where IsString(attributes["k8s.deployment.name"]) + - set(attributes["dt.kubernetes.workload.name"], attributes["k8s.deployment.name"]) where IsString(attributes["k8s.deployment.name"]) + - set(attributes["dt.kubernetes.workload.kind"], "daemonset") where IsString(attributes["k8s.daemonset.name"]) + - set(attributes["dt.kubernetes.workload.name"], attributes["k8s.daemonset.name"]) where IsString(attributes["k8s.daemonset.name"]) + - set(attributes["dt.kubernetes.cluster.id"], attributes["k8s.cluster.uid"]) where IsString(attributes["k8s.cluster.uid"]) +exporters: + otlphttp: + endpoint: ${env:DT_ENDPOINT} + headers: + Authorization: "Api-Token ${env:API_TOKEN}" +service: + extensions: + - health_check + pipelines: + traces: + receivers: + - otlp + processors: + - k8sattributes + - transform + exporters: + - otlphttp diff --git a/config_examples/prometheus.yaml b/config_examples/prometheus.yaml new file mode 100644 index 00000000..0f1db63c --- /dev/null +++ b/config_examples/prometheus.yaml @@ -0,0 +1,33 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 +receivers: + prometheus: + config: + scrape_configs: + - job_name: 'node-exporter' + scrape_interval: 60s + static_configs: + - targets: ['prometheus-node-exporter:9100'] + - job_name: opentelemetry-collector + scrape_interval: 60s + static_configs: + - targets: + - 127.0.0.1:8888 + +processors: + cumulativetodelta: + +exporters: + otlphttp: + endpoint: ${env:DT_ENDPOINT} + headers: + Authorization: "Api-Token ${env:DT_API_TOKEN}" + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [prometheus] + processors: [cumulativetodelta] + exporters: [otlphttp] diff --git a/config_examples/zipkin.yaml b/config_examples/zipkin.yaml index 4ca5f645..02f3d542 100644 --- a/config_examples/zipkin.yaml +++ b/config_examples/zipkin.yaml @@ -1,3 +1,6 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 receivers: zipkin: endpoint: 0.0.0.0:9411 @@ -9,6 +12,7 @@ exporters: Authorization: "Api-Token ${env:API_TOKEN}" service: + extensions: [health_check] pipelines: traces: receivers: [zipkin] diff --git a/internal/testbed/integration/k8senrichment/e2e_test.go b/internal/testbed/integration/k8senrichment/e2e_test.go index 8590d3e1..625aee86 100644 --- a/internal/testbed/integration/k8senrichment/e2e_test.go +++ b/internal/testbed/integration/k8senrichment/e2e_test.go @@ -5,6 +5,7 @@ package k8senrichment import ( "fmt" "os" + "path" "path/filepath" "testing" @@ -31,6 +32,7 @@ type expectedValue struct { // See: https://docs.dynatrace.com/docs/shortlink/otel-collector-cases-k8s-enrich func TestE2E_ClusterRBAC(t *testing.T) { testDir := filepath.Join("testdata") + configExamplesDir := "../../../../config_examples" k8sClient, err := k8stest.NewK8sClient() require.NoError(t, err) @@ -56,7 +58,16 @@ func TestE2E_ClusterRBAC(t *testing.T) { defer shutdownSinks() testID := uuid.NewString()[:8] - collectorObjs := k8stest.CreateCollectorObjects(t, k8sClient, testID, filepath.Join(testDir, "collector"), "") + collectorObjs := k8stest.CreateCollectorObjects( + t, + k8sClient, + testID, + filepath.Join(testDir, "collector"), + path.Join( + configExamplesDir, + "k8s_enrichment.yaml", + ), + ) createTeleOpts := &k8stest.TelemetrygenCreateOpts{ ManifestsDir: filepath.Join(testDir, "telemetrygen"), TestID: testID, diff --git a/internal/testbed/integration/k8senrichment/testdata/collector/configmap.yaml b/internal/testbed/integration/k8senrichment/testdata/collector/configmap.yaml index d8924300..93e8eda1 100644 --- a/internal/testbed/integration/k8senrichment/testdata/collector/configmap.yaml +++ b/internal/testbed/integration/k8senrichment/testdata/collector/configmap.yaml @@ -5,68 +5,4 @@ metadata: namespace: e2ek8senrichment data: relay: | - extensions: - health_check: - endpoint: 0.0.0.0:13133 - receivers: - otlp: - protocols: - grpc: - endpoint: ${env:MY_POD_IP}:4317 - processors: - k8sattributes: - extract: - metadata: - - k8s.pod.name - - k8s.pod.uid - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.job.name - - k8s.cronjob.name - - k8s.namespace.name - - k8s.node.name - - k8s.cluster.uid - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.name - - from: resource_attribute - name: k8s.namespace.name - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - transform: - error_mode: ignore - trace_statements: - - context: resource - statements: - - set(attributes["dt.kubernetes.workload.kind"], "statefulset") where IsString(attributes["k8s.statefulset.name"]) - - set(attributes["dt.kubernetes.workload.name"], attributes["k8s.statefulset.name"]) where IsString(attributes["k8s.statefulset.name"]) - - set(attributes["dt.kubernetes.workload.kind"], "deployment") where IsString(attributes["k8s.deployment.name"]) - - set(attributes["dt.kubernetes.workload.name"], attributes["k8s.deployment.name"]) where IsString(attributes["k8s.deployment.name"]) - - set(attributes["dt.kubernetes.workload.kind"], "daemonset") where IsString(attributes["k8s.daemonset.name"]) - - set(attributes["dt.kubernetes.workload.name"], attributes["k8s.daemonset.name"]) where IsString(attributes["k8s.daemonset.name"]) - - set(attributes["dt.kubernetes.cluster.id"], attributes["k8s.cluster.uid"]) where IsString(attributes["k8s.cluster.uid"]) - exporters: - otlp: - endpoint: {{ .HostEndpoint }}:4317 - tls: - insecure: true - service: - extensions: - - health_check - pipelines: - traces: - receivers: - - otlp - processors: - - k8sattributes - - transform - exporters: - - otlp +{{ .CollectorConfig }} diff --git a/internal/testbed/integration/prometheus/e2e_test.go b/internal/testbed/integration/prometheus/e2e_test.go index 80199134..ec5859e6 100644 --- a/internal/testbed/integration/prometheus/e2e_test.go +++ b/internal/testbed/integration/prometheus/e2e_test.go @@ -20,6 +20,7 @@ import ( // See: https://docs.dynatrace.com/docs/shortlink/otel-collector-cases-prometheus func TestE2E_PrometheusNodeExporter(t *testing.T) { testDir := filepath.Join("testdata") + configExamplesDir := "../../../../config_examples" k8sClient, err := k8stest.NewK8sClient() require.NoError(t, err) @@ -49,7 +50,13 @@ func TestE2E_PrometheusNodeExporter(t *testing.T) { defer shutdownSinks() testID := uuid.NewString()[:8] - collectorObjs := k8stest.CreateCollectorObjects(t, k8sClient, testID, filepath.Join(testDir, "collector"), "") + collectorObjs := k8stest.CreateCollectorObjects( + t, + k8sClient, + testID, + filepath.Join(testDir, "collector"), + filepath.Join(configExamplesDir, "prometheus.yaml"), + ) defer func() { for _, obj := range collectorObjs { require.NoErrorf(t, k8stest.DeleteObject(k8sClient, obj), "failed to delete object %s", obj.GetName()) diff --git a/internal/testbed/integration/prometheus/testdata/collector/configmap.yaml b/internal/testbed/integration/prometheus/testdata/collector/configmap.yaml index 67eb0268..d474d1c8 100644 --- a/internal/testbed/integration/prometheus/testdata/collector/configmap.yaml +++ b/internal/testbed/integration/prometheus/testdata/collector/configmap.yaml @@ -5,37 +5,4 @@ metadata: namespace: e2eprometheus data: relay: | - extensions: - health_check: - endpoint: 0.0.0.0:13133 - receivers: - prometheus: - config: - scrape_configs: - - job_name: 'node-exporter' - scrape_interval: 60s - static_configs: - - targets: ['prometheus-node-exporter.e2eprometheus.svc.cluster.local:9100'] - - job_name: opentelemetry-collector - scrape_interval: 60s - static_configs: - - targets: - - 127.0.0.1:8888 - processors: - cumulativetodelta: - exporters: - otlp: - endpoint: {{ .HostEndpoint }}:4317 - tls: - insecure: true - service: - extensions: - - health_check - pipelines: - metrics: - receivers: - - prometheus - processors: - - cumulativetodelta - exporters: - - otlp +{{ .CollectorConfig }} diff --git a/internal/testbed/integration/redaction/e2e_test.go b/internal/testbed/integration/redaction/e2e_test.go index 32c92212..5f4d8d7b 100644 --- a/internal/testbed/integration/redaction/e2e_test.go +++ b/internal/testbed/integration/redaction/e2e_test.go @@ -17,6 +17,7 @@ import ( func TestE2E_RedactionProcessor(t *testing.T) { testDir := filepath.Join("testdata") + configExamplesDir := "../../../../config_examples" k8sClient, err := k8stest.NewK8sClient() require.NoError(t, err) @@ -42,7 +43,13 @@ func TestE2E_RedactionProcessor(t *testing.T) { defer shutdownSinks() testID := uuid.NewString()[:8] - collectorObjs := k8stest.CreateCollectorObjects(t, k8sClient, testID, filepath.Join(testDir, "collector"), "") + collectorObjs := k8stest.CreateCollectorObjects( + t, + k8sClient, + testID, + filepath.Join(testDir, "collector"), + filepath.Join(configExamplesDir, "k8s_attribute_redaction.yaml"), + ) createTeleOpts := &k8stest.TelemetrygenCreateOpts{ ManifestsDir: filepath.Join(testDir, "telemetrygen"), diff --git a/internal/testbed/integration/redaction/testdata/collector/configmap.yaml b/internal/testbed/integration/redaction/testdata/collector/configmap.yaml index 7e95457b..66d00dd6 100644 --- a/internal/testbed/integration/redaction/testdata/collector/configmap.yaml +++ b/internal/testbed/integration/redaction/testdata/collector/configmap.yaml @@ -5,73 +5,4 @@ metadata: namespace: e2eredaction data: relay: | - extensions: - health_check: - endpoint: 0.0.0.0:13133 - receivers: - otlp: - protocols: - grpc: - endpoint: ${env:MY_POD_IP}:4317 - processors: - k8sattributes: - extract: - metadata: - - k8s.pod.name - - k8s.pod.uid - - k8s.deployment.name - - k8s.statefulset.name - - k8s.daemonset.name - - k8s.job.name - - k8s.cronjob.name - - k8s.namespace.name - - k8s.node.name - - k8s.cluster.uid - pod_association: - - sources: - - from: resource_attribute - name: k8s.pod.name - - from: resource_attribute - name: k8s.namespace.name - - sources: - - from: resource_attribute - name: k8s.pod.ip - - sources: - - from: resource_attribute - name: k8s.pod.uid - - sources: - - from: connection - redaction: - allow_all_keys: false - allowed_keys: - - "k8s.pod.uid" - - "k8s.namespace.name" - - "k8s.cluster.uid" - - "k8s.job.name" - - "k8s.statefulset.name" - - "k8s.deployment.name" - - "k8s.daemonset.name" - ignored_keys: - - "k8s.pod.name" - - "service.name" - blocked_values: - - "^[a-f0-9]{32}$" - - "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" - summary: info - exporters: - otlp: - endpoint: {{ .HostEndpoint }}:4317 - tls: - insecure: true - service: - extensions: - - health_check - pipelines: - traces: - receivers: - - otlp - processors: - - k8sattributes - - redaction - exporters: - - otlp +{{ .CollectorConfig }} diff --git a/internal/testbed/integration/zipkin/e2e_test.go b/internal/testbed/integration/zipkin/e2e_test.go index aaabbf77..b546057f 100644 --- a/internal/testbed/integration/zipkin/e2e_test.go +++ b/internal/testbed/integration/zipkin/e2e_test.go @@ -19,6 +19,7 @@ import ( // See: https://docs.dynatrace.com/docs/extend-dynatrace/opentelemetry/collector/use-cases/zipkin func TestE2E_ZipkinReceiver(t *testing.T) { testDir := filepath.Join("testdata") + configExamplesDir := "../../../../config_examples" k8sClient, err := k8stest.NewK8sClient() require.NoError(t, err) @@ -44,7 +45,13 @@ func TestE2E_ZipkinReceiver(t *testing.T) { defer shutdownSinks() testID := uuid.NewString()[:8] - collectorObjs := k8stest.CreateCollectorObjects(t, k8sClient, testID, filepath.Join(testDir, "collector"), "") + collectorObjs := k8stest.CreateCollectorObjects( + t, + k8sClient, + testID, + filepath.Join(testDir, "collector"), + filepath.Join(configExamplesDir, "zipkin.yaml"), + ) createZipkinOpts := &k8stest.ZipkinAppCreateOpts{ ManifestsDir: filepath.Join(testDir, "zipkin"), TestID: testID, diff --git a/internal/testbed/integration/zipkin/testdata/collector/configmap.yaml b/internal/testbed/integration/zipkin/testdata/collector/configmap.yaml index 3742d6f8..698cdb92 100644 --- a/internal/testbed/integration/zipkin/testdata/collector/configmap.yaml +++ b/internal/testbed/integration/zipkin/testdata/collector/configmap.yaml @@ -5,24 +5,4 @@ metadata: namespace: e2ezipkin data: relay: | - extensions: - health_check: - endpoint: 0.0.0.0:13133 - receivers: - zipkin: - endpoint: 0.0.0.0:9411 - - exporters: - otlp: - endpoint: {{ .HostEndpoint }}:4317 - tls: - insecure: true - service: - extensions: - - health_check - pipelines: - traces: - receivers: - - zipkin - exporters: - - otlp +{{ .CollectorConfig }} diff --git a/internal/testcommon/k8stest/collector.go b/internal/testcommon/k8stest/collector.go index dd692cb5..d30c30f7 100644 --- a/internal/testcommon/k8stest/collector.go +++ b/internal/testcommon/k8stest/collector.go @@ -79,6 +79,8 @@ func getCollectorConfig(path, host string) (string, error) { fmt.Sprintf("http://%s:4318", host), "${env:DT_API_TOKEN}", "", + "${env:API_TOKEN}", + "", ) parsedConfig = r.Replace(parsedConfig)