feat: add redaction processor (#275)

Signed-off-by: odubajDT <[email protected]> Co-authored-by: Florian Bacher <[email protected]> Co-authored-by: Armin Ruech <[email protected]>
Dynatrace · Aug 29, 2024 · d10b8e1 · d10b8e1
1 parent fdd73b1
commit d10b8e1
Show file tree

Hide file tree

Showing 19 changed files with 508 additions and 0 deletions.
diff --git a/.chloggen/add-redaction-processor.yaml b/.chloggen/add-redaction-processor.yaml
@@ -0,0 +1,18 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: redactionprocessor
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Added the redaction processor to the Dynatrace collector distro.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [275]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -82,6 +82,7 @@ jobs:
           - prometheus
           - zipkin
           - statsd
+          - redaction
     runs-on: ubuntu-latest
     needs: docker-build
     steps:

diff --git a/README.md b/README.md
@@ -53,6 +53,7 @@ and the Collector's [troubleshooting guide].
 * [k8sattributesprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor)
 * [memorylimiterprocessor](https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/memorylimiterprocessor)
 * [probabilisticsamplerprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/probabilisticsamplerprocessor)
+* [redactionprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/redactionprocessor)
 * [resourcedetectionprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor)
 * [resourceprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor)
 * [tailsamplingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor)

diff --git a/config_examples/README.md b/config_examples/README.md
@@ -22,6 +22,7 @@ Dynatrace distribution of the OpenTelemetry Collector.
 - [StatsD Receiver](statsd.yaml)
 - [Syslog Receiver](syslog.yaml)
 - [Zipkin Receiver](zipkin.yaml)
+- [Redaction Processor](redaction.yaml)
 
 ## Sending data to Dynatrace
 

diff --git a/config_examples/redaction.yaml b/config_examples/redaction.yaml
@@ -0,0 +1,42 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  redaction:
+    allow_all_keys: false
+    allowed_keys:
+      - description
+      - group
+      - name
+    ignored_keys:
+      - safe_attribute
+    blocked_values:
+      - "4[0-9]{12}(?:[0-9]{3})?" ## Visa credit card number
+      - "(5[1-5][0-9]{14})"       ## MasterCard number
+      - "^((25[0-5]|(2[0-4]|1\\d|[1-9]|)\\d)\\.?\b){4}$" ## IPv4 addresses
+      - "^[\\w\\.]+@([\\w-]+\\.)+[\\w-]{2,4}$" ## email addresses (simplified example)
+    summary: info
+
+exporters:
+  otlphttp:
+    endpoint: ${env:DT_ENDPOINT}
+    headers:
+      Authorization: "Api-Token ${env:API_TOKEN}"
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+service:
+  extensions:
+    - health_check
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [redaction]
+      exporters: [otlphttp]
diff --git a/internal/testbed/integration/redaction/e2e_test.go b/internal/testbed/integration/redaction/e2e_test.go
@@ -0,0 +1,134 @@
+//go:build e2e
+
+package redaction
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/Dynatrace/dynatrace-otel-collector/internal/testcommon/k8stest"
+	oteltest "github.com/Dynatrace/dynatrace-otel-collector/internal/testcommon/oteltest"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/consumer/consumertest"
+)
+
+func TestE2E_RedactionProcessor(t *testing.T) {
+	testDir := filepath.Join("testdata")
+
+	k8sClient, err := k8stest.NewK8sClient()
+	require.NoError(t, err)
+
+	// Create the namespace specific for the test
+	nsFile := filepath.Join(testDir, "namespace.yaml")
+	buf, err := os.ReadFile(nsFile)
+	require.NoErrorf(t, err, "failed to read namespace object file %s", nsFile)
+	nsObj, err := k8stest.CreateObject(k8sClient, buf)
+	require.NoErrorf(t, err, "failed to create k8s namespace from file %s", nsFile)
+
+	testNs := nsObj.GetName()
+	defer func() {
+		require.NoErrorf(t, k8stest.DeleteObject(k8sClient, nsObj), "failed to delete namespace %s", testNs)
+	}()
+
+	tracesConsumer := new(consumertest.TracesSink)
+	shutdownSinks := oteltest.StartUpSinks(t, oteltest.ReceiverSinks{Traces: tracesConsumer})
+	defer shutdownSinks()
+
+	testID := uuid.NewString()[:8]
+	collectorObjs := k8stest.CreateCollectorObjects(t, k8sClient, testID, filepath.Join(testDir, "collector"))
+
+	createTeleOpts := &k8stest.TelemetrygenCreateOpts{
+		ManifestsDir: filepath.Join(testDir, "telemetrygen"),
+		TestID:       testID,
+		OtlpEndpoint: fmt.Sprintf("otelcol-%s.%s:4317", testID, testNs),
+		DataTypes:    []string{"traces"},
+	}
+	telemetryGenObjs, telemetryGenObjInfos := k8stest.CreateTelemetryGenObjects(t, k8sClient, createTeleOpts)
+	defer func() {
+		for _, obj := range append(collectorObjs, telemetryGenObjs...) {
+			require.NoErrorf(t, k8stest.DeleteObject(k8sClient, obj), "failed to delete object %s", obj.GetName())
+		}
+	}()
+
+	for _, info := range telemetryGenObjInfos {
+		k8stest.WaitForTelemetryGenToStart(t, k8sClient, info.Namespace, info.PodLabelSelectors, info.Workload, info.DataType)
+	}
+
+	wantEntries := 30 // Minimal number of traces to wait for.
+	oteltest.WaitForTraces(t, wantEntries, tracesConsumer)
+
+	tcs := []struct {
+		name    string
+		service string
+		attrs   map[string]oteltest.ExpectedValue
+	}{
+		{
+			name:    "traces-job",
+			service: "test-traces-job",
+			attrs: map[string]oteltest.ExpectedValue{
+				"service.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "test-traces-job"),
+				"k8s.pod.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeRegex, "telemetrygen-"+testID+"-traces-job-[a-z0-9]*"),
+				"k8s.pod.uid":              oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"k8s.job.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "telemetrygen-"+testID+"-traces-job"),
+				"k8s.namespace.name":       oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, testNs),
+				"k8s.cluster.uid":          oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"redaction.redacted.count": oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.masked.count":   oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.ignored.count":  oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+			},
+		},
+		{
+			name:    "traces-statefulset",
+			service: "test-traces-statefulset",
+			attrs: map[string]oteltest.ExpectedValue{
+				"service.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "test-traces-statefulset"),
+				"k8s.pod.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "telemetrygen-"+testID+"-traces-statefulset-0"),
+				"k8s.pod.uid":              oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"k8s.statefulset.name":     oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "telemetrygen-"+testID+"-traces-statefulset"),
+				"k8s.namespace.name":       oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, testNs),
+				"k8s.cluster.uid":          oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"redaction.redacted.count": oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.masked.count":   oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.ignored.count":  oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+			},
+		},
+		{
+			name:    "traces-deployment",
+			service: "test-traces-deployment",
+			attrs: map[string]oteltest.ExpectedValue{
+				"service.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "test-traces-deployment"),
+				"k8s.pod.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeRegex, "telemetrygen-"+testID+"-traces-deployment-[a-z0-9]*-[a-z0-9]*"),
+				"k8s.pod.uid":              oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"k8s.deployment.name":      oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "telemetrygen-"+testID+"-traces-deployment"),
+				"k8s.namespace.name":       oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, testNs),
+				"k8s.cluster.uid":          oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"redaction.redacted.count": oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.masked.count":   oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.ignored.count":  oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+			},
+		},
+		{
+			name:    "traces-daemonset",
+			service: "test-traces-daemonset",
+			attrs: map[string]oteltest.ExpectedValue{
+				"service.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "test-traces-daemonset"),
+				"k8s.pod.name":             oteltest.NewExpectedValue(oteltest.AttributeMatchTypeRegex, "telemetrygen-"+testID+"-traces-daemonset-[a-z0-9]*"),
+				"k8s.pod.uid":              oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"k8s.daemonset.name":       oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "telemetrygen-"+testID+"-traces-daemonset"),
+				"k8s.namespace.name":       oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, testNs),
+				"k8s.cluster.uid":          oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "****"),
+				"redaction.redacted.count": oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.masked.count":   oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+				"redaction.ignored.count":  oteltest.NewExpectedValue(oteltest.AttributeMatchTypeEqual, "2"),
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		oteltest.ScanTracesForAttributes(t, tracesConsumer, tc.service, tc.attrs, nil)
+	}
+
+}
diff --git a/internal/testbed/integration/redaction/testdata/collector/clusterrole.yaml b/internal/testbed/integration/redaction/testdata/collector/clusterrole.yaml
@@ -0,0 +1,18 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Name }}
+  labels:
+    app: {{ .Name }}
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "namespaces"]
+    verbs: ["get", "watch", "list"]
+
+  - apiGroups: ["apps"]
+    resources: ["replicasets"]
+    verbs: ["get", "watch", "list"]
+
+  - apiGroups: ["extensions"]
+    resources: ["replicasets"]
+    verbs: ["get", "watch", "list"]
diff --git a/internal/testbed/integration/redaction/testdata/collector/clusterrolebinding.yaml b/internal/testbed/integration/redaction/testdata/collector/clusterrolebinding.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Name }}
+  labels:
+    app: {{ .Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Name }}
+    namespace: e2eredaction
diff --git a/internal/testbed/integration/redaction/testdata/collector/configmap.yaml b/internal/testbed/integration/redaction/testdata/collector/configmap.yaml
@@ -0,0 +1,77 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Name }}-config
+  namespace: e2eredaction
+data:
+  relay: |
+    extensions:
+      health_check:
+        endpoint: 0.0.0.0:13133
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: ${env:MY_POD_IP}:4317
+    processors:
+      k8sattributes:
+        extract:
+          metadata:
+          - k8s.pod.name
+          - k8s.pod.uid
+          - k8s.deployment.name
+          - k8s.statefulset.name
+          - k8s.daemonset.name
+          - k8s.job.name
+          - k8s.cronjob.name
+          - k8s.namespace.name
+          - k8s.node.name
+          - k8s.cluster.uid
+        pod_association:
+        - sources:
+          - from: resource_attribute
+            name: k8s.pod.name
+          - from: resource_attribute
+            name: k8s.namespace.name
+        - sources:
+          - from: resource_attribute
+            name: k8s.pod.ip
+        - sources:
+          - from: resource_attribute
+            name: k8s.pod.uid
+        - sources:
+          - from: connection
+      redaction:
+        allow_all_keys: false
+        allowed_keys:
+          - "k8s.pod.uid"
+          - "k8s.namespace.name"
+          - "k8s.cluster.uid"
+          - "k8s.job.name"
+          - "k8s.statefulset.name"
+          - "k8s.deployment.name"
+          - "k8s.daemonset.name"
+        ignored_keys:
+          - "k8s.pod.name"
+          - "service.name"
+        blocked_values:
+          - "^[a-f0-9]{32}$"
+          - "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$"
+        summary: info
+    exporters:
+      otlp:
+        endpoint: {{ .HostEndpoint }}:4317
+        tls:
+          insecure: true
+    service:
+      extensions:
+        - health_check
+      pipelines:
+        traces:
+          receivers:
+            - otlp
+          processors:
+            - k8sattributes
+            - redaction
+          exporters:
+            - otlp
diff --git a/internal/testbed/integration/redaction/testdata/collector/deployment.yaml b/internal/testbed/integration/redaction/testdata/collector/deployment.yaml
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Name }}
+  namespace: e2eredaction
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: opentelemetry-collector
+      app.kubernetes.io/instance: {{ .Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: opentelemetry-collector
+        app.kubernetes.io/instance: {{ .Name }}
+    spec:
+      serviceAccountName: {{ .Name }}
+      containers:
+        - name: opentelemetry-collector
+          command:
+            - /dynatrace-otel-collector
+            - --config=/conf/relay.yaml
+          image: "{{ .ContainerRegistry }}dynatrace-otel-collector:e2e-test"
+          ports:
+            - name: otlp
+              containerPort: 4317
+              protocol: TCP
+          env:
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 13133
+            initialDelaySeconds: 3
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 13133
+            initialDelaySeconds: 3
+          resources:
+            limits:
+              cpu: 128m
+              memory: 256Mi
+          volumeMounts:
+            - mountPath: /conf
+              name: opentelemetry-collector-configmap
+      volumes:
+        - name: opentelemetry-collector-configmap
+          configMap:
+            name: {{ .Name }}-config
+            items:
+              - key: relay
+                path: relay.yaml