From a321638c6b7766735b7b0fac274c0db1fcc6c1f7 Mon Sep 17 00:00:00 2001 From: Ben Fisler Date: Mon, 23 Aug 2021 14:06:11 -0400 Subject: [PATCH 1/2] updates to optionally include or exclude specific headers from the span's grpc.headers tag --- .../grpc/TracingServerInterceptor.java | 58 ++++++++++++++++- .../grpc/TracingServerInterceptorTest.java | 63 +++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/opentracing/contrib/grpc/TracingServerInterceptor.java b/src/main/java/io/opentracing/contrib/grpc/TracingServerInterceptor.java index e699995..83e28d7 100644 --- a/src/main/java/io/opentracing/contrib/grpc/TracingServerInterceptor.java +++ b/src/main/java/io/opentracing/contrib/grpc/TracingServerInterceptor.java @@ -56,6 +56,8 @@ public class TracingServerInterceptor implements ServerInterceptor { private final Set tracedAttributes; private final ImmutableList serverSpanDecorators; private final ImmutableList serverCloseDecorators; + private final Set> excludedHeaders; + private final Set> includedHeaders; private TracingServerInterceptor(Builder builder) { this.tracer = builder.tracer; @@ -65,6 +67,11 @@ private TracingServerInterceptor(Builder builder) { this.tracedAttributes = builder.tracedAttributes; this.serverSpanDecorators = ImmutableList.copyOf(builder.serverSpanDecorators.values()); this.serverCloseDecorators = ImmutableList.copyOf(builder.serverCloseDecorators.values()); + if(builder.tracedAttributes.contains(ServerRequestAttribute.HEADERS) && builder.excludedHeaders.size() > 0 && builder.includedHeaders.size() > 0) { + throw new IllegalArgumentException("Only one of excludedHeaders or includedHeaders can be defined at the same time."); + } + this.excludedHeaders = builder.excludedHeaders; + this.includedHeaders = builder.includedHeaders; } /** @@ -131,7 +138,7 @@ public ServerCall.Listener interceptCall( GrpcTags.GRPC_CALL_ATTRIBUTES.set(span, call.getAttributes()); break; case HEADERS: - GrpcTags.GRPC_HEADERS.set(span, headers); + GrpcTags.GRPC_HEADERS.set(span, massageHeaders(headers)); break; case PEER_ADDRESS: GrpcTags.PEER_ADDRESS.set(span, call.getAttributes()); @@ -303,6 +310,22 @@ Span getSpanFromHeaders(Map headers, String operationName) { } return span; } + + private Metadata massageHeaders(Metadata headers) { + if(this.excludedHeaders.size() > 0) { + Metadata massagedHeaders = new Metadata(); + massagedHeaders.merge(headers); + for(Metadata.Key excludedHeader : this.excludedHeaders) { + massagedHeaders.removeAll(excludedHeader); + } + return massagedHeaders; + } else if(this.includedHeaders.size() > 0) { + Metadata massagedHeaders = new Metadata(); + massagedHeaders.merge(headers, this.includedHeaders); + return massagedHeaders; + } + return headers; + } /** * Builds the configuration of a TracingServerInterceptor. @@ -316,6 +339,8 @@ public static class Builder { private Set tracedAttributes; private Map, ServerSpanDecorator> serverSpanDecorators; private Map, ServerCloseDecorator> serverCloseDecorators; + private Set> excludedHeaders; + private Set> includedHeaders; /** * Creates a Builder with GlobalTracer if present else NoopTracer. @@ -328,6 +353,8 @@ private Builder() { this.tracedAttributes = new HashSet<>(); this.serverSpanDecorators = new HashMap<>(); this.serverCloseDecorators = new HashMap<>(); + this.excludedHeaders = new HashSet<>(); + this.includedHeaders = new HashSet<>(); } /** @@ -362,6 +389,35 @@ public Builder withTracedAttributes(ServerRequestAttribute... attributes) { this.tracedAttributes = new HashSet<>(Arrays.asList(attributes)); return this; } + + /** + * Provide keys for headers that should be excluded from the trace. Only applicable + * when {@link #withTracedAttributes(ServerRequestAttribute...)} contains {@link ServerRequestAttribute#HEADERS}. + * + * Mutually exclusive with {@link #withIncludedHeaders(io.grpc.Metadata.Key...)}. + * + * @param headerKeys + * @return this Builder configured to exclude the header keys + */ + public Builder withExcludedHeaders(Metadata.Key... headerKeys) { + this.excludedHeaders = new HashSet<>(Arrays.asList(headerKeys)); + return this; + } + + /** + * Provide keys for headers that should be included in the trace. Headers with any other + * key will be excluded. Only applicable when {@link #withTracedAttributes(ServerRequestAttribute...)} + * contains {@link ServerRequestAttribute#HEADERS}. + * + * Mutually exclusive with {@link #withExcludedHeaders(io.grpc.Metadata.Key...)}. + * + * @param headerKeys + * @return this Builder configured to exclude the header keys + */ + public Builder withIncludedHeaders(Metadata.Key... headerKeys) { + this.includedHeaders = new HashSet<>(Arrays.asList(headerKeys)); + return this; + } /** * Logs streaming events to server spans. diff --git a/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java b/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java index 2141a97..825b99b 100644 --- a/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java +++ b/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.grpc.Metadata; +import io.grpc.Metadata.Key; import io.grpc.MethodDescriptor; import io.grpc.ServerCall; import io.grpc.Status; @@ -274,6 +275,68 @@ public void testTracedServerWithTracedAttributes() { .containsAll(SERVER_ATTRIBUTE_TAGS); assertFalse("span should have no baggage", span.context().baggageItems().iterator().hasNext()); } + + @Test + public void testTracedServerWithExcludedHeader() { + Key headerKey = Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER); + TracingServerInterceptor tracingInterceptor = + TracingServerInterceptor.newBuilder() + .withTracer(serverTracer) + .withTracedAttributes(TracingServerInterceptor.ServerRequestAttribute.HEADERS) + .withExcludedHeaders(headerKey) + .build(); + TracedService.addGeeterService(grpcServer.getServiceRegistry(), tracingInterceptor); + + assertEquals("call should complete successfully", "Hello world", client.greet().getMessage()); + await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(serverTracer), equalTo(1)); + assertEquals( + "one span should have been created and finished for one client request", + serverTracer.finishedSpans().size(), + 1); + + MockSpan span = serverTracer.finishedSpans().get(0); + assertEquals("span should have prefix", span.operationName(), "helloworld.Greeter/SayHello"); + assertEquals("span should have no parents", span.parentId(), 0); + assertEquals("span should have no logs", span.logEntries().size(), 0); + Assertions.assertThat(span.tags()) + .as("span should have base server tags") + .containsKey(GrpcTags.GRPC_HEADERS.getKey()); + String headers = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); + Assertions.assertThat(headers) + .as("headers should have user agent") + .doesNotContain("user-agent"); + } + + @Test + public void testTracedServerWithIncludedHeader() { + Key headerKey = Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER); + TracingServerInterceptor tracingInterceptor = + TracingServerInterceptor.newBuilder() + .withTracer(serverTracer) + .withTracedAttributes(TracingServerInterceptor.ServerRequestAttribute.HEADERS) + .withIncludedHeaders(headerKey) + .build(); + TracedService.addGeeterService(grpcServer.getServiceRegistry(), tracingInterceptor); + + assertEquals("call should complete successfully", "Hello world", client.greet().getMessage()); + await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(serverTracer), equalTo(1)); + assertEquals( + "one span should have been created and finished for one client request", + serverTracer.finishedSpans().size(), + 1); + + MockSpan span = serverTracer.finishedSpans().get(0); + assertEquals("span should have prefix", span.operationName(), "helloworld.Greeter/SayHello"); + assertEquals("span should have no parents", span.parentId(), 0); + assertEquals("span should have no logs", span.logEntries().size(), 0); + Assertions.assertThat(span.tags()) + .as("span should have base server tags") + .containsKey(GrpcTags.GRPC_HEADERS.getKey()); + String headers = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); + Assertions.assertThat(headers) + .as("headers should have user agent") + .contains("user-agent"); + } @Test public void testTracedServerWithServerSpanDecorator() { From 97067cb54a6218b152533f5c493c6ccfa00cf961 Mon Sep 17 00:00:00 2001 From: Ben Fisler Date: Mon, 23 Aug 2021 14:43:04 -0400 Subject: [PATCH 2/2] added client tests; refactored server tests to be like client tests --- .../grpc/TracingClientInterceptor.java | 60 ++++++++++++++- .../contrib/grpc/TracedClient.java | 17 ++++ .../grpc/TracingClientInterceptorTest.java | 77 ++++++++++++++++++- .../grpc/TracingServerInterceptorTest.java | 49 +++++++----- 4 files changed, 183 insertions(+), 20 deletions(-) diff --git a/src/main/java/io/opentracing/contrib/grpc/TracingClientInterceptor.java b/src/main/java/io/opentracing/contrib/grpc/TracingClientInterceptor.java index 552b332..e5eaa98 100644 --- a/src/main/java/io/opentracing/contrib/grpc/TracingClientInterceptor.java +++ b/src/main/java/io/opentracing/contrib/grpc/TracingClientInterceptor.java @@ -31,6 +31,8 @@ import io.opentracing.Span; import io.opentracing.SpanContext; import io.opentracing.Tracer; +import io.opentracing.contrib.grpc.TracingServerInterceptor.Builder; +import io.opentracing.contrib.grpc.TracingServerInterceptor.ServerRequestAttribute; import io.opentracing.log.Fields; import io.opentracing.propagation.Format; import io.opentracing.propagation.TextMap; @@ -60,6 +62,8 @@ public class TracingClientInterceptor implements ClientInterceptor { private final ActiveSpanContextSource activeSpanContextSource; private final ImmutableList clientSpanDecorators; private final ImmutableList clientCloseDecorators; + private final Set> excludedHeaders; + private final Set> includedHeaders; private TracingClientInterceptor(Builder builder) { this.tracer = builder.tracer; @@ -71,6 +75,11 @@ private TracingClientInterceptor(Builder builder) { this.activeSpanContextSource = builder.activeSpanContextSource; this.clientSpanDecorators = ImmutableList.copyOf(builder.clientSpanDecorators.values()); this.clientCloseDecorators = ImmutableList.copyOf(builder.clientCloseDecorators.values()); + if(builder.tracedAttributes.contains(ClientRequestAttribute.HEADERS) && builder.excludedHeaders.size() > 0 && builder.includedHeaders.size() > 0) { + throw new IllegalArgumentException("Only one of excludedHeaders or includedHeaders can be defined at the same time."); + } + this.excludedHeaders = builder.excludedHeaders; + this.includedHeaders = builder.includedHeaders; } /** @@ -150,7 +159,7 @@ public void start(Listener responseListener, final Metadata headers) { .build()); } if (tracedAttributes.contains(ClientRequestAttribute.HEADERS)) { - GrpcTags.GRPC_HEADERS.set(span, headers); + GrpcTags.GRPC_HEADERS.set(span, massageHeaders(headers)); } tracer.inject( @@ -321,6 +330,22 @@ private Span createSpanFromParent(SpanContext parentSpanContext, String operatio .withTag(Tags.COMPONENT.getKey(), GrpcTags.COMPONENT_NAME) .start(); } + + private Metadata massageHeaders(Metadata headers) { + if(this.excludedHeaders.size() > 0) { + Metadata massagedHeaders = new Metadata(); + massagedHeaders.merge(headers); + for(Metadata.Key excludedHeader : this.excludedHeaders) { + massagedHeaders.removeAll(excludedHeader); + } + return massagedHeaders; + } else if(this.includedHeaders.size() > 0) { + Metadata massagedHeaders = new Metadata(); + massagedHeaders.merge(headers, this.includedHeaders); + return massagedHeaders; + } + return headers; + } /** * Builds the configuration of a TracingClientInterceptor. @@ -336,6 +361,8 @@ public static class Builder { private ActiveSpanContextSource activeSpanContextSource; private Map, ClientSpanDecorator> clientSpanDecorators; private Map, ClientCloseDecorator> clientCloseDecorators; + private Set> excludedHeaders; + private Set> includedHeaders; /** * Creates a Builder with GlobalTracer if present else NoopTracer. @@ -350,6 +377,8 @@ private Builder() { this.activeSpanContextSource = ActiveSpanContextSource.NONE; this.clientSpanDecorators = new HashMap<>(); this.clientCloseDecorators = new HashMap<>(); + this.excludedHeaders = new HashSet<>(); + this.includedHeaders = new HashSet<>(); } /** @@ -394,6 +423,35 @@ public Builder withTracedAttributes(ClientRequestAttribute... tracedAttributes) this.tracedAttributes = new HashSet<>(Arrays.asList(tracedAttributes)); return this; } + + /** + * Provide keys for headers that should be excluded from the trace. Only applicable + * when {@link #withTracedAttributes(ClientRequestAttribute...)} contains {@link ClientRequestAttribute#HEADERS}. + * + * Mutually exclusive with {@link #withIncludedHeaders(io.grpc.Metadata.Key...)}. + * + * @param headerKeys + * @return this Builder configured to exclude the header keys + */ + public Builder withExcludedHeaders(Metadata.Key... headerKeys) { + this.excludedHeaders = new HashSet<>(Arrays.asList(headerKeys)); + return this; + } + + /** + * Provide keys for headers that should be included in the trace. Headers with any other + * key will be excluded. Only applicable when {@link #withTracedAttributes(ClientRequestAttribute...)} + * contains {@link ClientRequestAttribute#HEADERS}. + * + * Mutually exclusive with {@link #withExcludedHeaders(io.grpc.Metadata.Key...)}. + * + * @param headerKeys + * @return this Builder configured to exclude the header keys + */ + public Builder withIncludedHeaders(Metadata.Key... headerKeys) { + this.includedHeaders = new HashSet<>(Arrays.asList(headerKeys)); + return this; + } /** * Logs all request life-cycle events to client spans. diff --git a/src/test/java/io/opentracing/contrib/grpc/TracedClient.java b/src/test/java/io/opentracing/contrib/grpc/TracedClient.java index a720cc5..2d437a1 100644 --- a/src/test/java/io/opentracing/contrib/grpc/TracedClient.java +++ b/src/test/java/io/opentracing/contrib/grpc/TracedClient.java @@ -17,9 +17,13 @@ import io.grpc.ClientInterceptor; import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; +import io.grpc.Metadata; +import io.grpc.stub.MetadataUtils; import io.opentracing.contrib.grpc.gen.GreeterGrpc; +import io.opentracing.contrib.grpc.gen.GreeterGrpc.GreeterBlockingStub; import io.opentracing.contrib.grpc.gen.HelloReply; import io.opentracing.contrib.grpc.gen.HelloRequest; +import java.util.Map; import java.util.concurrent.TimeUnit; class TracedClient { @@ -48,4 +52,17 @@ HelloReply greet() { return null; } } + + HelloReply greetWithHeaders(Map headers) { + try { + Metadata metadata = new Metadata(); + for(Map.Entry entry : headers.entrySet()) { + metadata.put(Metadata.Key.of(entry.getKey(), Metadata.ASCII_STRING_MARSHALLER), entry.getValue()); + } + GreeterBlockingStub stub = MetadataUtils.attachHeaders(blockingStub, metadata); + return stub.sayHello(HelloRequest.newBuilder().setName("world").build()); + } catch(Exception ignored) { + return null; + } + } } diff --git a/src/test/java/io/opentracing/contrib/grpc/TracingClientInterceptorTest.java b/src/test/java/io/opentracing/contrib/grpc/TracingClientInterceptorTest.java index ccef3f6..e067f51 100644 --- a/src/test/java/io/opentracing/contrib/grpc/TracingClientInterceptorTest.java +++ b/src/test/java/io/opentracing/contrib/grpc/TracingClientInterceptorTest.java @@ -34,6 +34,7 @@ import io.opentracing.tag.Tags; import io.opentracing.util.GlobalTracerTestUtil; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -240,7 +241,7 @@ public void testTracedClientWithTracedAttributes() { .withTracer(clientTracer) .withTracedAttributes(TracingClientInterceptor.ClientRequestAttribute.values()) .build(); - TracedClient client = new TracedClient(grpcServer.getChannel(), 50, "gzip", tracingInterceptor); + TracedClient client = new TracedClient(grpcServer.getChannel(), 5000, "gzip", tracingInterceptor); assertEquals("call should complete successfully", "Hello world", client.greet().getMessage()); await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(clientTracer), equalTo(1)); @@ -261,6 +262,80 @@ public void testTracedClientWithTracedAttributes() { .containsAll(CLIENT_ATTRIBUTE_TAGS); assertFalse("span should have no baggage", span.context().baggageItems().iterator().hasNext()); } + + @Test + public void testTracedClientWithExcludedHeader() { + TracingClientInterceptor tracingInterceptor = + TracingClientInterceptor.newBuilder() + .withTracer(clientTracer) + .withTracedAttributes(TracingClientInterceptor.ClientRequestAttribute.HEADERS) + .withExcludedHeaders(Metadata.Key.of("header2", Metadata.ASCII_STRING_MARSHALLER)) + .build(); + TracedClient client = new TracedClient(grpcServer.getChannel(), 5000, "gzip", tracingInterceptor); + + Map headers = new HashMap<>(); + headers.put("header1", "value1"); + headers.put("header2", "value2"); + + assertEquals("call should complete successfully", "Hello world", client.greetWithHeaders(headers).getMessage()); + await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(clientTracer), equalTo(1)); + assertEquals( + "one span should have been created and finished for one client request", + clientTracer.finishedSpans().size(), + 1); + + MockSpan span = clientTracer.finishedSpans().get(0); + assertEquals("span should have prefix", span.operationName(), "helloworld.Greeter/SayHello"); + assertEquals("span should have no parents", span.parentId(), 0); + assertEquals("span should have no logs", span.logEntries().size(), 0); + Assertions.assertThat(span.tags()) + .as("span should have base server tags") + .containsKey(GrpcTags.GRPC_HEADERS.getKey()); + String headerString = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); + Assertions.assertThat(headerString) + .as("headers should have header1") + .contains("header1"); + Assertions.assertThat(headerString) + .as("headers should NOT have header2") + .doesNotContain("header2"); + } + + @Test + public void testTracedClientWithIncludedHeader() { + TracingClientInterceptor tracingInterceptor = + TracingClientInterceptor.newBuilder() + .withTracer(clientTracer) + .withTracedAttributes(TracingClientInterceptor.ClientRequestAttribute.HEADERS) + .withIncludedHeaders(Metadata.Key.of("header2", Metadata.ASCII_STRING_MARSHALLER)) + .build(); + TracedClient client = new TracedClient(grpcServer.getChannel(), 5000, "gzip", tracingInterceptor); + + Map headers = new HashMap<>(); + headers.put("header1", "value1"); + headers.put("header2", "value2"); + + assertEquals("call should complete successfully", "Hello world", client.greetWithHeaders(headers).getMessage()); + await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(clientTracer), equalTo(1)); + assertEquals( + "one span should have been created and finished for one client request", + clientTracer.finishedSpans().size(), + 1); + + MockSpan span = clientTracer.finishedSpans().get(0); + assertEquals("span should have prefix", span.operationName(), "helloworld.Greeter/SayHello"); + assertEquals("span should have no parents", span.parentId(), 0); + assertEquals("span should have no logs", span.logEntries().size(), 0); + Assertions.assertThat(span.tags()) + .as("span should have base server tags") + .containsKey(GrpcTags.GRPC_HEADERS.getKey()); + String headerString = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); + Assertions.assertThat(headerString) + .as("headers should NOT have header1") + .doesNotContain("header1"); + Assertions.assertThat(headerString) + .as("headers should have header2") + .contains("header2"); + } @Test public void testTracedClientwithActiveSpanSource() { diff --git a/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java b/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java index 825b99b..ce15e27 100644 --- a/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java +++ b/src/test/java/io/opentracing/contrib/grpc/TracingServerInterceptorTest.java @@ -46,6 +46,7 @@ import io.opentracing.util.GlobalTracerTestUtil; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -278,16 +279,19 @@ public void testTracedServerWithTracedAttributes() { @Test public void testTracedServerWithExcludedHeader() { - Key headerKey = Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER); TracingServerInterceptor tracingInterceptor = TracingServerInterceptor.newBuilder() .withTracer(serverTracer) .withTracedAttributes(TracingServerInterceptor.ServerRequestAttribute.HEADERS) - .withExcludedHeaders(headerKey) + .withExcludedHeaders(Metadata.Key.of("header2", Metadata.ASCII_STRING_MARSHALLER)) .build(); TracedService.addGeeterService(grpcServer.getServiceRegistry(), tracingInterceptor); + + Map headers = new HashMap<>(); + headers.put("header1", "value1"); + headers.put("header2", "value2"); - assertEquals("call should complete successfully", "Hello world", client.greet().getMessage()); + assertEquals("call should complete successfully", "Hello world", client.greetWithHeaders(headers).getMessage()); await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(serverTracer), equalTo(1)); assertEquals( "one span should have been created and finished for one client request", @@ -299,26 +303,32 @@ public void testTracedServerWithExcludedHeader() { assertEquals("span should have no parents", span.parentId(), 0); assertEquals("span should have no logs", span.logEntries().size(), 0); Assertions.assertThat(span.tags()) - .as("span should have base server tags") - .containsKey(GrpcTags.GRPC_HEADERS.getKey()); - String headers = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); - Assertions.assertThat(headers) - .as("headers should have user agent") - .doesNotContain("user-agent"); + .as("span should have base server tags") + .containsKey(GrpcTags.GRPC_HEADERS.getKey()); + String headerString = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); + Assertions.assertThat(headerString) + .as("headers should have header1") + .contains("header1"); + Assertions.assertThat(headerString) + .as("headers should NOT have header2") + .doesNotContain("header2"); } @Test public void testTracedServerWithIncludedHeader() { - Key headerKey = Metadata.Key.of("user-agent", Metadata.ASCII_STRING_MARSHALLER); TracingServerInterceptor tracingInterceptor = TracingServerInterceptor.newBuilder() .withTracer(serverTracer) .withTracedAttributes(TracingServerInterceptor.ServerRequestAttribute.HEADERS) - .withIncludedHeaders(headerKey) + .withIncludedHeaders(Metadata.Key.of("header2", Metadata.ASCII_STRING_MARSHALLER)) .build(); TracedService.addGeeterService(grpcServer.getServiceRegistry(), tracingInterceptor); + + Map headers = new HashMap<>(); + headers.put("header1", "value1"); + headers.put("header2", "value2"); - assertEquals("call should complete successfully", "Hello world", client.greet().getMessage()); + assertEquals("call should complete successfully", "Hello world", client.greetWithHeaders(headers).getMessage()); await().atMost(5, TimeUnit.SECONDS).until(reportedSpansSize(serverTracer), equalTo(1)); assertEquals( "one span should have been created and finished for one client request", @@ -330,12 +340,15 @@ public void testTracedServerWithIncludedHeader() { assertEquals("span should have no parents", span.parentId(), 0); assertEquals("span should have no logs", span.logEntries().size(), 0); Assertions.assertThat(span.tags()) - .as("span should have base server tags") - .containsKey(GrpcTags.GRPC_HEADERS.getKey()); - String headers = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); - Assertions.assertThat(headers) - .as("headers should have user agent") - .contains("user-agent"); + .as("span should have base server tags") + .containsKey(GrpcTags.GRPC_HEADERS.getKey()); + String headerString = (String) span.tags().get(GrpcTags.GRPC_HEADERS.getKey()); + Assertions.assertThat(headerString) + .as("headers should NOT have header1") + .doesNotContain("header1"); + Assertions.assertThat(headerString) + .as("headers should have header2") + .contains("header2"); } @Test