diff --git a/data-model/src/main/avro/Event.avdl b/data-model/src/main/avro/Event.avdl index 86faa27..3a6cac9 100644 --- a/data-model/src/main/avro/Event.avdl +++ b/data-model/src/main/avro/Event.avdl @@ -34,6 +34,9 @@ protocol EventProtocol { // Event may be associated with multiple entities array entity_id_list = []; + // The index of the associated resource in the structured trace resource list, or -1 if none + int resource_index = -1; + // Raw attributes corresponding to this event that came from the original tracer. union { null, Attributes } attributes = null; diff --git a/data-model/src/main/avro/RawSpan.avdl b/data-model/src/main/avro/RawSpan.avdl index ab36f36..749fd14 100644 --- a/data-model/src/main/avro/RawSpan.avdl +++ b/data-model/src/main/avro/RawSpan.avdl @@ -4,6 +4,7 @@ protocol RawSpanProtocol { import idl "Attributes.avdl"; import idl "Entity.avdl"; import idl "Event.avdl"; + import idl "Resource.avdl"; record RawSpan { string customer_id; @@ -12,6 +13,8 @@ protocol RawSpanProtocol { array entity_list = []; + union { null, Resource } resource = null; + Event event; long received_time_millis = 0; diff --git a/data-model/src/main/avro/Resource.avdl b/data-model/src/main/avro/Resource.avdl new file mode 100644 index 0000000..d23e667 --- /dev/null +++ b/data-model/src/main/avro/Resource.avdl @@ -0,0 +1,8 @@ +@namespace("org.hypertrace.core.datamodel") +protocol ResourceProtocol { + import idl "Attributes.avdl"; + + record Resource { + Attributes attributes; + } +} \ No newline at end of file diff --git a/data-model/src/main/avro/StructuredTrace.avdl b/data-model/src/main/avro/StructuredTrace.avdl index 7dfa3ed..cf54b4a 100644 --- a/data-model/src/main/avro/StructuredTrace.avdl +++ b/data-model/src/main/avro/StructuredTrace.avdl @@ -6,6 +6,7 @@ protocol StructuredTraceProtocol { import idl "Entity.avdl"; import idl "Event.avdl"; import idl "Timestamps.avdl"; + import idl "Resource.avdl"; enum EdgeType { EVENT_EVENT, @@ -54,6 +55,8 @@ protocol StructuredTraceProtocol { // For e.g. Service (Entity) generates Span (Event) array entity_list; + array resource_list = []; + // Each event_node represents a Span array event_list; diff --git a/data-model/src/main/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilder.java b/data-model/src/main/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilder.java index b303956..ea6ba5d 100644 --- a/data-model/src/main/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilder.java +++ b/data-model/src/main/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilder.java @@ -1,5 +1,7 @@ package org.hypertrace.core.datamodel.shared.trace; +import static java.util.Objects.nonNull; + import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import java.nio.ByteBuffer; @@ -7,6 +9,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -23,6 +26,7 @@ import org.hypertrace.core.datamodel.MetricValue; import org.hypertrace.core.datamodel.Metrics; import org.hypertrace.core.datamodel.RawSpan; +import org.hypertrace.core.datamodel.Resource; import org.hypertrace.core.datamodel.StructuredTrace; import org.hypertrace.core.datamodel.StructuredTrace.Builder; import org.hypertrace.core.datamodel.Timestamps; @@ -42,6 +46,7 @@ public class StructuredTraceBuilder { private final Map entityMap; private final Map eventMap; + private final List resourceList; private ArrayList orderedEventNodes; private ArrayList orderedEntityNodes; @@ -59,16 +64,11 @@ public StructuredTraceBuilder( List eventList, Map entityMap, String customerId, - ByteBuffer traceId) { - this(eventList, entityMap, customerId, traceId, null); - } - - public StructuredTraceBuilder( - List eventList, - Map entityMap, - String customerId, - ByteBuffer traceId, Timestamps timestamps) { + ByteBuffer traceId, + Timestamps timestamps, + List resourceList) { this.eventList = eventList; + this.resourceList = resourceList; this.customerId = customerId; this.traceId = traceId; this.entityMap = Map.copyOf(entityMap); @@ -191,6 +191,7 @@ private StructuredTrace build() { builder.setEventEdgeList(new ArrayList<>()); builder.setEntityEdgeList(new ArrayList<>()); builder.setEntityEventEdgeList(new ArrayList<>()); + builder.setResourceList(this.resourceList); //Node Builders //Initialize EVENT NODE @@ -420,10 +421,21 @@ public static StructuredTrace buildStructuredTraceFromRawSpans(List raw String customerId, Timestamps timestamps) { Map entityMap = new HashMap<>(); + // Relying on insertion ordered keyset + LinkedHashMap resourceIndexMap = new LinkedHashMap<>(); List eventList = new ArrayList<>(); for (RawSpan rawSpan : rawSpanList) { eventList.add(rawSpan.getEvent()); List entitiesList = rawSpan.getEntityList(); + + if (nonNull(rawSpan.getResource())) { + // If a resource exists on span, record the resource and its index + rawSpan + .getEvent() + .setResourceIndex( + resourceIndexMap.computeIfAbsent( + rawSpan.getResource(), unused -> resourceIndexMap.size())); + } for (Entity entity : entitiesList) { if (!entityMap.containsKey(entity.getEntityId())) { entityMap.put(entity.getEntityId(), entity); @@ -440,7 +452,8 @@ public static StructuredTrace buildStructuredTraceFromRawSpans(List raw entityMap, customerId, traceId, - timestamps); + timestamps, + List.copyOf(resourceIndexMap.keySet())); return structuredTraceBuilder.buildStructuredTrace(); } diff --git a/data-model/src/test/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilderTest.java b/data-model/src/test/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilderTest.java new file mode 100644 index 0000000..a98599f --- /dev/null +++ b/data-model/src/test/java/org/hypertrace/core/datamodel/shared/trace/StructuredTraceBuilderTest.java @@ -0,0 +1,91 @@ +package org.hypertrace.core.datamodel.shared.trace; + +import static java.util.Objects.nonNull; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import javax.annotation.Nullable; +import org.hypertrace.core.datamodel.Attributes; +import org.hypertrace.core.datamodel.Event; +import org.hypertrace.core.datamodel.RawSpan; +import org.hypertrace.core.datamodel.Resource; +import org.hypertrace.core.datamodel.StructuredTrace; +import org.junit.jupiter.api.Test; + +class StructuredTraceBuilderTest { + private static final String CUSTOMER_ID = "customer-id"; + private static final AtomicInteger CREATION_COUNTER = new AtomicInteger(0); + private static final ByteBuffer TRACE_ID = ByteBuffer.wrap("trace-id".getBytes()); + + @Test + void correctlyDematerializesResourcesFromRawSpans() { + Resource firstHostResource = buildResource(Map.of("host", "first-host")); + Resource duplicateFirstHostResource = buildResource(Map.of("host", "first-host")); + Resource secondHostResource = buildResource(Map.of("host", "second-host")); + Resource firstHostExtendedResource = + buildResource(Map.of("host", "first-host", "other-key", "other-value")); + + List rawSpanList = + List.of( + this.buildRawSpanHoldingResource("first", firstHostResource), + this.buildRawSpanHoldingResource("second", secondHostResource), + this.buildRawSpanHoldingResource("third", duplicateFirstHostResource), + this.buildRawSpanHoldingResource("fourth", null), + this.buildRawSpanHoldingResource("fifth", firstHostExtendedResource)); + + StructuredTrace trace = + StructuredTraceBuilder.buildStructuredTraceFromRawSpans(rawSpanList, TRACE_ID, CUSTOMER_ID); + + assertEquals( + List.of(firstHostResource, secondHostResource, firstHostExtendedResource), + trace.getResourceList()); + + assertEventMatches(trace.getEventList().get(0), "first", 0); + assertEventMatches(trace.getEventList().get(1), "second", 1); + assertEventMatches(trace.getEventList().get(2), "third", 0); + assertEventMatches(trace.getEventList().get(3), "fourth", -1); + assertEventMatches(trace.getEventList().get(4), "fifth", 2); + } + + private RawSpan buildRawSpanHoldingResource(String id, @Nullable Resource resource) { + RawSpan.Builder builder = + RawSpan.newBuilder() + .setCustomerId(CUSTOMER_ID) + .setTraceId(TRACE_ID) + .setEvent( + Event.newBuilder() + .setEventId(ByteBuffer.wrap(id.getBytes())) + .setStartTimeMillis( + CREATION_COUNTER.getAndIncrement()) // Used for predictable ordering + .setCustomerId(CUSTOMER_ID) + .build()); + + if (nonNull(resource)) { + builder.setResource(resource); + } + return builder.build(); + } + + private void assertEventMatches(Event event, String id, int resourceIndex) { + assertEquals(id, Charset.defaultCharset().decode(event.getEventId()).toString()); + assertEquals(resourceIndex, event.getResourceIndex()); + } + + private Resource buildResource(Map resourceMap) { + return resourceMap.entrySet().stream() + .collect( + Collectors.collectingAndThen( + Collectors.toMap( + Entry::getKey, entry -> AttributeValueCreator.create(entry.getValue())), + map -> + Resource.newBuilder() + .setAttributes(Attributes.newBuilder().setAttributeMap(map).build()) + .build())); + } +}