From 4d88246e9cb528b43e5937d53d4ebcf2bb1cc978 Mon Sep 17 00:00:00 2001 From: Michael Nedokushev Date: Sat, 20 Apr 2024 22:44:13 +0100 Subject: [PATCH] Add propagation of ZIO log annotations to OTEL metric attributes (#823) * Add propagation of ZIO log annotations to OTEL metric attributes * Update docs --- docs/opentelemetry.md | 8 ++- .../opentelemetry/OpenTelemetry.scala | 5 +- .../opentelemetry/metrics/Counter.scala | 13 +++- .../opentelemetry/metrics/Histogram.scala | 13 +++- .../opentelemetry/metrics/UpDownCounter.scala | 13 +++- .../metrics/internal/Instrument.scala | 8 +-- .../metrics/internal/package.scala | 15 ++++ .../opentelemetry/metrics/MeterTest.scala | 69 +++++++++++++++++-- scala-cli/opentelemetry/MetricsApp.scala | 7 +- 9 files changed, 125 insertions(+), 26 deletions(-) diff --git a/docs/opentelemetry.md b/docs/opentelemetry.md index 3a82968a..d4ff7490 100644 --- a/docs/opentelemetry.md +++ b/docs/opentelemetry.md @@ -187,6 +187,7 @@ object TracingApp extends ZIOAppDefault { To send [Metric signals](https://opentelemetry.io/docs/concepts/signals/metrics/), you will need a `Meter` service in your environment. For this, use the `OpenTelemetry.meter` layer which in turn requires an instance of `OpenTelemetry` provided by Java SDK and a suitable `ContextStorage` implementation. The `Meter` API lets you create [Counter](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#counter), [UpDownCounter](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#updowncounter), [Gauge](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#gauge), [Histogram](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#histogram) and their [asynchronous](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md#asynchronous-instrument-api) (aka observable) counterparts. As a rule of thumb, observable instruments must be initialized on an application startup. They are scoped, so you should not be worried about shutting them down manually. +By default the metric instruments does not take ZIO log annotations into account. To turn it on pass `logAnnotated = true` parameter to the `OpenTelemetry.metrics` layer initializer. ```scala //> using scala "2.13.13" @@ -316,12 +317,13 @@ object MetricsApp extends ZIOAppDefault { _ <- messageLengthCounter.add(message.length, Attributes(Attribute.string("message", message))) } yield message - // By wrapping our logic into a span, we make the `messageLengthCounter` data points correlated with a "root_span" automatically - logic @@ tracing.aspects.root("root_span") + // By wrapping our logic into a span, we make the `messageLengthCounter` data points correlated with a "root_span" automatically. + // Additionally we implicitly add one more attribute to the `messageLenghtCounter` as it is wrapped into a `ZIO.logAnnotate` call. + ZIO.logAnnotate("zio", "annotation")(logic) @@ tracing.aspects.root("root_span") } .provide( otelSdkLayer, - OpenTelemetry.metrics(instrumentationScopeName), + OpenTelemetry.metrics(instrumentationScopeName, logAnnotated = true), OpenTelemetry.tracing(instrumentationScopeName), OpenTelemetry.contextZIO, tickCounterLayer, diff --git a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/OpenTelemetry.scala b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/OpenTelemetry.scala index 36ef7169..26b774a6 100644 --- a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/OpenTelemetry.scala +++ b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/OpenTelemetry.scala @@ -87,7 +87,8 @@ object OpenTelemetry { def metrics( instrumentationScopeName: String, instrumentationVersion: Option[String] = None, - schemaUrl: Option[String] = None + schemaUrl: Option[String] = None, + logAnnotated: Boolean = false ): URLayer[api.OpenTelemetry with ContextStorage, Meter with Instrument.Builder] = { val meterLayer = ZLayer( ZIO.serviceWith[api.OpenTelemetry] { openTelemetry => @@ -99,7 +100,7 @@ object OpenTelemetry { builder.build() } ) - val builderLayer = meterLayer >>> Instrument.Builder.live + val builderLayer = meterLayer >>> Instrument.Builder.live(logAnnotated) builderLayer >+> (builderLayer >>> Meter.live) } diff --git a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Counter.scala b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Counter.scala index cf9ed3ee..762477c1 100644 --- a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Counter.scala +++ b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Counter.scala @@ -5,7 +5,7 @@ import io.opentelemetry.api.metrics.LongCounter import io.opentelemetry.context.Context import zio._ import zio.telemetry.opentelemetry.context.ContextStorage -import zio.telemetry.opentelemetry.metrics.internal.Instrument +import zio.telemetry.opentelemetry.metrics.internal.{Instrument, logAnnotatedAttributes} /** * A Counter instrument that records values of type `A` @@ -42,14 +42,21 @@ trait Counter[-A] extends Instrument[A] { object Counter { - private[metrics] def long(counter: LongCounter, ctxStorage: ContextStorage): Counter[Long] = + private[metrics] def long( + counter: LongCounter, + ctxStorage: ContextStorage, + logAnnotated: Boolean + ): Counter[Long] = new Counter[Long] { override def record0(value: Long, attributes: Attributes = Attributes.empty, context: Context): Unit = counter.add(value, attributes, context) override def add(value: Long, attributes: Attributes = Attributes.empty)(implicit trace: Trace): UIO[Unit] = - ctxStorage.get.map(record0(value, attributes, _)) + for { + annotated <- logAnnotatedAttributes(attributes, logAnnotated) + ctx <- ctxStorage.get + } yield record0(value, annotated, ctx) override def inc(attributes: Attributes = Attributes.empty)(implicit trace: Trace): UIO[Unit] = add(1L, attributes) diff --git a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Histogram.scala b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Histogram.scala index ed265692..e935d6f3 100644 --- a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Histogram.scala +++ b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/Histogram.scala @@ -5,7 +5,7 @@ import io.opentelemetry.api.metrics.DoubleHistogram import io.opentelemetry.context.Context import zio._ import zio.telemetry.opentelemetry.context.ContextStorage -import zio.telemetry.opentelemetry.metrics.internal.Instrument +import zio.telemetry.opentelemetry.metrics.internal.{Instrument, logAnnotatedAttributes} /** * A Histogram instrument that records values of type `A` @@ -32,14 +32,21 @@ trait Histogram[-A] extends Instrument[A] { object Histogram { - private[metrics] def double(histogram: DoubleHistogram, ctxStorage: ContextStorage): Histogram[Double] = + private[metrics] def double( + histogram: DoubleHistogram, + ctxStorage: ContextStorage, + logAnnotated: Boolean + ): Histogram[Double] = new Histogram[Double] { override def record0(value: Double, attributes: Attributes = Attributes.empty, context: Context): Unit = histogram.record(value, attributes, context) override def record(value: Double, attributes: Attributes = Attributes.empty)(implicit trace: Trace): UIO[Unit] = - ctxStorage.get.map(record0(value, attributes, _)) + for { + annotated <- logAnnotatedAttributes(attributes, logAnnotated) + ctx <- ctxStorage.get + } yield record0(value, annotated, ctx) } diff --git a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/UpDownCounter.scala b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/UpDownCounter.scala index 1d512362..de53804a 100644 --- a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/UpDownCounter.scala +++ b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/UpDownCounter.scala @@ -5,7 +5,7 @@ import io.opentelemetry.api.metrics.LongUpDownCounter import io.opentelemetry.context.Context import zio._ import zio.telemetry.opentelemetry.context.ContextStorage -import zio.telemetry.opentelemetry.metrics.internal.Instrument +import zio.telemetry.opentelemetry.metrics.internal.{Instrument, logAnnotatedAttributes} /** * A UpDownCounter instrument that records values of type `A` @@ -54,14 +54,21 @@ trait UpDownCounter[-A] extends Instrument[A] { object UpDownCounter { - private[metrics] def long(counter: LongUpDownCounter, ctxStorage: ContextStorage): UpDownCounter[Long] = + private[metrics] def long( + counter: LongUpDownCounter, + ctxStorage: ContextStorage, + logAnnotated: Boolean + ): UpDownCounter[Long] = new UpDownCounter[Long] { override def record0(value: Long, attributes: Attributes = Attributes.empty, context: Context): Unit = counter.add(value, attributes, context) override def add(value: Long, attributes: Attributes = Attributes.empty)(implicit trace: Trace): UIO[Unit] = - ctxStorage.get.map(record0(value, attributes, _)) + for { + annotated <- logAnnotatedAttributes(attributes, logAnnotated) + ctx <- ctxStorage.get + } yield record0(value, annotated, ctx) override def inc(attributes: Attributes = Attributes.empty)(implicit trace: Trace): UIO[Unit] = add(1L, attributes) diff --git a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/Instrument.scala b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/Instrument.scala index 1a8713eb..0ffb48d9 100644 --- a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/Instrument.scala +++ b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/Instrument.scala @@ -52,7 +52,7 @@ object Instrument { private[opentelemetry] object Builder { - def live: URLayer[api.metrics.Meter with ContextStorage, Builder] = + def live(logAnnotated: Boolean = false): URLayer[api.metrics.Meter with ContextStorage, Builder] = ZLayer( for { meter <- ZIO.service[api.metrics.Meter] @@ -69,7 +69,7 @@ object Instrument { unit.foreach(builder.setUnit) description.foreach(builder.setDescription) - Counter.long(builder.build(), ctxStorage) + Counter.long(builder.build(), ctxStorage, logAnnotated) } override def upDownCounter( @@ -82,7 +82,7 @@ object Instrument { unit.foreach(builder.setUnit) description.foreach(builder.setDescription) - UpDownCounter.long(builder.build(), ctxStorage) + UpDownCounter.long(builder.build(), ctxStorage, logAnnotated) } override def histogram( @@ -95,7 +95,7 @@ object Instrument { unit.foreach(builder.setUnit) description.foreach(builder.setDescription) - Histogram.double(builder.build(), ctxStorage) + Histogram.double(builder.build(), ctxStorage, logAnnotated) } override def observableCounter( diff --git a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/package.scala b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/package.scala index beb0f703..32902328 100644 --- a/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/package.scala +++ b/opentelemetry/src/main/scala/zio/telemetry/opentelemetry/metrics/internal/package.scala @@ -1,6 +1,7 @@ package zio.telemetry.opentelemetry.metrics import io.opentelemetry.api +import zio._ import zio.metrics.MetricLabel import zio.telemetry.opentelemetry.common.{Attribute, Attributes} @@ -9,4 +10,18 @@ package object internal { private[metrics] def attributes(tags: Set[MetricLabel]): api.common.Attributes = Attributes(tags.map(t => Attribute.string(t.key, t.value)).toSeq: _*) + private[metrics] def logAnnotatedAttributes(attributes: api.common.Attributes, logAnnotated: Boolean)(implicit + trace: Trace + ): UIO[api.common.Attributes] = + if (logAnnotated) + for { + annotations <- ZIO.logAnnotations + annotated = Attributes(annotations.map { case (k, v) => Attribute.string(k, v) }.toSeq: _*) + builder = api.common.Attributes.builder() + _ = builder.putAll(annotated) + _ = builder.putAll(attributes) + } yield builder.build() + else + ZIO.succeed(attributes) + } diff --git a/opentelemetry/src/test/scala/zio/telemetry/opentelemetry/metrics/MeterTest.scala b/opentelemetry/src/test/scala/zio/telemetry/opentelemetry/metrics/MeterTest.scala index 2f40a3e7..aab5e051 100644 --- a/opentelemetry/src/test/scala/zio/telemetry/opentelemetry/metrics/MeterTest.scala +++ b/opentelemetry/src/test/scala/zio/telemetry/opentelemetry/metrics/MeterTest.scala @@ -16,7 +16,7 @@ object MeterTest extends ZIOSpecDefault { val inMemoryMetricReaderLayer: ZLayer[Any, Nothing, InMemoryMetricReader] = ZLayer(ZIO.succeed(InMemoryMetricReader.create())) - val meterLayer: ZLayer[InMemoryMetricReader with ContextStorage, Nothing, Meter] = { + def meterLayer(logAnnotated: Boolean = false): ZLayer[InMemoryMetricReader with ContextStorage, Nothing, Meter] = { val jmeter = ZLayer { for { metricReader <- ZIO.service[InMemoryMetricReader] @@ -24,7 +24,7 @@ object MeterTest extends ZIOSpecDefault { meter <- ZIO.succeed(meterProvider.get("MeterTest")) } yield meter } - val builder = jmeter >>> Instrument.Builder.live + val builder = jmeter >>> Instrument.Builder.live(logAnnotated) builder >>> Meter.live } @@ -44,7 +44,8 @@ object MeterTest extends ZIOSpecDefault { suite("zio opentelemetry")( suite("Meter")( normalSpec, - contextualSpec + contextualSpec, + logAnnotatedSpec ) ) @@ -131,8 +132,26 @@ object MeterTest extends ZIOSpecDefault { } yield assertTrue(metricValue == 14L) } ) + }, + test("zio log annotations are not included when turned off") { + ZIO.serviceWithZIO[Meter] { meter => + for { + reader <- ZIO.service[InMemoryMetricReader] + counter <- meter.counter("test_counter") + _ <- ZIO.logAnnotate("zio", "annotation") { + counter.inc() + } + metric = reader.collectAllMetrics().asScala.toList.head + metricPoint = metric.getLongSumData.getPoints.asScala.toList.head + metricValue = metricPoint.getValue + metricAttributes = metricPoint.getAttributes() + } yield assertTrue( + metricValue == 1L, + metricAttributes == Attributes.empty + ) + } } - ).provide(inMemoryMetricReaderLayer, meterLayer, ContextStorage.fiberRef, observableRefLayer) + ).provide(inMemoryMetricReaderLayer, meterLayer(), ContextStorage.fiberRef, observableRefLayer) private val contextualSpec = suite("contextual")( @@ -155,6 +174,46 @@ object MeterTest extends ZIOSpecDefault { ) } } - ).provide(inMemoryMetricReaderLayer, meterLayer, ContextStorage.fiberRef, TracingTest.tracingMockLayer) + ).provide(inMemoryMetricReaderLayer, meterLayer(), ContextStorage.fiberRef, TracingTest.tracingMockLayer) + + private val logAnnotatedSpec = + suite("log annotated")( + test("new attributes") { + ZIO.serviceWithZIO[Meter] { meter => + for { + reader <- ZIO.service[InMemoryMetricReader] + counter <- meter.counter("test_counter") + _ <- ZIO.logAnnotate("zio", "annotation") { + counter.inc() + } + metric = reader.collectAllMetrics().asScala.toList.head + metricPoint = metric.getLongSumData.getPoints.asScala.toList.head + metricValue = metricPoint.getValue + metricAttributes = metricPoint.getAttributes() + } yield assertTrue( + metricValue == 1L, + metricAttributes == Attributes(Attribute.string("zio", "annotation")) + ) + } + }, + test("instrumented attributes override log annotated") { + ZIO.serviceWithZIO[Meter] { meter => + for { + reader <- ZIO.service[InMemoryMetricReader] + counter <- meter.counter("test_counter") + _ <- ZIO.logAnnotate("zio", "annotation") { + counter.inc(Attributes(Attribute.string("zio", "annotation2"))) + } + metric = reader.collectAllMetrics().asScala.toList.head + metricPoint = metric.getLongSumData.getPoints.asScala.toList.head + metricValue = metricPoint.getValue + metricAttributes = metricPoint.getAttributes() + } yield assertTrue( + metricValue == 1L, + metricAttributes == Attributes(Attribute.string("zio", "annotation2")) + ) + } + } + ).provide(inMemoryMetricReaderLayer, meterLayer(logAnnotated = true), ContextStorage.fiberRef) } diff --git a/scala-cli/opentelemetry/MetricsApp.scala b/scala-cli/opentelemetry/MetricsApp.scala index ad020e6d..0fb789b0 100644 --- a/scala-cli/opentelemetry/MetricsApp.scala +++ b/scala-cli/opentelemetry/MetricsApp.scala @@ -125,12 +125,13 @@ object MetricsApp extends ZIOAppDefault { _ <- messageLengthCounter.add(message.length, Attributes(Attribute.string("message", message))) } yield message - // By wrapping our logic into a span, we make the `messageLengthCounter` data points correlated with a "root_span" automatically - logic @@ tracing.aspects.root("root_span") + // By wrapping our logic into a span, we make the `messageLengthCounter` data points correlated with a "root_span" automatically. + // Additionally we implicitly add one more attribute to the `messageLenghtCounter` as it is wrapped into a `ZIO.logAnnotate` call. + ZIO.logAnnotate("zio", "annotation")(logic) @@ tracing.aspects.root("root_span") } .provide( otelSdkLayer, - OpenTelemetry.metrics(instrumentationScopeName), + OpenTelemetry.metrics(instrumentationScopeName, logAnnotated = true), OpenTelemetry.tracing(instrumentationScopeName), OpenTelemetry.contextZIO, tickCounterLayer,