Skip to content

Commit

Permalink
OpenTelemetry instrumentation example (#296)
Browse files Browse the repository at this point in the history
OTeL instrumentation example
  • Loading branch information
emanuelef authored Jan 16, 2024
1 parent aeb6fa0 commit 2590097
Show file tree
Hide file tree
Showing 9 changed files with 279 additions and 3 deletions.
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,9 @@ Each sample demonstrates one feature of the SDK, together with tests.
Execution and Activity Executions. Additional
documentation: [How to use tracing in Go](https://docs.temporal.io/go/tracing).

- [**OpenTelemetry**](./opentelemetry): Demonstrates how to instrument the Workflows and
Activities with OpenTelemetry.

- [**Updatable Timer**](./updatabletimer): Demonstrates timer
cancellation and use of a Selector to wait on a Future and a Channel simultaneously.

Expand Down
8 changes: 5 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,13 @@ require (
github.com/temporalio/tctl v1.18.0
github.com/uber-go/tally/v4 v4.1.7
github.com/uber/jaeger-client-go v2.30.0+incompatible
go.opentelemetry.io/otel v1.16.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0
go.opentelemetry.io/otel/sdk v1.16.0
go.opentelemetry.io/otel/trace v1.16.0
go.temporal.io/api v1.24.1-0.20231003165936-bb03061759c8
go.temporal.io/sdk v1.25.1
go.temporal.io/sdk/contrib/opentelemetry v0.3.0
go.temporal.io/sdk/contrib/opentracing v0.1.0
go.temporal.io/sdk/contrib/tally v0.2.0
go.temporal.io/sdk/contrib/tools/workflowcheck v0.0.0-20230612164027-11c2cb9e7d2d
Expand Down Expand Up @@ -75,17 +80,14 @@ require (
github.com/twmb/murmur3 v1.1.8 // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.39.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
go.opentelemetry.io/otel/sdk/metric v0.39.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.opentelemetry.io/proto/otlp v0.20.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/dig v1.17.0 // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1270,6 +1270,7 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ=
go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I=
go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0=
Expand All @@ -1284,12 +1285,16 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo=
go.opentelemetry.io/otel/exporters/prometheus v0.39.0 h1:whAaiHxOatgtKd+w0dOi//1KUxj3KoPINZdtDaDj3IA=
go.opentelemetry.io/otel/exporters/prometheus v0.39.0/go.mod h1:4jo5Q4CROlCpSPsXLhymi+LYrDXd2ObU5wbKayfZs7Y=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0 h1:+XWJd3jf75RXJq29mxbuXhCXFDG3S3R4vBUeSI2P7tE=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.16.0/go.mod h1:hqgzBPTf4yONMFgdZvL/bK42R/iinTyVQtiWihs3SZc=
go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo=
go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U=
go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE=
go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4=
go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI=
go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI=
go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0=
go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
Expand All @@ -1303,6 +1308,8 @@ go.temporal.io/api v1.24.1-0.20231003165936-bb03061759c8/go.mod h1:GyVOkCMSlZSqS
go.temporal.io/sdk v1.12.0/go.mod h1:lSp3lH1lI0TyOsus0arnO3FYvjVXBZGi/G7DjnAnm6o=
go.temporal.io/sdk v1.25.1 h1:jC9l9vHHz5OJ7PR6OjrpYSN4+uEG0bLe5rdF9nlMSGk=
go.temporal.io/sdk v1.25.1/go.mod h1:X7iFKZpsj90BfszfpFCzLX8lwEJXbnRrl351/HyEgmU=
go.temporal.io/sdk/contrib/opentelemetry v0.3.0 h1:wOEErZugJF1NcBpcxEAmOQmrcRBw3Plx5+LacEbZ1CE=
go.temporal.io/sdk/contrib/opentelemetry v0.3.0/go.mod h1:V0kUHBikUWh3fb4WufUKXd8tKrFO3isZM+MmrfJKIGM=
go.temporal.io/sdk/contrib/opentracing v0.1.0 h1:z5F1EoZQ6v8bMH/pzcFRgzoPKZCB+exUKMYt0jzuWME=
go.temporal.io/sdk/contrib/opentracing v0.1.0/go.mod h1:rQavYHnrTXlFKvKrn5R91eOLdvaJJ27gtxaKFqH3LuM=
go.temporal.io/sdk/contrib/tally v0.2.0 h1:XnTJIQcjOv+WuCJ1u8Ve2nq+s2H4i/fys34MnWDRrOo=
Expand Down
52 changes: 52 additions & 0 deletions opentelemetry/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
### Steps to run this sample:
1) Run a [Temporal service](https://github.com/temporalio/samples-go/tree/main/#how-to-use).

One way could be just to use the Temporal CLI.

```bash
temporal server start-dev
```

2) Run the following command to start the worker
```bash
go run opentelemetry/worker/main.go
```
3) In another terminal, run the following command to run the workflow
```bash
go run opentelemetry/starter/main.go
```

The example outputs the traces in the stdout, both the worker and the starter.

If all is needed is to see Workflows and Activities there's no need to set up instrumentation for the Temporal cluster.

In order to send the traces to a real service you need to replace

```go
exp, err := stdouttrace.New(stdouttrace.WithPrettyPrint())
if err != nil {
log.Fatalln("failed to initialize stdouttrace exporter", err)
}
```
with
```go
// Configure a new OTLP exporter using environment variables for sending data to Honeycomb over gRPC
clientOTel := otlptracegrpc.NewClient()
exp, err := otlptrace.New(ctx, clientOTel)
if err != nil {
log.Fatalf("failed to initialize exporter: %e", err)
}
```

And provide the required additional parameters like the OTLP endpoint.
For many services that would mean just to set the standard OTeL env vars like:

```
OTEL_SERVICE_NAME
OTEL_EXPORTER_OTLP_ENDPOINT
OTEL_EXPORTER_OTLP_HEADERS
```

As an example this is what is the rendered by Honeycomb.io.

![Honeycomb.io](honeycomb_traces.png)
Binary file added opentelemetry/honeycomb_traces.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
59 changes: 59 additions & 0 deletions opentelemetry/opentelemetry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package opentelemetry

import (
"context"
"time"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.temporal.io/sdk/activity"
"go.temporal.io/sdk/workflow"
)

var tracer trace.Tracer

func init() {
// Name the tracer after the package, or the service if you are in main
tracer = otel.Tracer("github.com/temporalio/samples-go/otel")
}

func Workflow(ctx workflow.Context, name string) error {
logger := workflow.GetLogger(ctx)
logger.Info("HelloWorld workflow started", "name", name)

ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
StartToCloseTimeout: 10 * time.Second,
})

err := workflow.ExecuteActivity(ctx, Activity).Get(ctx, nil)

if err != nil {
logger.Error("Activity failed.", "Error", err)
return err
}

logger.Info("HelloWorld workflow completed.")
return nil
}

func Activity(ctx context.Context, name string) error {
logger := activity.GetLogger(ctx)
logger.Info("Activity", "name", name)

// Get current span and add new attributes
span := trace.SpanFromContext(ctx)
span.SetAttributes(attribute.Bool("isTrue", true), attribute.String("stringAttr", "Ciao"))

// Create a child span
_, childSpan := tracer.Start(ctx, "custom-span")
time.Sleep(1 * time.Second)
childSpan.End()

time.Sleep(1 * time.Second)

// Add an event to the current span
span.AddEvent("Done Activity")

return nil
}
36 changes: 36 additions & 0 deletions opentelemetry/opentelemetry_setup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package opentelemetry

import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
)

func InitializeGlobalTracerProvider() (*sdktrace.TracerProvider, error) {
// Initialize tracer
exp, err := stdouttrace.New(stdouttrace.WithPrettyPrint())
if err != nil {
return nil, err
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exp),
sdktrace.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceName("temporal-example"),
semconv.ServiceVersion("0.0.1"),
)),
)
otel.SetTracerProvider(tp)

otel.SetTextMapPropagator(
propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
),
)

return tp, nil
}
63 changes: 63 additions & 0 deletions opentelemetry/starter/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package main

import (
"context"
"log"

otelworkflow "github.com/temporalio/samples-go/opentelemetry"
"go.temporal.io/sdk/client"
"go.temporal.io/sdk/contrib/opentelemetry"
"go.temporal.io/sdk/interceptor"
)

func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

tp, err := otelworkflow.InitializeGlobalTracerProvider()
if err != nil {
log.Fatalln("Unable to create a global trace provider", err)
}

defer func() {
if err := tp.Shutdown(ctx); err != nil {
log.Println("Error shutting down trace provider:", err)
}
}()

tracingInterceptor, err := opentelemetry.NewTracingInterceptor(opentelemetry.TracerOptions{})
if err != nil {
log.Fatalln("Unable to create interceptor", err)
}

options := client.Options{
Interceptors: []interceptor.ClientInterceptor{tracingInterceptor},
}

// The client is a heavyweight object that should be created once per process.
c, err := client.Dial(options)
if err != nil {
log.Fatalln("Unable to create client", err)
}
defer c.Close()

workflowOptions := client.StartWorkflowOptions{
ID: "otel_workflowID",
TaskQueue: "otel",
}

we, err := c.ExecuteWorkflow(context.Background(), workflowOptions, otelworkflow.Workflow, "Temporal")
if err != nil {
log.Fatalln("Unable to execute workflow", err)
}

log.Println("Started workflow", "WorkflowID", we.GetID(), "RunID", we.GetRunID())

// Synchronously wait for the workflow completion.
var result string
err = we.Get(context.Background(), &result)
if err != nil {
log.Fatalln("Unable get workflow result", err)
}
log.Println("Workflow result:", result)
}
54 changes: 54 additions & 0 deletions opentelemetry/worker/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package main

import (
"context"
"log"

otelworkflow "github.com/temporalio/samples-go/opentelemetry"
"go.temporal.io/sdk/client"
"go.temporal.io/sdk/contrib/opentelemetry"
"go.temporal.io/sdk/interceptor"
"go.temporal.io/sdk/worker"
)

func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

tp, err := otelworkflow.InitializeGlobalTracerProvider()
if err != nil {
log.Fatalln("Unable to create a global trace provider", err)
}

defer func() {
if err := tp.Shutdown(ctx); err != nil {
log.Println("Error shutting down trace provider:", err)
}
}()

tracingInterceptor, err := opentelemetry.NewTracingInterceptor(opentelemetry.TracerOptions{})
if err != nil {
log.Fatalln("Unable to create interceptor", err)
}

options := client.Options{
Interceptors: []interceptor.ClientInterceptor{tracingInterceptor},
}

// The client and worker are heavyweight objects that should be created once per process.
c, err := client.Dial(options)
if err != nil {
log.Fatalln("Unable to create client", err)
}
defer c.Close()

w := worker.New(c, "otel", worker.Options{})

w.RegisterWorkflow(otelworkflow.Workflow)
w.RegisterActivity(otelworkflow.Activity)

err = w.Run(worker.InterruptCh())
if err != nil {
log.Fatalln("Worker run failed", err)
}
}

0 comments on commit 2590097

Please sign in to comment.