Add stdout layer + review comments

rust/worker/Cargo.toml

@@ -45,6 +45,7 @@ arrow = "50.0.0"
 roaring = "0.10.3"
 tantivy = "0.21.1"
 tracing = "0.1"
+tracing-bunyan-formatter = "0.3.3"
 tracing-opentelemetry = "0.19.0"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 opentelemetry = { version = "0.19.0", default-features = false, features = ["trace", "rt-tokio"] }

rust/worker/chroma_config.yaml

@@ -4,7 +4,7 @@
 # for now we nest it in the worker directory
 
 query_service:
-    svc_name: "query-service"
+    service_name: "query-service"
     otel_endpoint: "http://jaeger:4317"
     my_ip: "10.244.0.9"
     my_port: 50051
@@ -34,7 +34,7 @@ query_service:
         worker_queue_size: 100
 
 compaction_service:
-    svc_name: "compaction-service"
+    service_name: "compaction-service"
     otel_endpoint: "http://jaeger:4317"
     my_ip: "10.244.0.9"
     my_port: 50051

rust/worker/src/config.rs

@@ -93,7 +93,7 @@ impl RootConfig {
 /// Each submodule that needs to be configured from the config object should implement the Configurable trait and
 /// have its own field in this struct for its Config struct.
 pub(crate) struct QueryServiceConfig {
-    pub(crate) svc_name: String,
+    pub(crate) service_name: String,
     pub(crate) otel_endpoint: String,
     pub(crate) my_ip: String,
     pub(crate) my_port: u16,
@@ -117,7 +117,7 @@ pub(crate) struct QueryServiceConfig {
 /// Each submodule that needs to be configured from the config object should implement the Configurable trait and
 /// have its own field in this struct for its Config struct.
 pub(crate) struct CompactionServiceConfig {
-    pub(crate) svc_name: String,
+    pub(crate) service_name: String,
     pub(crate) otel_endpoint: String,
     pub(crate) my_ip: String,
     pub(crate) my_port: u16,

rust/worker/src/lib.rs

@@ -38,7 +38,7 @@ pub async fn query_service_entrypoint() {
     let config = config.query_service;
 
     crate::tracing::opentelemetry_config::init_otel_tracing(
-        &config.svc_name,
+        &config.service_name,
         &config.otel_endpoint,
     );
 
@@ -103,7 +103,7 @@ pub async fn compaction_service_entrypoint() {
     let config = config.compaction_service;
 
     crate::tracing::opentelemetry_config::init_otel_tracing(
-        &config.svc_name,
+        &config.service_name,
         &config.otel_endpoint,
     );
 

rust/worker/src/tracing/opentelemetry_config.rs

@@ -2,10 +2,14 @@ use opentelemetry::global;
 use opentelemetry::sdk::propagation::TraceContextPropagator;
 use opentelemetry::sdk::trace;
 use opentelemetry_otlp::WithExportConfig;
-use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::{EnvFilter, Registry};
+use tracing_bunyan_formatter::BunyanFormattingLayer;
+use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer};
 
 pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) {
+    println!(
+        "Registering jaeger subscriber for {} at endpoint {}",
+        service_name, otel_endpoint
+    );
     let resource = opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
         "service.name",
         service_name.clone(),
@@ -14,7 +18,7 @@ pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) {
     let trace_config = trace::config()
         .with_sampler(opentelemetry::sdk::trace::Sampler::AlwaysOn)
         .with_resource(resource);
-    // Prepare exporter. Jaeger only for now.
+    // Prepare exporter.
     let exporter = opentelemetry_otlp::new_exporter()
         .tonic()
         .with_endpoint(otel_endpoint);
@@ -25,13 +29,23 @@ pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) {
         .install_batch(opentelemetry::runtime::Tokio)
         .expect("Error - Failed to create tracer.");
     // Layer for adding our configured tracer.
-    let tracing_layer = tracing_opentelemetry::layer().with_tracer(otlp_tracer);
-    // Level filter layer to filter traces based on level (trace, debug, info, warn, error).
-    let level_filter_layer = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO"));
-    let subscriber = Registry::default()
-        .with(tracing_layer)
-        .with(level_filter_layer);
+    // Export everything at this layer. The backend i.e. honeycomb or jaeger will filter at its end.
+    let exporter_layer = tracing_opentelemetry::layer()
+        .with_tracer(otlp_tracer)
+        .with_filter(tracing_subscriber::filter::LevelFilter::TRACE);
+    // Layer for printing spans to stdout. Only print INFO logs by default.
+    let stdout_layer =
+        BunyanFormattingLayer::new(service_name.clone().to_string(), std::io::stdout)
+            .with_filter(tracing_subscriber::filter::LevelFilter::INFO);
+    // global filter layer. Don't filter anything at global layer.
+    let global_layer = EnvFilter::new("TRACE");
+    // Create subscriber.
+    let subscriber = tracing_subscriber::registry()
+        .with(global_layer)
+        .with(stdout_layer)
+        .with(exporter_layer);
     global::set_text_map_propagator(TraceContextPropagator::new());
     tracing::subscriber::set_global_default(subscriber)
         .expect("Set global default subscriber failed");
+    println!("Set global subscriber for {}", service_name);
 }