diff --git a/Cargo.lock b/Cargo.lock index a9afb35aee5..ace1e4af6da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,6 +1488,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "gethostname" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ebd34e35c46e00bb73e81363248d627782724609fe1b6396f553f68fe3862e" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -3939,6 +3949,24 @@ dependencies = [ "syn 2.0.52", ] +[[package]] +name = "tracing-bunyan-formatter" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c266b9ac83dedf0e0385ad78514949e6d89491269e7065bee51d2bb8ec7373" +dependencies = [ + "ahash", + "gethostname", + "log", + "serde", + "serde_json", + "time", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + [[package]] name = "tracing-core" version = "0.1.32" @@ -4488,6 +4516,7 @@ dependencies = [ "tonic 0.10.2", "tonic-build", "tracing", + "tracing-bunyan-formatter", "tracing-opentelemetry", "tracing-subscriber", "uuid", diff --git a/rust/worker/Cargo.toml b/rust/worker/Cargo.toml index a70028e8c05..bd248dd9638 100644 --- a/rust/worker/Cargo.toml +++ b/rust/worker/Cargo.toml @@ -45,6 +45,7 @@ arrow = "50.0.0" roaring = "0.10.3" tantivy = "0.21.1" tracing = "0.1" +tracing-bunyan-formatter = "0.3.3" tracing-opentelemetry = "0.19.0" tracing-subscriber = { version = "0.3", features = ["env-filter"] } opentelemetry = { version = "0.19.0", default-features = false, features = ["trace", "rt-tokio"] } diff --git a/rust/worker/chroma_config.yaml b/rust/worker/chroma_config.yaml index 565fc370016..453bad7c99e 100644 --- a/rust/worker/chroma_config.yaml +++ b/rust/worker/chroma_config.yaml @@ -4,7 +4,7 @@ # for now we nest it in the worker directory query_service: - svc_name: "query-service" + service_name: "query-service" otel_endpoint: "http://jaeger:4317" my_ip: "10.244.0.9" my_port: 50051 @@ -34,7 +34,7 @@ query_service: worker_queue_size: 100 compaction_service: - svc_name: "compaction-service" + service_name: "compaction-service" otel_endpoint: "http://jaeger:4317" my_ip: "10.244.0.9" my_port: 50051 diff --git a/rust/worker/src/config.rs b/rust/worker/src/config.rs index 3c23aba0dca..eb53759f5d4 100644 --- a/rust/worker/src/config.rs +++ b/rust/worker/src/config.rs @@ -93,7 +93,7 @@ impl RootConfig { /// Each submodule that needs to be configured from the config object should implement the Configurable trait and /// have its own field in this struct for its Config struct. pub(crate) struct QueryServiceConfig { - pub(crate) svc_name: String, + pub(crate) service_name: String, pub(crate) otel_endpoint: String, pub(crate) my_ip: String, pub(crate) my_port: u16, @@ -117,7 +117,7 @@ pub(crate) struct QueryServiceConfig { /// Each submodule that needs to be configured from the config object should implement the Configurable trait and /// have its own field in this struct for its Config struct. pub(crate) struct CompactionServiceConfig { - pub(crate) svc_name: String, + pub(crate) service_name: String, pub(crate) otel_endpoint: String, pub(crate) my_ip: String, pub(crate) my_port: u16, diff --git a/rust/worker/src/lib.rs b/rust/worker/src/lib.rs index d28e4487a66..1ae4fde5675 100644 --- a/rust/worker/src/lib.rs +++ b/rust/worker/src/lib.rs @@ -38,7 +38,7 @@ pub async fn query_service_entrypoint() { let config = config.query_service; crate::tracing::opentelemetry_config::init_otel_tracing( - &config.svc_name, + &config.service_name, &config.otel_endpoint, ); @@ -103,7 +103,7 @@ pub async fn compaction_service_entrypoint() { let config = config.compaction_service; crate::tracing::opentelemetry_config::init_otel_tracing( - &config.svc_name, + &config.service_name, &config.otel_endpoint, ); diff --git a/rust/worker/src/tracing/opentelemetry_config.rs b/rust/worker/src/tracing/opentelemetry_config.rs index 36a0f7cd352..2630123ebcc 100644 --- a/rust/worker/src/tracing/opentelemetry_config.rs +++ b/rust/worker/src/tracing/opentelemetry_config.rs @@ -2,10 +2,14 @@ use opentelemetry::global; use opentelemetry::sdk::propagation::TraceContextPropagator; use opentelemetry::sdk::trace; use opentelemetry_otlp::WithExportConfig; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::{EnvFilter, Registry}; +use tracing_bunyan_formatter::BunyanFormattingLayer; +use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Layer}; pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) { + println!( + "Registering jaeger subscriber for {} at endpoint {}", + service_name, otel_endpoint + ); let resource = opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new( "service.name", service_name.clone(), @@ -14,7 +18,7 @@ pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) { let trace_config = trace::config() .with_sampler(opentelemetry::sdk::trace::Sampler::AlwaysOn) .with_resource(resource); - // Prepare exporter. Jaeger only for now. + // Prepare exporter. let exporter = opentelemetry_otlp::new_exporter() .tonic() .with_endpoint(otel_endpoint); @@ -25,13 +29,23 @@ pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) { .install_batch(opentelemetry::runtime::Tokio) .expect("Error - Failed to create tracer."); // Layer for adding our configured tracer. - let tracing_layer = tracing_opentelemetry::layer().with_tracer(otlp_tracer); - // Level filter layer to filter traces based on level (trace, debug, info, warn, error). - let level_filter_layer = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO")); - let subscriber = Registry::default() - .with(tracing_layer) - .with(level_filter_layer); + // Export everything at this layer. The backend i.e. honeycomb or jaeger will filter at its end. + let exporter_layer = tracing_opentelemetry::layer() + .with_tracer(otlp_tracer) + .with_filter(tracing_subscriber::filter::LevelFilter::TRACE); + // Layer for printing spans to stdout. Only print INFO logs by default. + let stdout_layer = + BunyanFormattingLayer::new(service_name.clone().to_string(), std::io::stdout) + .with_filter(tracing_subscriber::filter::LevelFilter::INFO); + // global filter layer. Don't filter anything at global layer. + let global_layer = EnvFilter::new("TRACE"); + // Create subscriber. + let subscriber = tracing_subscriber::registry() + .with(global_layer) + .with(stdout_layer) + .with(exporter_layer); global::set_text_map_propagator(TraceContextPropagator::new()); tracing::subscriber::set_global_default(subscriber) .expect("Set global default subscriber failed"); + println!("Set global subscriber for {}", service_name); }