diff --git a/rust/worker/Cargo.toml b/rust/worker/Cargo.toml index 93a5130d410..a70028e8c05 100644 --- a/rust/worker/Cargo.toml +++ b/rust/worker/Cargo.toml @@ -46,7 +46,7 @@ roaring = "0.10.3" tantivy = "0.21.1" tracing = "0.1" tracing-opentelemetry = "0.19.0" -tracing-subscriber = "0.3" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } opentelemetry = { version = "0.19.0", default-features = false, features = ["trace", "rt-tokio"] } opentelemetry-otlp = "0.12.0" diff --git a/rust/worker/chroma_config.yaml b/rust/worker/chroma_config.yaml index 95eb1924932..565fc370016 100644 --- a/rust/worker/chroma_config.yaml +++ b/rust/worker/chroma_config.yaml @@ -4,6 +4,8 @@ # for now we nest it in the worker directory query_service: + svc_name: "query-service" + otel_endpoint: "http://jaeger:4317" my_ip: "10.244.0.9" my_port: 50051 assignment_policy: @@ -32,6 +34,8 @@ query_service: worker_queue_size: 100 compaction_service: + svc_name: "compaction-service" + otel_endpoint: "http://jaeger:4317" my_ip: "10.244.0.9" my_port: 50051 assignment_policy: diff --git a/rust/worker/src/config.rs b/rust/worker/src/config.rs index a441074f146..3c23aba0dca 100644 --- a/rust/worker/src/config.rs +++ b/rust/worker/src/config.rs @@ -93,6 +93,8 @@ impl RootConfig { /// Each submodule that needs to be configured from the config object should implement the Configurable trait and /// have its own field in this struct for its Config struct. pub(crate) struct QueryServiceConfig { + pub(crate) svc_name: String, + pub(crate) otel_endpoint: String, pub(crate) my_ip: String, pub(crate) my_port: u16, pub(crate) assignment_policy: crate::assignment::config::AssignmentPolicyConfig, @@ -115,6 +117,8 @@ pub(crate) struct QueryServiceConfig { /// Each submodule that needs to be configured from the config object should implement the Configurable trait and /// have its own field in this struct for its Config struct. pub(crate) struct CompactionServiceConfig { + pub(crate) svc_name: String, + pub(crate) otel_endpoint: String, pub(crate) my_ip: String, pub(crate) my_port: u16, pub(crate) assignment_policy: crate::assignment::config::AssignmentPolicyConfig, diff --git a/rust/worker/src/lib.rs b/rust/worker/src/lib.rs index 47292a5d515..d28e4487a66 100644 --- a/rust/worker/src/lib.rs +++ b/rust/worker/src/lib.rs @@ -35,9 +35,13 @@ pub async fn query_service_entrypoint() { Err(_) => config::RootConfig::load(), }; - crate::tracing::opentelemetry_config::init_oltp_tracing(); - let config = config.query_service; + + crate::tracing::opentelemetry_config::init_otel_tracing( + &config.svc_name, + &config.otel_endpoint, + ); + let system: system::System = system::System::new(); let dispatcher = match execution::dispatcher::Dispatcher::try_from_config(&config.dispatcher).await { @@ -97,6 +101,12 @@ pub async fn compaction_service_entrypoint() { }; let config = config.compaction_service; + + crate::tracing::opentelemetry_config::init_otel_tracing( + &config.svc_name, + &config.otel_endpoint, + ); + let system: system::System = system::System::new(); let mut memberlist = match memberlist::CustomResourceMemberlistProvider::try_from_config( diff --git a/rust/worker/src/tracing/opentelemetry_config.rs b/rust/worker/src/tracing/opentelemetry_config.rs index ac8c01dded1..36a0f7cd352 100644 --- a/rust/worker/src/tracing/opentelemetry_config.rs +++ b/rust/worker/src/tracing/opentelemetry_config.rs @@ -3,13 +3,12 @@ use opentelemetry::sdk::propagation::TraceContextPropagator; use opentelemetry::sdk::trace; use opentelemetry_otlp::WithExportConfig; use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::Registry; +use tracing_subscriber::{EnvFilter, Registry}; -pub(crate) fn init_oltp_tracing() { +pub(crate) fn init_otel_tracing(service_name: &String, otel_endpoint: &String) { let resource = opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new( "service.name", - "sanket-test", + service_name.clone(), )]); // Prepare trace config. let trace_config = trace::config() @@ -18,7 +17,7 @@ pub(crate) fn init_oltp_tracing() { // Prepare exporter. Jaeger only for now. let exporter = opentelemetry_otlp::new_exporter() .tonic() - .with_endpoint("http://jaeger:4317"); + .with_endpoint(otel_endpoint); let otlp_tracer = opentelemetry_otlp::new_pipeline() .tracing() .with_exporter(exporter) @@ -27,7 +26,12 @@ pub(crate) fn init_oltp_tracing() { .expect("Error - Failed to create tracer."); // Layer for adding our configured tracer. let tracing_layer = tracing_opentelemetry::layer().with_tracer(otlp_tracer); + // Level filter layer to filter traces based on level (trace, debug, info, warn, error). + let level_filter_layer = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO")); + let subscriber = Registry::default() + .with(tracing_layer) + .with(level_filter_layer); global::set_text_map_propagator(TraceContextPropagator::new()); - - Registry::default().with(tracing_layer).init(); + tracing::subscriber::set_global_default(subscriber) + .expect("Set global default subscriber failed"); }