From 836a658cea5d2851e14cd409af6022a51ad2573f Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Mon, 9 Oct 2023 14:26:14 +0800 Subject: [PATCH] feat: add entity count distribution --- .../serving/monitoring/MonitoringConfig.java | 3 +++ .../serving/monitoring/ServingMetrics.java | 21 +++++++++++++++++++ .../src/main/resources/application.yaml | 8 +++++++ 3 files changed, 32 insertions(+) diff --git a/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/MonitoringConfig.java b/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/MonitoringConfig.java index fda14e9..dc9bcda 100644 --- a/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/MonitoringConfig.java +++ b/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/MonitoringConfig.java @@ -13,5 +13,8 @@ public class MonitoringConfig { record TimerConfig(List percentiles, Integer minBucketMs, Integer maxBucketMs) {} + record EntityCountDistributionConfig(List percentiles, Integer maxExpectedCount) {} + private TimerConfig timer; + private EntityCountDistributionConfig entityCountDistribution; } diff --git a/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/ServingMetrics.java b/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/ServingMetrics.java index 114d4a0..5824996 100644 --- a/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/ServingMetrics.java +++ b/caraml-store-serving/src/main/java/dev/caraml/serving/monitoring/ServingMetrics.java @@ -4,6 +4,7 @@ import dev.caraml.store.protobuf.serving.ServingServiceProto.GetOnlineFeaturesRequest; import io.grpc.Status; import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Timer; import java.time.Duration; @@ -61,6 +62,24 @@ private List newEntityCounters(GetOnlineFeaturesRequest featureRequest) .toList(); } + private List newEntityCountHistograms( + GetOnlineFeaturesRequest featureRequest) { + return featureRequest.getFeaturesList().stream() + .map(FeatureReference::getFeatureTable) + .distinct() + .map( + table -> + DistributionSummary.builder("caraml_serving_entity_count_histogram") + .tag("project", featureRequest.getProject()) + .tag("feature_table", table) + .publishPercentiles( + config.getEntityCountDistribution().percentiles().stream() + .mapToDouble(Double::doubleValue) + .toArray()) + .register(registry)) + .toList(); + } + private Counter newKeyRetrievalCounter(String project) { return Counter.builder("caraml_serving_key_retrieval_count") .tag("project", project) @@ -75,6 +94,8 @@ public void onRequestReceived(ReqT requestMessage) { newKeyRetrievalCounter(project).increment(featureRequest.getEntityRowsCount()); newEntityCounters(featureRequest) .forEach(counter -> counter.increment(featureRequest.getEntityRowsCount())); + newEntityCountHistograms(featureRequest) + .forEach(histogram -> histogram.record(featureRequest.getEntityRowsCount())); } } diff --git a/caraml-store-serving/src/main/resources/application.yaml b/caraml-store-serving/src/main/resources/application.yaml index bbf4fe8..5b01830 100644 --- a/caraml-store-serving/src/main/resources/application.yaml +++ b/caraml-store-serving/src/main/resources/application.yaml @@ -23,6 +23,14 @@ caraml: minBucketMs: 1 # Maximum expected latency maxBucketMs: 200 + entityCountDistribution: + # Published percentiles + percentiles: + - 0.5 + - 0.95 + - 0.99 + # Maximum expected entity count + maxExpectedCount: 150 store: # Active store. Possible values: [redisCluster, redis, bigtable]