From 30e59a848230d8ccf9412f95b5005034669643ce Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Thu, 14 Nov 2024 15:32:49 +0100 Subject: [PATCH] Move invalid metrics_name to a dedicated place and re-implement a small parser to collect them Signed-off-by: Augustin Husson --- database/database.go | 87 +- go.mod | 16 +- go.sum | 52 +- pkg/analyze/grafana/grafana.go | 71 +- pkg/analyze/grafana/grafana_test.go | 71 +- pkg/analyze/grafana/parser.go | 81 + pkg/analyze/grafana/parser_test.go | 50 + pkg/analyze/grafana/tests/d4.json | 5427 +++++++++++++++++++++ pkg/analyze/perses/perses.go | 4 +- pkg/analyze/prometheus/prometheus.go | 31 +- pkg/analyze/prometheus/prometheus_test.go | 2 +- pkg/client/client.go | 16 + source/grafana/grafana.go | 5 +- source/metric/endpoint.go | 21 + 14 files changed, 5840 insertions(+), 94 deletions(-) create mode 100644 pkg/analyze/grafana/parser.go create mode 100644 pkg/analyze/grafana/parser_test.go create mode 100644 pkg/analyze/grafana/tests/d4.json diff --git a/database/database.go b/database/database.go index 5024263..729a210 100644 --- a/database/database.go +++ b/database/database.go @@ -28,23 +28,29 @@ import ( type Database interface { GetMetric(name string) *v1.Metric ListMetrics() map[string]*v1.Metric + ListInvalidMetrics() map[string]*v1.Metric + ListPendingUsage() map[string]*v1.MetricUsage EnqueueMetricList(metrics []string) + EnqueueInvalidMetricsUsage(usages map[string]*v1.MetricUsage) EnqueueUsage(usages map[string]*v1.MetricUsage) EnqueueLabels(labels map[string][]string) } func New(cfg config.Database) Database { d := &db{ - metrics: make(map[string]*v1.Metric), - usage: make(map[string]*v1.MetricUsage), - usageQueue: make(chan map[string]*v1.MetricUsage, 250), - labelsQueue: make(chan map[string][]string, 250), - metricsQueue: make(chan []string, 10), - path: cfg.Path, + metrics: make(map[string]*v1.Metric), + invalidMetrics: make(map[string]*v1.Metric), + usage: make(map[string]*v1.MetricUsage), + usageQueue: make(chan map[string]*v1.MetricUsage, 250), + invalidMetricsUsageQueue: make(chan map[string]*v1.MetricUsage, 250), + labelsQueue: make(chan map[string][]string, 250), + metricsQueue: make(chan []string, 10), + path: cfg.Path, } go d.watchUsageQueue() go d.watchMetricsQueue() + go d.watchInvalidMetricsUsageQueue() go d.watchLabelsQueue() if !*cfg.InMemory { if err := d.readMetricsInJSONFile(); err != nil { @@ -60,6 +66,8 @@ type db struct { // metrics is the list of metric name (as a key) associated to their usage based on the different collector activated. // This struct is our "database". metrics map[string]*v1.Metric + // invalidMetrics is the list of metric name that likely contains a variable or a regexp and as such cannot be a valid metric name. + invalidMetrics map[string]*v1.Metric // usage is a buffer in case the metric name has not yet been collected usage map[string]*v1.MetricUsage // metricsQueue is the channel that should be used to send and receive the list of metric name to keep in memory. @@ -73,6 +81,10 @@ type db struct { // There will be no other way to write in it. // Doing that allows us to accept more HTTP requests to write data and to delay the actual writing. usageQueue chan map[string]*v1.MetricUsage + // invalidMetricsUsageQueue is the way to send the usage per metric that is not valid to write in the database. + // There will be no other way to write in it. + // Doing that allows us to accept more HTTP requests to write data and to delay the actual writing. + invalidMetricsUsageQueue chan map[string]*v1.MetricUsage // path is the path to the JSON file where metrics is flushed periodically // It is empty if the database is purely in memory. path string @@ -83,37 +95,54 @@ type db struct { // 1. Then let's flush the data into a file periodically (or once the queue is empty (if it happens)) // 2. Read the file directly when a read query is coming // Like that we have two different ways to read and write the data. - mutex sync.Mutex + metricsMutex sync.Mutex + invalidMetricsUsageMutex sync.Mutex } func (d *db) GetMetric(name string) *v1.Metric { - d.mutex.Lock() - defer d.mutex.Unlock() + d.metricsMutex.Lock() + defer d.metricsMutex.Unlock() return d.metrics[name] } func (d *db) ListMetrics() map[string]*v1.Metric { - d.mutex.Lock() - defer d.mutex.Unlock() + d.metricsMutex.Lock() + defer d.metricsMutex.Unlock() return d.metrics } +func (d *db) ListInvalidMetrics() map[string]*v1.Metric { + d.invalidMetricsUsageMutex.Lock() + defer d.invalidMetricsUsageMutex.Unlock() + return d.invalidMetrics +} + func (d *db) EnqueueMetricList(metrics []string) { d.metricsQueue <- metrics } +func (d *db) ListPendingUsage() map[string]*v1.MetricUsage { + d.metricsMutex.Lock() + defer d.metricsMutex.Unlock() + return d.usage +} + func (d *db) EnqueueUsage(usages map[string]*v1.MetricUsage) { d.usageQueue <- usages } +func (d *db) EnqueueInvalidMetricsUsage(usages map[string]*v1.MetricUsage) { + d.invalidMetricsUsageQueue <- usages +} + func (d *db) EnqueueLabels(labels map[string][]string) { d.labelsQueue <- labels } func (d *db) watchMetricsQueue() { - for _metrics := range d.metricsQueue { - d.mutex.Lock() - for _, metricName := range _metrics { + for metricsName := range d.metricsQueue { + d.metricsMutex.Lock() + for _, metricName := range metricsName { if _, ok := d.metrics[metricName]; !ok { // As this queue only serves the purpose of storing missing metrics, we are only looking for the one not already present in the database. d.metrics[metricName] = &v1.Metric{} @@ -125,13 +154,29 @@ func (d *db) watchMetricsQueue() { } } } - d.mutex.Unlock() + d.metricsMutex.Unlock() + } +} + +func (d *db) watchInvalidMetricsUsageQueue() { + for data := range d.invalidMetricsUsageQueue { + d.invalidMetricsUsageMutex.Lock() + for metricName, usage := range data { + if _, ok := d.invalidMetrics[metricName]; !ok { + d.invalidMetrics[metricName] = &v1.Metric{ + Usage: usage, + } + } else { + d.invalidMetrics[metricName].Usage = mergeUsage(d.invalidMetrics[metricName].Usage, usage) + } + } + d.invalidMetricsUsageMutex.Unlock() } } func (d *db) watchUsageQueue() { for data := range d.usageQueue { - d.mutex.Lock() + d.metricsMutex.Lock() for metricName, usage := range data { if _, ok := d.metrics[metricName]; !ok { logrus.Debugf("metric_name %q is used but it's not found by the metric collector", metricName) @@ -148,13 +193,13 @@ func (d *db) watchUsageQueue() { d.metrics[metricName].Usage = mergeUsage(d.metrics[metricName].Usage, usage) } } - d.mutex.Unlock() + d.metricsMutex.Unlock() } } func (d *db) watchLabelsQueue() { for data := range d.labelsQueue { - d.mutex.Lock() + d.metricsMutex.Lock() for metricName, labels := range data { if _, ok := d.metrics[metricName]; !ok { // In this case, we should add the metric, because it means the metrics has been found from another source. @@ -165,7 +210,7 @@ func (d *db) watchLabelsQueue() { d.metrics[metricName].Labels = utils.Merge(d.metrics[metricName].Labels, labels) } } - d.mutex.Unlock() + d.metricsMutex.Unlock() } } @@ -180,8 +225,8 @@ func (d *db) flush(period time.Duration) { } func (d *db) writeMetricsInJSONFile() error { - d.mutex.Lock() - defer d.mutex.Unlock() + d.metricsMutex.Lock() + defer d.metricsMutex.Unlock() data, err := json.Marshal(d.metrics) if err != nil { return err diff --git a/go.mod b/go.mod index f8ef29f..7b65bf0 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/perses/perses v0.49.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/common v0.60.1 - github.com/prometheus/prometheus v0.55.1 + github.com/prometheus/prometheus v0.300.0-rc.1 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 golang.org/x/oauth2 v0.24.0 @@ -45,8 +45,6 @@ require ( github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/go-git/go-git/v5 v5.12.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect - github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect @@ -75,7 +73,7 @@ require ( github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.10 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -111,13 +109,13 @@ require ( github.com/zitadel/schema v1.3.0 // indirect gitlab.com/digitalxero/go-conventional-commit v1.0.7 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect - go.opentelemetry.io/otel v1.29.0 // indirect - go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/sdk v1.29.0 // indirect - go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect diff --git a/go.sum b/go.sum index 0c0544c..0e15133 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,10 @@ cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= -cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U= -cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk= +cloud.google.com/go/auth v0.9.5 h1:4CTn43Eynw40aFVr3GpPqsQponx2jv0BQpjvajsbbzw= +cloud.google.com/go/auth v0.9.5/go.mod h1:Xo0n7n66eHyOWWCnitop6870Ilwo3PiZyodVkkH1xWM= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= -cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= -cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= +cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/AlekSi/pointer v1.2.0 h1:glcy/gc4h8HnG2Z3ZECSzZ1IX1x2JxRVuDzaJwQE0+w= @@ -100,10 +100,6 @@ github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZt github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= -github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -187,8 +183,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= +github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -269,8 +265,8 @@ github.com/prometheus/common/sigv4 v0.1.0 h1:qoVebwtwwEhS85Czm2dSROY5fTo2PAPEVdD github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57JrvHu9k5YwTjsNtI= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/prometheus v0.55.1 h1:+NM9V/h4A+wRkOyQzGewzgPPgq/iX2LUQoISNvmjZmI= -github.com/prometheus/prometheus v0.55.1/go.mod h1:GGS7QlWKCqCbcEzWsVahYIfQwiGhcExkarHyLJTsv6I= +github.com/prometheus/prometheus v0.300.0-rc.1 h1:NU7Wu96JnUPhNoNS9bjJOkMY/VgeWfzgvpeUFhiCJnI= +github.com/prometheus/prometheus v0.300.0-rc.1/go.mod h1:gtTPY/XVyCdqqnjA3NzDMb0/nc5H9hOu1RMame+gHyM= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= @@ -322,16 +318,16 @@ go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= -go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -355,8 +351,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -404,13 +400,13 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.195.0 h1:Ude4N8FvTKnnQJHU48RFI40jOBgIrL8Zqr3/QeST6yU= -google.golang.org/api v0.195.0/go.mod h1:DOGRWuv3P8TU8Lnz7uQc4hyNqrBpMtD9ppW3wBJurgc= +google.golang.org/api v0.199.0 h1:aWUXClp+VFJmqE0JPvpZOK3LDQMyFKYIow4etYd9qxs= +google.golang.org/api v0.199.0/go.mod h1:ohG4qSztDJmZdjK/Ar6MhbAmb/Rpi4JHOqagsh90K28= google.golang.org/genproto v0.0.0-20240415180920-8c6c420018be h1:g4aX8SUFA8V5F4LrSY5EclyGYw1OZN4HS1jTyjB9ZDc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/analyze/grafana/grafana.go b/pkg/analyze/grafana/grafana.go index e2ec9eb..c05f574 100644 --- a/pkg/analyze/grafana/grafana.go +++ b/pkg/analyze/grafana/grafana.go @@ -145,43 +145,59 @@ var ( variableReplacer = strings.NewReplacer(generateGrafanaTupleVariableSyntaxReplacer(globalVariableList)...) ) -func Analyze(dashboard *SimplifiedDashboard) ([]string, []*modelAPIV1.LogError) { +func Analyze(dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) { staticVariables := strings.NewReplacer(generateGrafanaVariableSyntaxReplacer(extractStaticVariables(dashboard.Templating.List))...) - m1, err1 := extractMetricsFromPanels(dashboard.Panels, staticVariables, dashboard) + m1, inv1, err1 := extractMetricsFromPanels(dashboard.Panels, staticVariables, dashboard) for _, r := range dashboard.Rows { - m2, err2 := extractMetricsFromPanels(r.Panels, staticVariables, dashboard) + m2, inv2, err2 := extractMetricsFromPanels(r.Panels, staticVariables, dashboard) m1 = utils.Merge(m1, m2) + inv1 = utils.Merge(inv1, inv2) err1 = append(err1, err2...) } - m3, err3 := extractMetricsFromVariables(dashboard.Templating.List, staticVariables, dashboard) - return utils.Merge(m1, m3), append(err1, err3...) + m3, inv3, err3 := extractMetricsFromVariables(dashboard.Templating.List, staticVariables, dashboard) + return utils.Merge(m1, m3), utils.Merge(inv1, inv3), append(err1, err3...) } -func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, dashboard *SimplifiedDashboard) ([]string, []*modelAPIV1.LogError) { +func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError var result []string + var invalidMetricsResult []string for _, p := range panels { for _, t := range extractTarget(p) { if len(t.Expr) == 0 { continue } - metrics, err := prometheus.AnalyzePromQLExpression(replaceVariables(t.Expr, staticVariables)) + exprWithVariableReplaced := replaceVariables(t.Expr, staticVariables) + metrics, invalidMetrics, err := prometheus.AnalyzePromQLExpression(exprWithVariableReplaced) if err != nil { - errs = append(errs, &modelAPIV1.LogError{ - Error: err, - Message: fmt.Sprintf("failed to extract metric names from PromQL expression in the panel %q for the dashboard %s/%s", p.Title, dashboard.Title, dashboard.UID), - }) + otherMetrics := extractMetricNameWithVariable(exprWithVariableReplaced) + if len(otherMetrics) > 0 { + for _, m := range otherMetrics { + if prometheus.IsValidMetricName(m) { + result = utils.InsertIfNotPresent(result, m) + } else { + invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, m) + } + } + } else { + errs = append(errs, &modelAPIV1.LogError{ + Error: err, + Message: fmt.Sprintf("failed to extract metric names from PromQL expression in the panel %q for the dashboard %s/%s", p.Title, dashboard.Title, dashboard.UID), + }) + } } else { result = utils.Merge(result, metrics) + invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics) } } } - return result, errs + return result, invalidMetricsResult, errs } -func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, dashboard *SimplifiedDashboard) ([]string, []*modelAPIV1.LogError) { +func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError var result []string + var invalidMetricsResult []string for _, v := range variables { if v.Type != "query" { continue @@ -211,17 +227,30 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin // query_result(query) query = queryResultRegexp.FindStringSubmatch(query)[1] } - metrics, err := prometheus.AnalyzePromQLExpression(replaceVariables(query, staticVariables)) + exprWithVariableReplaced := replaceVariables(query, staticVariables) + metrics, invalidMetrics, err := prometheus.AnalyzePromQLExpression(replaceVariables(query, staticVariables)) if err != nil { - errs = append(errs, &modelAPIV1.LogError{ - Error: err, - Message: fmt.Sprintf("failed to extract metric names from PromQL expression in variable %q for the dashboard %s/%s", v.Name, dashboard.Title, dashboard.UID), - }) + otherMetrics := extractMetricNameWithVariable(exprWithVariableReplaced) + if len(otherMetrics) > 0 { + for _, m := range otherMetrics { + if prometheus.IsValidMetricName(m) { + result = utils.InsertIfNotPresent(result, m) + } else { + invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, m) + } + } + } else { + errs = append(errs, &modelAPIV1.LogError{ + Error: err, + Message: fmt.Sprintf("failed to extract metric names from PromQL expression in variable %q for the dashboard %s/%s", v.Name, dashboard.Title, dashboard.UID), + }) + } } else { result = utils.Merge(result, metrics) + invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics) } } - return result, errs + return result, invalidMetricsResult, errs } func extractStaticVariables(variables []templateVar) map[string]string { @@ -233,6 +262,10 @@ func extractStaticVariables(variables []templateVar) map[string]string { } if len(v.Options) > 0 { result[v.Name] = v.Options[0].Value + if v.Type == "custom" { + // It seems the variable format ca be used for the "custom" variables. + result[fmt.Sprintf("%s:value", v.Name)] = v.Options[0].Value + } } } return result diff --git a/pkg/analyze/grafana/grafana_test.go b/pkg/analyze/grafana/grafana_test.go index 0aa9cc9..23c663f 100644 --- a/pkg/analyze/grafana/grafana_test.go +++ b/pkg/analyze/grafana/grafana_test.go @@ -33,10 +33,11 @@ func unmarshalDashboard(path string) (*SimplifiedDashboard, error) { func TestAnalyze(t *testing.T) { tests := []struct { - name string - dashboardFile string - resultMetrics []string - resultErrs []*modelAPIV1.LogError + name string + dashboardFile string + resultMetrics []string + invalidMetrics []string + resultErrs []*modelAPIV1.LogError }{ { name: "from/to variables", @@ -53,6 +54,65 @@ func TestAnalyze(t *testing.T) { dashboardFile: "tests/d3.json", resultMetrics: []string{"probe_success"}, }, + { + name: "variable in metrics", + dashboardFile: "tests/d4.json", + resultMetrics: []string{ + "otelcol_process_memory_rss", + "otelcol_rpc_client_request_size_bucket", + "otelcol_rpc_server_request_size_bucket", + "otelcol_rpc_client_duration_bucket", + "otelcol_rpc_server_duration_bucket", + "otelcol_rpc_client_responses_per_rpc_count", + "otelcol_rpc_server_responses_per_rpc_count", + "otelcol_process_runtime_heap_alloc_bytes", + "otelcol_process_runtime_total_sys_memory_bytes", + "otelcol_exporter_queue_size", + "otelcol_exporter_queue_capacity", + "otelcol_processor_batch_batch_send_size_sum", + "otelcol_processor_batch_batch_send_size_count", + "otelcol_processor_batch_batch_send_size_bucket", + }, + invalidMetrics: []string{ + "otelcol_process_uptime.+", + "otelcol_exporter_.+", + "otelcol_processor_.+", + "otelcol_receiver_.+", + "otelcol_receiver_accepted_spans${suffix}", + "otelcol_receiver_refused_spans${suffix}", + "otelcol_receiver_accepted_metric_points${suffix}", + "otelcol_receiver_refused_metric_points${suffix}", + "otelcol_receiver_accepted_log_records${suffix}", + "otelcol_receiver_refused_log_records${suffix}", + "otelcol_processor_accepted_spans${suffix}", + "otelcol_processor_refused_spans${suffix}", + "otelcol_processor_dropped_spans${suffix}", + "otelcol_processor_accepted_metric_points${suffix}", + "otelcol_processor_refused_metric_points${suffix}", + "otelcol_processor_dropped_metric_points${suffix}", + "otelcol_processor_accepted_log_records${suffix}", + "otelcol_processor_refused_log_records${suffix}", + "otelcol_processor_dropped_log_records${suffix}", + "otelcol_processor_batch_batch_size_trigger_send${suffix}", + "otelcol_processor_batch_timeout_trigger_send${suffix}", + "otelcol_exporter_sent_spans${suffix}", + "otelcol_exporter_enqueue_failed_spans${suffix}", + "otelcol_exporter_send_failed_spans${suffix}", + "otelcol_exporter_sent_metric_points${suffix}", + "otelcol_exporter_enqueue_failed_metric_points${suffix}", + "otelcol_exporter_send_failed_metric_points${suffix}", + "otelcol_exporter_sent_log_records${suffix}", + "otelcol_exporter_enqueue_failed_log_records${suffix}", + "otelcol_exporter_send_failed_log_records${suffix}", + "otelcol_process_cpu_seconds${suffix}", + "otelcol_process_uptime${suffix}", + "otelcol_otelsvc_k8s_namespace_added${suffix}", + "otelcol_otelsvc_k8s_namespace_updated${suffix}", + "otelcol_otelsvc_k8s_pod_added${suffix}", + "otelcol_otelsvc_k8s_pod_updated${suffix}", + "otelcol_otelsvc_k8s_pod_deleted${suffix}", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -60,8 +120,9 @@ func TestAnalyze(t *testing.T) { if err != nil { t.Fatal(err) } - metrics, errs := Analyze(dashboard) + metrics, invalidMetrics, errs := Analyze(dashboard) assert.Equal(t, tt.resultMetrics, metrics) + assert.Equal(t, tt.invalidMetrics, invalidMetrics) assert.Equal(t, tt.resultErrs, errs) }) } diff --git a/pkg/analyze/grafana/parser.go b/pkg/analyze/grafana/parser.go new file mode 100644 index 0000000..88e5492 --- /dev/null +++ b/pkg/analyze/grafana/parser.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grafana + +func extractMetricNameWithVariable(expr string) []string { + p := &parser{} + return p.parse(expr) +} + +type parser struct { + metrics []string + currentMetric string +} + +func (p *parser) parse(expr string) []string { + query := []rune(expr) + for i := 0; i < len(query); i++ { + char := query[i] + if isWhitespace(char) { + // Whitespace + continue + } + if isValidMetricChar(char) { + p.currentMetric += string(char) + continue + } + if char == '(' || char == ')' { + // then it was not a metric name and we need to drop it + p.currentMetric = "" + continue + } + if char == '$' { + // That means we are starting to collect a variable hopefully into a metric name. + p.currentMetric += string(char) + if i+1 < len(query) && query[i+1] == '{' { + // if the variable is between bracket, we should loop other it to ensure we don't miss it by stopping collecting the metric + for j := i + 1; query[j] != '}' && j < len(query); j++ { + i = j + if isWhitespace(query[j]) { + continue + } + p.currentMetric += string(query[j]) + } + // here we increment i to match the value of j when it get out of the loop above. + i++ + if i < len(query) { + // At this moment, query[i] == '}' + p.currentMetric += string(query[i]) + } + } + continue + } + if char == '{' { + if len(p.currentMetric) > 0 { + // That means we reached the end of a metric, so we can save it + p.metrics = append(p.metrics, p.currentMetric) + p.currentMetric = "" + } + } + } + return p.metrics +} + +func isWhitespace(ch rune) bool { + return ch == ' ' || ch == '\t' || ch == '\n' +} + +func isValidMetricChar(ch rune) bool { + return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == ':' +} diff --git a/pkg/analyze/grafana/parser_test.go b/pkg/analyze/grafana/parser_test.go new file mode 100644 index 0000000..db78fe7 --- /dev/null +++ b/pkg/analyze/grafana/parser_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grafana + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtractMetricNameWithVariable(t *testing.T) { + tests := []struct { + title string + expr string + result []string + }{ + { + title: "single variable at the end of a metric", + expr: "sum(${metric:value}(otelcol_processor_batch_batch_size_trigger_send${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + result: []string{"otelcol_processor_batch_batch_size_trigger_send${suffix}"}, + }, + { + title: "multiple variable in metric", + expr: "sum(${metric:value}(${foo}otelcol_processor_${bar}batch_batch_size_trigger_send${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + result: []string{"${foo}otelcol_processor_${bar}batch_batch_size_trigger_send${suffix}"}, + }, + { + title: "variable in label", + expr: "rate(tomcat_requestprocessor_received_bytes{$onlyAddsExporter,phase=~\"$phase\",instance=~\"$instance\"}[5m])", + result: []string{"tomcat_requestprocessor_received_bytes"}, + }, + } + for _, test := range tests { + t.Run(test.title, func(t *testing.T) { + result := extractMetricNameWithVariable(test.expr) + assert.Equal(t, test.result, result) + }) + } +} diff --git a/pkg/analyze/grafana/tests/d4.json b/pkg/analyze/grafana/tests/d4.json new file mode 100644 index 0000000..e6f59bc --- /dev/null +++ b/pkg/analyze/grafana/tests/d4.json @@ -0,0 +1,5427 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.3.1" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "panel", + "id": "nodeGraph", + "name": "Node Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Visualize OpenTelemetry (OTEL) collector metrics (tested with OTEL contrib v0.101.0)", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15983, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 23, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Receivers", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Accepted: count/rate of spans successfully pushed into the pipeline.\nRefused: count/rate of spans that could not be pushed into the pipeline.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 28, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_receiver_accepted_spans${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Accepted: {{receiver}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_receiver_refused_spans${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{receiver}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Spans ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Accepted: count/rate of metric points successfully pushed into the pipeline.\nRefused: count/rate of metric points that could not be pushed into the pipeline.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 32, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_receiver_accepted_metric_points${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Accepted: {{receiver}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_receiver_refused_metric_points${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{receiver}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Metric Points ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Accepted: count/rate of log records successfully pushed into the pipeline.\nRefused: count/rate of log records that could not be pushed into the pipeline.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 47, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_receiver_accepted_log_records${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Accepted: {{receiver}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_receiver_refused_log_records${suffix}{receiver=~\"$receiver\",job=\"$job\"}[$__rate_interval])) by (receiver $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{receiver}} {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Log Records ${metric:text}", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 34, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Processors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Accepted: count/rate of spans successfully pushed into the next component in the pipeline.\nRefused: count/rate of spans that were rejected by the next component in the pipeline.\nDropped: count/rate of spans that were dropped", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 35, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_accepted_spans${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Accepted: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_refused_spans${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_dropped_spans${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Dropped: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Spans ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Accepted: count/rate of metric points successfully pushed into the next component in the pipeline.\nRefused: count/rate of metric points that were rejected by the next component in the pipeline.\nDropped: count/rate of metric points that were dropped", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 50, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_accepted_metric_points${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Accepted: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_refused_metric_points${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_dropped_metric_points${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Dropped: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Metric Points ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Accepted: count/rate of log records successfully pushed into the next component in the pipeline.\nRefused: count/rate of log records that were rejected by the next component in the pipeline.\nDropped: count/rate of log records that were dropped", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 51, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_accepted_log_records${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Accepted: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_refused_log_records${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Refused: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_dropped_log_records${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Dropped: {{processor}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Log Records ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of units in the batch", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 49, + "interval": "$minstep", + "links": [], + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 57 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(otelcol_processor_batch_batch_send_size_bucket{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "Batch Send Size Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 36, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_count{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Batch send size count: {{processor}} {{service_instance_id}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_send_size_sum{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Batch send size sum: {{processor}} {{service_instance_id}}", + "refId": "A" + } + ], + "title": "Batch Metrics 1", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of times the batch was sent due to a size trigger. Number of times the batch was sent due to a timeout trigger.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Refused.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 56, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_batch_batch_size_trigger_send${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Batch sent due to a size trigger: {{processor}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_processor_batch_timeout_trigger_send${suffix}{processor=~\"$processor\",job=\"$job\"}[$__rate_interval])) by (processor $grouping)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Batch sent due to a timeout trigger: {{processor}} {{service_instance_id}}", + "refId": "A" + } + ], + "title": "Batch Metrics 2", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 25, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Exporters", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Sent: count/rate of spans successfully sent to destination.\nEnqueue: count/rate of spans failed to be added to the sending queue.\nFailed: count/rate of spans in failed attempts to send to destination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Failed:.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 37, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_sent_spans${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_spans${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_spans${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Spans ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Sent: count/rate of metric points successfully sent to destination.\nEnqueue: count/rate of metric points failed to be added to the sending queue.\nFailed: count/rate of metric points in failed attempts to send to destination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Failed:.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 38, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_sent_metric_points${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_metric_points${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_metric_points${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Metric Points ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Sent: count/rate of log records successfully sent to destination.\nEnqueue: count/rate of log records failed to be added to the sending queue.\nFailed: count/rate of log records in failed attempts to send to destination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/Failed:.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 48, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_sent_log_records${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Sent: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_enqueue_failed_log_records${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Enqueue: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(${metric:value}(otelcol_exporter_send_failed_log_records${suffix}{exporter=~\"$exporter\",job=\"$job\"}[$__rate_interval])) by (exporter $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Failed: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Log Records ${metric:text}", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Current size of the retry queue (in batches)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 10, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(otelcol_exporter_queue_size{exporter=~\"$exporter\",job=\"$job\"}) by (exporter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max queue size: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Exporter Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Fixed capacity of the retry queue (in batches)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 36 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(otelcol_exporter_queue_capacity{exporter=~\"$exporter\",job=\"$job\"}) by (exporter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Queue capacity: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Exporter Queue Capacity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 36 + }, + "id": 67, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(\r\n otelcol_exporter_queue_size{\r\n exporter=~\"$exporter\", job=\"$job\"\r\n }\r\n) by (exporter $grouping)\r\n/\r\nmin(\r\n otelcol_exporter_queue_capacity{\r\n exporter=~\"$exporter\", job=\"$job\"\r\n }\r\n) by (exporter $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Queue capacity usage: {{exporter}} {{service_instance_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Exporter Queue Usage", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 21, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Collector", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total physical memory (resident set size)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg Memory RSS " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min Memory RSS " + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 40, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max Memory RSS {{service_instance_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(otelcol_process_memory_rss{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min Memory RSS {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Total RSS Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total bytes of memory obtained from the OS (see 'go doc runtime.MemStats.Sys')", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg Memory RSS " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min Memory RSS " + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 52, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max Memory RSS {{service_instance_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(otelcol_process_runtime_total_sys_memory_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min Memory RSS {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Total Runtime Sys Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Bytes of allocated heap objects (see 'go doc runtime.MemStats.HeapAlloc')", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg Memory RSS " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg Memory RSS " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min Memory RSS " + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 53, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max Memory RSS {{service_instance_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg Memory RSS {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(otelcol_process_runtime_heap_alloc_bytes{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min Memory RSS {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Total Runtime Heap Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Total CPU user and system time in percentage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max CPU usage " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Avg CPU usage " + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Avg CPU usage " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.fillBelowTo", + "value": "Min CPU usage " + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min CPU usage " + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 55 + }, + "id": 39, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(otelcol_process_cpu_seconds${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Max CPU usage {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(rate(otelcol_process_cpu_seconds${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Avg CPU usage {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "min(rate(otelcol_process_cpu_seconds${suffix}{job=\"$job\"}[$__rate_interval])*100) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Min CPU usage {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of service instances, which are reporting metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 55 + }, + "id": 41, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "count(count(otelcol_process_cpu_seconds${suffix}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id))", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Service instance count", + "range": true, + "refId": "B" + } + ], + "title": "Service Instance Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 55 + }, + "id": 54, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(otelcol_process_uptime${suffix}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Service instance uptime: {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Uptime by Service Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 57, + "interval": "$minstep", + "links": [], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(otelcol_process_uptime${suffix}{service_instance_id=~\".*\",job=\"$job\"}) by (service_instance_id,service_name,service_version)", + "format": "table", + "hide": false, + "instant": true, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "__auto", + "range": false, + "refId": "B" + } + ], + "title": "Service Instance Details", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 59, + "panels": [], + "title": "Signal Flows", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 70 + }, + "id": 58, + "options": { + "nodes": { + "mainStatUnit": "flops" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_receiver_accepted_spans${suffix}{job=\"$job\"}[$__rate_interval])\n ) by (receiver)\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime${suffix}{job=\"$job\"}[$__rate_interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_exporter_sent_spans${suffix}{job=\"$job\"}[$__rate_interval])\n ) by (exporter)\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_spans${suffix}{job=\"$job\"}[$__rate_interval])) by (receiver))\r\n ,\"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n ,\"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_spans${suffix}{job=\"$job\"}[$__rate_interval])) by (exporter))\r\n , \"target\", \"-exp-\", \"transport\", \"exporter\"\r\n )\r\n , \"source\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "edges" + } + ], + "title": "Spans Flow", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "mainstat" + } + }, + { + "disabled": true, + "id": "calculateField", + "options": { + "alias": "secondarystat", + "mode": "reduceRow", + "reduce": { + "include": [ + "mainstat" + ], + "reducer": "sum" + } + } + } + ], + "type": "nodeGraph" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 70 + }, + "id": 60, + "options": { + "nodes": { + "mainStatUnit": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n (sum(\n ${metric:value}(otelcol_receiver_accepted_metric_points${suffix}{job=\"$job\"}[$__rate_interval])\n ) by (receiver))\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime${suffix}{job=\"$job\"}[$__rate_interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n (sum(\n ${metric:value}(otelcol_exporter_sent_metric_points${suffix}{job=\"$job\"}[$__rate_interval])\n ) by (exporter))\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_metric_points${suffix}{job=\"$job\"}[$__rate_interval])) by (receiver))\r\n , \"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n , \"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor \r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_metric_points${suffix}{job=\"$job\"}[$__rate_interval])) by (exporter))\r\n , \"target\", \"-exp-\", \"transport\", \"exporter\"\r\n )\r\n , \"source\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-\", \"source\", \"target\"\r\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "edges" + } + ], + "title": "Metric Points Flow", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "mainstat" + } + }, + { + "disabled": true, + "id": "calculateField", + "options": { + "alias": "secondarystat", + "mode": "reduceRow", + "reduce": { + "include": [ + "Value #nodes" + ], + "reducer": "sum" + } + } + } + ], + "type": "nodeGraph" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Receivers -> Processor(s) -> Exporters (Node Graph panel is beta, so this panel may not show data correctly).", + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 70 + }, + "id": 61, + "options": { + "nodes": { + "mainStatUnit": "flops" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_receiver_accepted_log_records${suffix}{job=\"$job\"}[$__rate_interval])\n ) by (receiver)\n , \"id\", \"-rcv-\", \"transport\", \"receiver\"\n )\n , \"title\", \"\", \"transport\", \"receiver\"\n )\n , \"icon\", \"arrow-to-right\", \"\", \"\"\n)\n\n# dummy processor\nor\nlabel_replace(\n label_replace(\n label_replace(\n (sum(rate(otelcol_process_uptime${suffix}{job=\"$job\"}[$__rate_interval])))\n , \"id\", \"processor\", \"\", \"\"\n )\n , \"title\", \"Processor(s)\", \"\", \"\"\n )\n , \"icon\", \"arrow-random\", \"\", \"\"\n)\n\n# exporters\nor\nlabel_replace(\n label_join(\n label_join(\n sum(${metric:value}(\n otelcol_exporter_sent_log_records${suffix}{job=\"$job\"}[$__rate_interval])\n ) by (exporter)\n , \"id\", \"-exp-\", \"transport\", \"exporter\"\n )\n , \"title\", \"\", \"transport\", \"exporter\"\n )\n , \"icon\", \"arrow-from-right\", \"\", \"\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "nodes" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# receivers -> processor\r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_receiver_accepted_log_records${suffix}{job=\"$job\"}[$__rate_interval])) by (receiver))\r\n , \"source\", \"-rcv-\", \"transport\", \"receiver\"\r\n )\r\n , \"target\", \"processor\", \"\", \"\"\r\n )\r\n , \"id\", \"-edg-\", \"source\", \"target\"\r\n)\r\n\r\n# processor -> exporters\r\nor \r\nlabel_join(\r\n label_replace(\r\n label_join(\r\n (sum(rate(otelcol_exporter_sent_log_records${suffix}{job=\"$job\"}[$__rate_interval])) by (exporter))\r\n ,\"target\",\"-exp-\",\"transport\",\"exporter\"\r\n )\r\n ,\"source\",\"processor\",\"\",\"\"\r\n )\r\n ,\"id\",\"-edg-\",\"source\",\"target\"\r\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "edges" + } + ], + "title": "Log Records Flow", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "Value", + "renamePattern": "mainstat" + } + }, + { + "disabled": true, + "id": "calculateField", + "options": { + "alias": "secondarystat", + "mode": "reduceRow", + "reduce": { + "include": [ + "mainstat" + ], + "reducer": "sum" + } + } + } + ], + "type": "nodeGraph" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 68, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Measures the number of messages received per RPC. Should be 1 for all non-streaming RPCs. GRPC status codes: https://grpc.github.io/grpc/core/md_doc_statuscodes.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "displayName", + "value": "0 - OK" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1" + }, + "properties": [ + { + "id": "displayName", + "value": "1 - CANCELLED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2" + }, + "properties": [ + { + "id": "displayName", + "value": "2 - UNKNOWN" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3" + }, + "properties": [ + { + "id": "displayName", + "value": "3 - INVALID_ARGUMENT" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "displayName", + "value": "4 - DEADLINE_EXCEEDED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "displayName", + "value": "5 - NOT_FOUND" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "displayName", + "value": "6 - ALREADY_EXISTS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "7" + }, + "properties": [ + { + "id": "displayName", + "value": "7 - PERMISSION_DENIED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "displayName", + "value": "8 - RESOURCE_EXHAUSTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "9" + }, + "properties": [ + { + "id": "displayName", + "value": "9 - FAILED_PRECONDITION" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "displayName", + "value": "10 - ABORTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "11" + }, + "properties": [ + { + "id": "displayName", + "value": "11 - OUT_OF_RANGE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "displayName", + "value": "12 - UNIMPLEMENTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "13" + }, + "properties": [ + { + "id": "displayName", + "value": "13 - INTERNAL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "14" + }, + "properties": [ + { + "id": "displayName", + "value": "14 - UNAVAILABLE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "15" + }, + "properties": [ + { + "id": "displayName", + "value": "15 - DATA_LOSS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "16" + }, + "properties": [ + { + "id": "displayName", + "value": "16 - UNAUTHENTICATED" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 69, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(rpc_grpc_status_code) (${metric:value}(otelcol_rpc_server_responses_per_rpc_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC server responses by GRPC status code (receivers)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Measures the number of messages received per RPC. Should be 1 for all non-streaming RPCs. GRPC status codes: https://grpc.github.io/grpc/core/md_doc_statuscodes.html", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "displayName", + "value": "0 - OK" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "1" + }, + "properties": [ + { + "id": "displayName", + "value": "1 - CANCELLED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2" + }, + "properties": [ + { + "id": "displayName", + "value": "2 - UNKNOWN" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3" + }, + "properties": [ + { + "id": "displayName", + "value": "3 - INVALID_ARGUMENT" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "displayName", + "value": "4 - DEADLINE_EXCEEDED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "displayName", + "value": "5 - NOT_FOUND" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "displayName", + "value": "6 - ALREADY_EXISTS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "7" + }, + "properties": [ + { + "id": "displayName", + "value": "7 - PERMISSION_DENIED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "displayName", + "value": "8 - RESOURCE_EXHAUSTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "9" + }, + "properties": [ + { + "id": "displayName", + "value": "9 - FAILED_PRECONDITION" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "displayName", + "value": "10 - ABORTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "11" + }, + "properties": [ + { + "id": "displayName", + "value": "11 - OUT_OF_RANGE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "displayName", + "value": "12 - UNIMPLEMENTED" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "13" + }, + "properties": [ + { + "id": "displayName", + "value": "13 - INTERNAL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "14" + }, + "properties": [ + { + "id": "displayName", + "value": "14 - UNAVAILABLE" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "15" + }, + "properties": [ + { + "id": "displayName", + "value": "15 - DATA_LOSS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "16" + }, + "properties": [ + { + "id": "displayName", + "value": "16 - UNAUTHENTICATED" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 70, + "interval": "$minstep", + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum by(rpc_grpc_status_code) (${metric:value}(otelcol_rpc_client_responses_per_rpc_count{}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "RPC client responses by GRPC status code (exporters)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 72, + "interval": "$minstep", + "links": [], + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "ms" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(otelcol_rpc_server_duration_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC server duration (receivers)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 74, + "interval": "$minstep", + "links": [], + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "ms" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(otelcol_rpc_client_duration_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC client duration (exporters)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 73, + "interval": "$minstep", + "links": [], + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 0.1 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(otelcol_rpc_server_request_size_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC server request size (receivers)", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "links": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 97 + }, + "id": 75, + "interval": "$minstep", + "links": [], + "maxDataPoints": 50, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": true, + "scale": "exponential", + "scheme": "Reds", + "steps": 25 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 0.1 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(otelcol_rpc_client_request_size_bucket{}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + } + ], + "title": "RPC client request size (exporters)", + "type": "heatmap" + } + ], + "title": "RPC server/client", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 80 + }, + "id": 63, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Added: Number of namespace add events received.\nUpdated: Number of namespace update events received.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*updated.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 64, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(otelcol_otelsvc_k8s_namespace_added${suffix}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Added: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg(otelcol_otelsvc_k8s_namespace_updated${suffix}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Updated: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Namespaces", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Added: Number of pod add events received.\nUpdated: Number of pod update events received.\nDeleted: Number of pod delete events received.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 65, + "interval": "$minstep", + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(otelcol_otelsvc_k8s_pod_added${suffix}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Added: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(otelcol_otelsvc_k8s_pod_updated${suffix}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Updated: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(otelcol_otelsvc_k8s_pod_deleted${suffix}{job=\"$job\"}) by (job $grouping)", + "format": "time_series", + "hide": false, + "interval": "$minstep", + "intervalFactor": 1, + "legendFormat": "Deleted: {{transport}} {{service_instance_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Pods", + "type": "timeseries" + } + ], + "title": "Kubernetes", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 66, + "panels": [], + "title": "Documentation", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 45, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": " | \nOTEL collector troubleshooting (how to enable telemetry metrics) | \nScaling the Collector (metrics to watch) | \nCollector internal telemetry | \nOTelBin (OpenTelemetry collector configuration tool) | \nInstalled from Grafana.com dashboards", + "mode": "html" + }, + "pluginVersion": "10.2.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "refId": "A" + } + ], + "title": "Documentation", + "type": "text" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "opentelemetry", + "monitoringartist" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "label_values(otelcol_process_memory_rss,job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(otelcol_process_memory_rss,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "auto": true, + "auto_count": 300, + "auto_min": "10s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_minstep" + }, + "hide": 0, + "label": "Min step", + "name": "minstep", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_minstep" + }, + { + "selected": false, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + } + ], + "query": "10s,30s,1m,5m", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": true, + "text": "Rate", + "value": "rate" + }, + "hide": 0, + "includeAll": false, + "label": "Base metric", + "multi": false, + "name": "metric", + "options": [ + { + "selected": true, + "text": "Rate", + "value": "rate" + }, + { + "selected": false, + "text": "Count", + "value": "increase" + } + ], + "query": "Rate : rate, Count : increase", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result(avg by (receiver) ({__name__=~\"otelcol_receiver_.+\",job=\"$job\"}))", + "hide": 0, + "includeAll": true, + "label": "Receiver", + "multi": false, + "name": "receiver", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(avg by (receiver) ({__name__=~\"otelcol_receiver_.+\",job=\"$job\"}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "/.*receiver=\"(.*)\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result(avg by (processor) ({__name__=~\"otelcol_processor_.+\",job=\"$job\"}))", + "hide": 0, + "includeAll": true, + "label": "Processor", + "multi": false, + "name": "processor", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(avg by (processor) ({__name__=~\"otelcol_processor_.+\",job=\"$job\"}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "/.*processor=\"(.*)\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result(avg by (exporter) ({__name__=~\"otelcol_exporter_.+\",job=\"$job\"}))", + "hide": 0, + "includeAll": true, + "label": "Exporter", + "multi": false, + "name": "exporter", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(avg by (exporter) ({__name__=~\"otelcol_exporter_.+\",job=\"$job\"}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "/.*exporter=\"(.*)\".*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": "None (basic metrics)", + "value": "" + }, + "description": "Detailed metrics must be configured in the collector configuration. They add grouping by transport protocol (http/grpc) for receivers. ", + "hide": 0, + "includeAll": false, + "label": "Additional groupping", + "multi": false, + "name": "grouping", + "options": [ + { + "selected": true, + "text": "None (basic metrics)", + "value": "" + }, + { + "selected": false, + "text": "By transport (detailed metrics)", + "value": ",transport" + }, + { + "selected": false, + "text": "By service instance id", + "value": ",service_instance_id" + } + ], + "query": "None (basic metrics) : , By transport (detailed metrics) : \\,transport, By service instance id : \\,service_instance_id", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "definition": "query_result({__name__=~\"otelcol_process_uptime.+\",job=\"$job\"})", + "description": "Some newer prometheusremotewrite exporter versions/configurations add _total suffix, so this hidden variable detects if _total suffix should be used or not", + "hide": 2, + "includeAll": false, + "label": "Suffix", + "multi": false, + "name": "suffix", + "options": [], + "query": { + "qryType": 3, + "query": "query_result({__name__=~\"otelcol_process_uptime.+\",job=\"$job\"})", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "/otelcol_process_uptime(.*){.*/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 0, + "label": "Ad Hoc", + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "OpenTelemetry Collector", + "uid": "BKf2sowmj", + "version": 82, + "weekStart": "" +} \ No newline at end of file diff --git a/pkg/analyze/perses/perses.go b/pkg/analyze/perses/perses.go index 790431f..55050f8 100644 --- a/pkg/analyze/perses/perses.go +++ b/pkg/analyze/perses/perses.go @@ -66,7 +66,7 @@ func extractMetricUsageFromPanels(panels map[string]*v1.Panel, currentDashboard // No PromQL expression for the query continue } - metrics, err := prometheus.AnalyzePromQLExpression(replaceVariables(spec.Query)) + metrics, _, err := prometheus.AnalyzePromQLExpression(replaceVariables(spec.Query)) if err != nil { errs = append(errs, &modelAPIV1.LogError{ Error: err, @@ -106,7 +106,7 @@ func extractMetricUsageFromVariables(variables []dashboard.Variable, currentDash }) continue } - metrics, err := prometheus.AnalyzePromQLExpression(replaceVariables(spec.Expr)) + metrics, _, err := prometheus.AnalyzePromQLExpression(replaceVariables(spec.Expr)) if err != nil { errs = append(errs, &modelAPIV1.LogError{ Error: err, diff --git a/pkg/analyze/prometheus/prometheus.go b/pkg/analyze/prometheus/prometheus.go index d517e9b..24aa742 100644 --- a/pkg/analyze/prometheus/prometheus.go +++ b/pkg/analyze/prometheus/prometheus.go @@ -15,6 +15,7 @@ package prometheus import ( "fmt" + "regexp" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" "github.com/perses/metrics-usage/utils" @@ -23,6 +24,8 @@ import ( "github.com/prometheus/prometheus/promql/parser" ) +var validMetricName = regexp.MustCompile(`^[a-zA-Z_:][a-zA-Z0-9_:]*$`) + func Analyze(ruleGroups []v1.RuleGroup, source string) (map[string]*modelAPIV1.MetricUsage, []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError metricUsage := make(map[string]*modelAPIV1.MetricUsage) @@ -30,7 +33,7 @@ func Analyze(ruleGroups []v1.RuleGroup, source string) (map[string]*modelAPIV1.M for _, rule := range ruleGroup.Rules { switch v := rule.(type) { case v1.RecordingRule: - metricNames, parserErr := AnalyzePromQLExpression(v.Query) + metricNames, _, parserErr := AnalyzePromQLExpression(v.Query) if parserErr != nil { errs = append(errs, &modelAPIV1.LogError{ Message: fmt.Sprintf("Failed to extract metric name for the ruleGroup %q and the recordingRule %q", ruleGroup.Name, v.Name), @@ -49,7 +52,7 @@ func Analyze(ruleGroups []v1.RuleGroup, source string) (map[string]*modelAPIV1.M false, ) case v1.AlertingRule: - metricNames, parserErr := AnalyzePromQLExpression(v.Query) + metricNames, _, parserErr := AnalyzePromQLExpression(v.Query) if parserErr != nil { errs = append(errs, &modelAPIV1.LogError{ Message: fmt.Sprintf("Failed to extract metric name for the ruleGroup %q and the alertingRule %q", ruleGroup.Name, v.Name), @@ -77,12 +80,15 @@ func Analyze(ruleGroups []v1.RuleGroup, source string) (map[string]*modelAPIV1.M return metricUsage, errs } -func AnalyzePromQLExpression(query string) ([]string, error) { +// AnalyzePromQLExpression is returning a list of valid metric names extracted from the PromQL expression. +// It also returned a list of invalid metric names that likely look like a regexp. +func AnalyzePromQLExpression(query string) ([]string, []string, error) { expr, err := parser.ParseExpr(query) if err != nil { - return nil, err + return nil, nil, err } - var result []string + var metricNames []string + var invalidMetricNames []string parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { if n, ok := node.(*parser.VectorSelector); ok { // The metric name is only present when the node is a VectorSelector. @@ -90,19 +96,28 @@ func AnalyzePromQLExpression(query string) ([]string, error) { // Otherwise, we need to look at the labelName __name__ to find it. // Note: we will need to change this rule with Prometheus 3.0 if n.Name != "" { - result = append(result, n.Name) + metricNames = append(metricNames, n.Name) return nil } for _, m := range n.LabelMatchers { if m.Name == labels.MetricName { - result = append(result, m.Value) + if IsValidMetricName(m.Value) { + metricNames = append(metricNames, m.Value) + } else { + invalidMetricNames = append(invalidMetricNames, m.Value) + } + return nil } } } return nil }) - return result, nil + return metricNames, invalidMetricNames, nil +} + +func IsValidMetricName(name string) bool { + return validMetricName.MatchString(name) } func populateUsage(metricUsage map[string]*modelAPIV1.MetricUsage, metricNames []string, item modelAPIV1.RuleUsage, isAlertingRules bool) { diff --git a/pkg/analyze/prometheus/prometheus_test.go b/pkg/analyze/prometheus/prometheus_test.go index 70ec47d..218da95 100644 --- a/pkg/analyze/prometheus/prometheus_test.go +++ b/pkg/analyze/prometheus/prometheus_test.go @@ -7,7 +7,7 @@ import ( ) func TestAnalyzePromQLExpression(t *testing.T) { - result, err := AnalyzePromQLExpression("service_status{env=~\"$env\",region=~\"$region\"}") + result, _, err := AnalyzePromQLExpression("service_status{env=~\"$env\",region=~\"$region\"}") assert.NoError(t, err) assert.Equal(t, []string{"service_status"}, result) } diff --git a/pkg/client/client.go b/pkg/client/client.go index 7bac66b..3337292 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -62,6 +62,22 @@ func (c *client) Usage(metrics map[string]*modelAPIV1.MetricUsage) error { return nil } +func (c *client) InvalidMetricsUsage(metrics map[string]*modelAPIV1.MetricUsage) error { + data, err := json.Marshal(metrics) + if err != nil { + return err + } + body := bytes.NewBuffer(data) + resp, err := c.httpClient.Post(c.url("/api/v1/invalid_metrics").String(), "application/json", body) + if err != nil { + return err + } + if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusPartialContent { + return fmt.Errorf("when sending metrics usage, unexpected status code: %d", resp.StatusCode) + } + return nil +} + func (c *client) Labels(labels map[string][]string) error { data, err := json.Marshal(labels) if err != nil { diff --git a/source/grafana/grafana.go b/source/grafana/grafana.go index df1e227..92c6df5 100644 --- a/source/grafana/grafana.go +++ b/source/grafana/grafana.go @@ -85,12 +85,14 @@ func (c *grafanaCollector) Execute(ctx context.Context, _ context.CancelFunc) er continue } c.logger.Debugf("extracting metrics for the dashboard %s with UID %q", h.Title, h.UID) - metrics, errs := grafana.Analyze(dashboard) + metrics, invalidMetrics, errs := grafana.Analyze(dashboard) for _, logErr := range errs { logErr.Log(c.logger) } metricUsage := c.generateUsage(metrics, dashboard) + invalidMetricsUsage := c.generateUsage(invalidMetrics, dashboard) c.logger.Infof("%d metrics usage has been collected for the dashboard %q with UID %q", len(metricUsage), h.Title, h.UID) + c.logger.Infof("%d metrics containing regexp or variable has been collected for the dashboard %q with UID %q", len(invalidMetricsUsage), h.Title, h.UID) if len(metricUsage) > 0 { if c.metricUsageClient != nil { // In this case, that means we have to send the data to a remote server. @@ -99,6 +101,7 @@ func (c *grafanaCollector) Execute(ctx context.Context, _ context.CancelFunc) er } } else { c.db.EnqueueUsage(metricUsage) + c.db.EnqueueInvalidMetricsUsage(invalidMetricsUsage) } } } diff --git a/source/metric/endpoint.go b/source/metric/endpoint.go index 4975587..e6ff9a0 100644 --- a/source/metric/endpoint.go +++ b/source/metric/endpoint.go @@ -39,6 +39,10 @@ func (e *endpoint) RegisterRoute(ech *echo.Echo) { ech.POST(path, e.PushMetricsUsage) ech.GET(path, e.ListMetrics) ech.GET(fmt.Sprintf("%s/:id", path), e.GetMetric) + + ech.POST("/api/v1/invalid_metrics", e.PushMetricsUsage) + ech.GET("/api/v1/invalid_metrics", e.ListInvalidMetrics) + ech.GET("/api/v1/pending_usages", e.ListPendingUsages) } func (e *endpoint) GetMetric(ctx echo.Context) error { @@ -91,3 +95,20 @@ func (e *endpoint) PushMetricsUsage(ctx echo.Context) error { e.db.EnqueueUsage(data) return ctx.JSON(http.StatusAccepted, echo.Map{"message": "OK"}) } + +func (e *endpoint) ListInvalidMetrics(ctx echo.Context) error { + return ctx.JSON(http.StatusOK, e.db.ListInvalidMetrics()) +} + +func (e *endpoint) PushInvalidMetricsUsage(ctx echo.Context) error { + data := make(map[string]*v1.MetricUsage) + if err := ctx.Bind(&data); err != nil { + return ctx.JSON(http.StatusBadRequest, echo.Map{"message": err.Error()}) + } + e.db.EnqueueInvalidMetricsUsage(data) + return ctx.JSON(http.StatusAccepted, echo.Map{"message": "OK"}) +} + +func (e *endpoint) ListPendingUsages(ctx echo.Context) error { + return ctx.JSON(http.StatusOK, e.db.ListPendingUsage()) +}