-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Metrics to the sync manager (#244)
* Add metrics to sync manager This PR adds metrics to the sync manager. * Generate licenses, PR comments --------- Co-authored-by: Evan Foster <[email protected]>
- Loading branch information
1 parent
10d8858
commit e0348b1
Showing
5 changed files
with
292 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| /* | ||
| Copyright 2024 Adobe. All rights reserved. | ||
| This file is licensed to you under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. You may obtain a copy | ||
| of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software distributed under | ||
| the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS | ||
| OF ANY KIND, either express or implied. See the License for the specific language | ||
| governing permissions and limitations under the License. | ||
| */ | ||
|
|
||
| package monitoring | ||
|
|
||
| import "github.com/prometheus/client_golang/prometheus" | ||
| import "github.com/prometheus/client_golang/prometheus/promauto" | ||
|
|
||
| type MetricsI interface { | ||
| RecordRequeueCnt(target string) | ||
| RecordReconciliationCnt(target string) | ||
| RecordReconciliationDur(target string, elapsed float64) | ||
| RecordEnqueueCnt(target string) | ||
| RecordEnqueueDur(target string, elapsed float64) | ||
| RecordErrorCnt(target string) | ||
| } | ||
|
|
||
| type Metrics struct { | ||
| RequeueCnt *prometheus.CounterVec | ||
| ReconciliationCnt *prometheus.CounterVec | ||
| ReconciliationDur *prometheus.HistogramVec | ||
| EnqueueCnt *prometheus.CounterVec | ||
| EnqueueDur *prometheus.HistogramVec | ||
| ErrCnt *prometheus.CounterVec | ||
| metrics []prometheus.Collector | ||
| } | ||
|
|
||
| func NewMetrics() *Metrics { | ||
| return &Metrics{} | ||
| } | ||
|
|
||
| func (m *Metrics) Init(isUnitTest bool) { | ||
| reg := prometheus.DefaultRegisterer | ||
| if isUnitTest { | ||
| reg = prometheus.NewRegistry() | ||
| } | ||
| var requeueCnt prometheus.Collector = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ | ||
| Name: "cluster_registry_sync_manager_requeues_total", | ||
| Help: "The total number of controller-manager requeues partitioned by target.", | ||
| }, []string{"target"}) | ||
| m.RequeueCnt = requeueCnt.(*prometheus.CounterVec) | ||
| m.metrics = append(m.metrics, m.RequeueCnt) | ||
|
|
||
| var reconciliationCnt prometheus.Collector = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ | ||
| Name: "cluster_registry_sync_manager_reconciliation_total", | ||
| Help: "How many reconciliations occurred, partitioned by target.", | ||
| }, | ||
| []string{"target"}, | ||
| ) | ||
| m.ReconciliationCnt = reconciliationCnt.(*prometheus.CounterVec) | ||
| m.metrics = append(m.metrics, m.ReconciliationCnt) | ||
|
|
||
| var reconciliationDur prometheus.Collector = promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ | ||
| Name: "cluster_registry_sync_manager_reconciliation_duration_seconds", | ||
| Help: "The time taken to reconcile resources in seconds partitioned by target.", | ||
| }, | ||
| []string{"target"}, | ||
| ) | ||
| m.ReconciliationDur = reconciliationDur.(*prometheus.HistogramVec) | ||
| m.metrics = append(m.metrics, m.ReconciliationDur) | ||
|
|
||
| var enqueueCnt prometheus.Collector = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ | ||
| Name: "cluster_registry_sync_manager_enqueue_total", | ||
| Help: "How many reconciliations were enqueued, partitioned by target.", | ||
| }, | ||
| []string{"target"}, | ||
| ) | ||
| m.EnqueueCnt = enqueueCnt.(*prometheus.CounterVec) | ||
| m.metrics = append(m.metrics, m.EnqueueCnt) | ||
|
|
||
| var enqueueDur prometheus.Collector = promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ | ||
| Name: "cluster_registry_sync_manager_enqueue_duration_seconds", | ||
| Help: "The time taken to enqueue a reconciliation in seconds partitioned by target.", | ||
| }, | ||
| []string{"target"}, | ||
| ) | ||
| m.EnqueueDur = enqueueDur.(*prometheus.HistogramVec) | ||
| m.metrics = append(m.metrics, m.EnqueueDur) | ||
|
|
||
| var errorCnt prometheus.Collector = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ | ||
| Name: "cluster_registry_sync_manager_error_total", | ||
| Help: "The total number controller-manager errors partitioned by target.", | ||
| }, []string{"target"}) | ||
| m.ErrCnt = errorCnt.(*prometheus.CounterVec) | ||
| m.metrics = append(m.metrics, m.ErrCnt) | ||
| } | ||
|
|
||
| func (m *Metrics) RecordRequeueCnt(target string) { | ||
| m.RequeueCnt.WithLabelValues(target).Inc() | ||
| } | ||
|
|
||
| func (m *Metrics) RecordReconciliationCnt(target string) { | ||
| m.ReconciliationCnt.WithLabelValues(target).Inc() | ||
| } | ||
|
|
||
| func (m *Metrics) RecordReconciliationDur(target string, elapsed float64) { | ||
| m.ReconciliationDur.WithLabelValues(target).Observe(elapsed) | ||
| } | ||
|
|
||
| func (m *Metrics) RecordEnqueueCnt(target string) { | ||
| m.EnqueueCnt.WithLabelValues(target).Inc() | ||
| } | ||
|
|
||
| func (m *Metrics) RecordEnqueueDur(target string, elapsed float64) { | ||
| m.EnqueueDur.WithLabelValues(target).Observe(elapsed) | ||
| } | ||
|
|
||
| func (m *Metrics) RecordErrorCnt(target string) { | ||
| m.ErrCnt.WithLabelValues(target).Inc() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,145 @@ | ||
| /* | ||
| Copyright 2024 Adobe. All rights reserved. | ||
| This file is licensed to you under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. You may obtain a copy | ||
| of the License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software distributed under | ||
| the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS | ||
| OF ANY KIND, either express or implied. See the License for the specific language | ||
| governing permissions and limitations under the License. | ||
| */ | ||
|
|
||
| package monitoring | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/prometheus/client_golang/prometheus/testutil" | ||
| "github.com/stretchr/testify/assert" | ||
| "math/rand" | ||
| "strings" | ||
| "testing" | ||
| ) | ||
|
|
||
| const ( | ||
| clusterSyncTarget = "orgnumber-env-region-cluster-sync" | ||
| subsystem = "cluster_registry_sync_manager" | ||
| minRand = 1 | ||
| maxRand = 2.5 | ||
| ) | ||
|
|
||
| // Generate a random float number between min and max | ||
| func generateFloatRand(min, max float64) float64 { | ||
| return min + rand.Float64()*(max-min) | ||
| } | ||
|
|
||
| // Generate what we expect a histogram of some random number to look like. metricTopic is what the metric is about, e.g. | ||
| // reconciliation or enqueue. helpString is the literal help string from metrics.go. I'd grab this myself, but it's not | ||
| // exposed in the HistogramVec object AFAICT :( | ||
| func generateExpectedHistogram(randomFloat float64, metricTopic string, helpString string) string { | ||
| expected := fmt.Sprintf(` | ||
| # HELP %[1]s_%[5]s_duration_seconds %[4]s | ||
| # TYPE %[1]s_%[5]s_duration_seconds histogram | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.005"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.01"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.025"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.05"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.1"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.25"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="0.5"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="1"} 0 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="2.5"} 1 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="5"} 1 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="10"} 1 | ||
| %[1]s_%[5]s_duration_seconds_bucket{target="%[2]s",le="+Inf"} 1 | ||
| %[1]s_%[5]s_duration_seconds_sum{target="%[2]s"} %[3]s | ||
| %[1]s_%[5]s_duration_seconds_count{target="%[2]s"} 1 | ||
| `, subsystem, clusterSyncTarget, fmt.Sprintf("%.16f", randomFloat), helpString, metricTopic) | ||
| return expected | ||
| } | ||
|
|
||
| func TestNewMetrics(t *testing.T) { | ||
| test := assert.New(t) | ||
| m := NewMetrics() | ||
| test.NotNil(m) | ||
| } | ||
|
|
||
| func TestInit(t *testing.T) { | ||
| test := assert.New(t) | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| test.NotNil(m.RequeueCnt) | ||
| test.NotNil(m.ReconciliationCnt) | ||
| test.NotNil(m.ReconciliationDur) | ||
| test.NotNil(m.EnqueueCnt) | ||
| test.NotNil(m.EnqueueDur) | ||
| test.NotNil(m.ErrCnt) | ||
| } | ||
|
|
||
| func TestRecordRequeueCnt(t *testing.T) { | ||
| test := assert.New(t) | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| m.RecordRequeueCnt(clusterSyncTarget) | ||
| test.Equal(1, testutil.CollectAndCount(*m.RequeueCnt)) | ||
| test.Equal(float64(1), testutil.ToFloat64((*m.RequeueCnt).WithLabelValues(clusterSyncTarget))) | ||
| } | ||
|
|
||
| func TestRecordReconciliationCnt(t *testing.T) { | ||
| test := assert.New(t) | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| m.RecordReconciliationCnt(clusterSyncTarget) | ||
| test.Equal(1, testutil.CollectAndCount(*m.ReconciliationCnt)) | ||
| test.Equal(float64(1), testutil.ToFloat64((*m.ReconciliationCnt).WithLabelValues(clusterSyncTarget))) | ||
| } | ||
|
|
||
| func TestRecordReconciliationDur(t *testing.T) { | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| randomFloat := generateFloatRand(minRand, maxRand) | ||
| m.RecordReconciliationDur(clusterSyncTarget, randomFloat) | ||
| expected := generateExpectedHistogram(randomFloat, "reconciliation", "The time taken to reconcile resources in seconds partitioned by target.") | ||
| if err := testutil.CollectAndCompare( | ||
| *m.ReconciliationDur, | ||
| strings.NewReader(expected), | ||
| fmt.Sprintf("%s_%s_duration_seconds", subsystem, "reconciliation")); err != nil { | ||
| t.Errorf("unexpected collecting result:\n%s", err) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| func TestRecordEnqueueCnt(t *testing.T) { | ||
| test := assert.New(t) | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| m.RecordEnqueueCnt(clusterSyncTarget) | ||
| test.Equal(1, testutil.CollectAndCount(*m.EnqueueCnt)) | ||
| test.Equal(float64(1), testutil.ToFloat64((*m.EnqueueCnt).WithLabelValues(clusterSyncTarget))) | ||
|
|
||
| } | ||
|
|
||
| func TestRecordEnqueueDur(t *testing.T) { | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| randomFloat := generateFloatRand(minRand, maxRand) | ||
| m.RecordEnqueueDur(clusterSyncTarget, randomFloat) | ||
| expected := generateExpectedHistogram(randomFloat, "enqueue", "The time taken to enqueue a reconciliation in seconds partitioned by target.") | ||
| if err := testutil.CollectAndCompare( | ||
| *m.EnqueueDur, | ||
| strings.NewReader(expected), | ||
| fmt.Sprintf("%s_%s_duration_seconds", subsystem, "enqueue")); err != nil { | ||
| t.Errorf("unexpected collecting result:\n%s", err) | ||
| } | ||
|
|
||
| } | ||
|
|
||
| func TestRecordErrCnt(t *testing.T) { | ||
| test := assert.New(t) | ||
| m := NewMetrics() | ||
| m.Init(true) | ||
| m.RecordErrorCnt(clusterSyncTarget) | ||
| test.Equal(1, testutil.CollectAndCount(*m.ErrCnt)) | ||
| test.Equal(float64(1), testutil.ToFloat64((*m.ErrCnt).WithLabelValues(clusterSyncTarget))) | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters