From c6f3147edd3e410362f51863cc32ba5fa126433b Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Mon, 18 Nov 2024 10:25:55 +0100 Subject: [PATCH] use set to optimize usage comparison Signed-off-by: Augustin Husson --- database/database.go | 22 +++--- pkg/analyze/grafana/grafana.go | 55 +++++++------- pkg/analyze/grafana/grafana_test.go | 87 ++++++++++++----------- pkg/analyze/parser/parser.go | 14 ++-- pkg/analyze/parser/parser_test.go | 2 +- pkg/analyze/perses/perses.go | 39 +++++----- pkg/analyze/prometheus/prometheus.go | 27 +++---- pkg/analyze/prometheus/prometheus_test.go | 2 +- pkg/api/v1/metric_usage.go | 78 ++++++++++++++++++-- source/grafana/grafana.go | 9 ++- source/perses/perses.go | 9 ++- utils/array.go | 45 ------------ 12 files changed, 214 insertions(+), 175 deletions(-) delete mode 100644 utils/array.go diff --git a/database/database.go b/database/database.go index 729a210..35099c3 100644 --- a/database/database.go +++ b/database/database.go @@ -21,7 +21,6 @@ import ( "github.com/perses/metrics-usage/config" v1 "github.com/perses/metrics-usage/pkg/api/v1" - "github.com/perses/metrics-usage/utils" "github.com/sirupsen/logrus" ) @@ -145,7 +144,9 @@ func (d *db) watchMetricsQueue() { for _, metricName := range metricsName { if _, ok := d.metrics[metricName]; !ok { // As this queue only serves the purpose of storing missing metrics, we are only looking for the one not already present in the database. - d.metrics[metricName] = &v1.Metric{} + d.metrics[metricName] = &v1.Metric{ + Labels: make(v1.Set[string]), + } // Since it's a new metric, potentially we already have a usage stored in the buffer. if usage, usageExists := d.usage[metricName]; usageExists { // TODO at some point we need to erase the usage map because it will cause a memory leak @@ -204,10 +205,14 @@ func (d *db) watchLabelsQueue() { if _, ok := d.metrics[metricName]; !ok { // In this case, we should add the metric, because it means the metrics has been found from another source. d.metrics[metricName] = &v1.Metric{ - Labels: labels, + Labels: v1.NewSet(labels...), } } else { - d.metrics[metricName].Labels = utils.Merge(d.metrics[metricName].Labels, labels) + if d.metrics[metricName].Labels == nil { + d.metrics[metricName].Labels = v1.NewSet(labels...) + } else { + d.metrics[metricName].Labels.Add(labels...) + } } } d.metricsMutex.Unlock() @@ -249,9 +254,8 @@ func mergeUsage(old, new *v1.MetricUsage) *v1.MetricUsage { if new == nil { return old } - return &v1.MetricUsage{ - Dashboards: utils.Merge(old.Dashboards, new.Dashboards), - RecordingRules: utils.Merge(old.RecordingRules, new.RecordingRules), - AlertRules: utils.Merge(old.AlertRules, new.AlertRules), - } + old.Dashboards.Merge(new.Dashboards) + old.AlertRules.Merge(new.AlertRules) + old.RecordingRules.Merge(new.RecordingRules) + return old } diff --git a/pkg/analyze/grafana/grafana.go b/pkg/analyze/grafana/grafana.go index 2b6b54a..d268e46 100644 --- a/pkg/analyze/grafana/grafana.go +++ b/pkg/analyze/grafana/grafana.go @@ -21,7 +21,6 @@ import ( "github.com/perses/metrics-usage/pkg/analyze/parser" "github.com/perses/metrics-usage/pkg/analyze/prometheus" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" - "github.com/perses/metrics-usage/utils" ) type variableTuple struct { @@ -146,24 +145,26 @@ var ( variableReplacer = strings.NewReplacer(generateGrafanaTupleVariableSyntaxReplacer(globalVariableList)...) ) -func Analyze(dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) { +func Analyze(dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) { staticVariables := strings.NewReplacer(generateGrafanaVariableSyntaxReplacer(extractStaticVariables(dashboard.Templating.List))...) allVariableNames := collectAllVariableName(dashboard.Templating.List) m1, inv1, err1 := extractMetricsFromPanels(dashboard.Panels, staticVariables, allVariableNames, dashboard) for _, r := range dashboard.Rows { m2, inv2, err2 := extractMetricsFromPanels(r.Panels, staticVariables, allVariableNames, dashboard) - m1 = utils.Merge(m1, m2) - inv1 = utils.Merge(inv1, inv2) + m1.Merge(m2) + inv1.Merge(inv2) err1 = append(err1, err2...) } m3, inv3, err3 := extractMetricsFromVariables(dashboard.Templating.List, staticVariables, allVariableNames, dashboard) - return utils.Merge(m1, m3), utils.Merge(inv1, inv3), append(err1, err3...) + m1.Merge(m3) + inv1.Merge(inv3) + return m1, inv1, append(err1, err3...) } -func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, allVariableNames []string, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) { +func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, allVariableNames modelAPIV1.Set[string], dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError - var result []string - var invalidMetricsResult []string + result := modelAPIV1.Set[string]{} + invalidMetricsResult := modelAPIV1.Set[string]{} for _, p := range panels { for _, t := range extractTarget(p) { if len(t.Expr) == 0 { @@ -174,11 +175,11 @@ func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, if err != nil { otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced) if len(otherMetrics) > 0 { - for _, m := range otherMetrics { + for m := range otherMetrics { if prometheus.IsValidMetricName(m) { - result = utils.InsertIfNotPresent(result, m) + result.Add(m) } else { - invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, formatVariableInMetricName(m, allVariableNames)) + invalidMetricsResult.Add(formatVariableInMetricName(m, allVariableNames)) } } } else { @@ -188,18 +189,18 @@ func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, }) } } else { - result = utils.Merge(result, metrics) - invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics) + result.Merge(metrics) + invalidMetricsResult.Merge(invalidMetrics) } } } return result, invalidMetricsResult, errs } -func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, allVariableNames []string, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) { +func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, allVariableNames modelAPIV1.Set[string], dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError - var result []string - var invalidMetricsResult []string + result := modelAPIV1.Set[string]{} + invalidMetricsResult := modelAPIV1.Set[string]{} for _, v := range variables { if v.Type != "query" { continue @@ -234,11 +235,11 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin if err != nil { otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced) if len(otherMetrics) > 0 { - for _, m := range otherMetrics { + for m := range otherMetrics { if prometheus.IsValidMetricName(m) { - result = utils.InsertIfNotPresent(result, m) + result.Add(m) } else { - invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, formatVariableInMetricName(m, allVariableNames)) + invalidMetricsResult.Add(formatVariableInMetricName(m, allVariableNames)) } } } else { @@ -248,8 +249,8 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin }) } } else { - result = utils.Merge(result, metrics) - invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics) + result.Merge(metrics) + invalidMetricsResult.Merge(invalidMetrics) } } return result, invalidMetricsResult, errs @@ -273,10 +274,10 @@ func extractStaticVariables(variables []templateVar) map[string]string { return result } -func collectAllVariableName(variables []templateVar) []string { - result := make([]string, len(variables)) - for i, v := range variables { - result[i] = v.Name +func collectAllVariableName(variables []templateVar) modelAPIV1.Set[string] { + result := modelAPIV1.Set[string]{} + for _, v := range variables { + result.Add(v.Name) } return result } @@ -291,8 +292,8 @@ func replaceVariables(expr string, staticVariables *strings.Replacer) string { // formatVariableInMetricName will replace the syntax of the variable by another one that can actually be parsed. // It will be useful for later when we want to know which metrics, this metric with variable is covered. -func formatVariableInMetricName(metric string, variables []string) string { - for _, v := range variables { +func formatVariableInMetricName(metric string, variables modelAPIV1.Set[string]) string { + for v := range variables { metric = strings.Replace(metric, fmt.Sprintf("$%s", v), fmt.Sprintf("${%s}", v), -1) } return metric diff --git a/pkg/analyze/grafana/grafana_test.go b/pkg/analyze/grafana/grafana_test.go index 23c663f..1359fdc 100644 --- a/pkg/analyze/grafana/grafana_test.go +++ b/pkg/analyze/grafana/grafana_test.go @@ -16,6 +16,7 @@ package grafana import ( "encoding/json" "os" + "slices" "testing" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" @@ -58,59 +59,59 @@ func TestAnalyze(t *testing.T) { name: "variable in metrics", dashboardFile: "tests/d4.json", resultMetrics: []string{ + "otelcol_exporter_queue_capacity", + "otelcol_exporter_queue_size", "otelcol_process_memory_rss", - "otelcol_rpc_client_request_size_bucket", - "otelcol_rpc_server_request_size_bucket", - "otelcol_rpc_client_duration_bucket", - "otelcol_rpc_server_duration_bucket", - "otelcol_rpc_client_responses_per_rpc_count", - "otelcol_rpc_server_responses_per_rpc_count", "otelcol_process_runtime_heap_alloc_bytes", "otelcol_process_runtime_total_sys_memory_bytes", - "otelcol_exporter_queue_size", - "otelcol_exporter_queue_capacity", - "otelcol_processor_batch_batch_send_size_sum", - "otelcol_processor_batch_batch_send_size_count", "otelcol_processor_batch_batch_send_size_bucket", + "otelcol_processor_batch_batch_send_size_count", + "otelcol_processor_batch_batch_send_size_sum", + "otelcol_rpc_client_duration_bucket", + "otelcol_rpc_client_request_size_bucket", + "otelcol_rpc_client_responses_per_rpc_count", + "otelcol_rpc_server_duration_bucket", + "otelcol_rpc_server_request_size_bucket", + "otelcol_rpc_server_responses_per_rpc_count", }, invalidMetrics: []string{ - "otelcol_process_uptime.+", "otelcol_exporter_.+", - "otelcol_processor_.+", - "otelcol_receiver_.+", - "otelcol_receiver_accepted_spans${suffix}", - "otelcol_receiver_refused_spans${suffix}", - "otelcol_receiver_accepted_metric_points${suffix}", - "otelcol_receiver_refused_metric_points${suffix}", - "otelcol_receiver_accepted_log_records${suffix}", - "otelcol_receiver_refused_log_records${suffix}", - "otelcol_processor_accepted_spans${suffix}", - "otelcol_processor_refused_spans${suffix}", - "otelcol_processor_dropped_spans${suffix}", - "otelcol_processor_accepted_metric_points${suffix}", - "otelcol_processor_refused_metric_points${suffix}", - "otelcol_processor_dropped_metric_points${suffix}", - "otelcol_processor_accepted_log_records${suffix}", - "otelcol_processor_refused_log_records${suffix}", - "otelcol_processor_dropped_log_records${suffix}", - "otelcol_processor_batch_batch_size_trigger_send${suffix}", - "otelcol_processor_batch_timeout_trigger_send${suffix}", - "otelcol_exporter_sent_spans${suffix}", - "otelcol_exporter_enqueue_failed_spans${suffix}", - "otelcol_exporter_send_failed_spans${suffix}", - "otelcol_exporter_sent_metric_points${suffix}", + "otelcol_exporter_enqueue_failed_log_records${suffix}", "otelcol_exporter_enqueue_failed_metric_points${suffix}", + "otelcol_exporter_enqueue_failed_spans${suffix}", + "otelcol_exporter_send_failed_log_records${suffix}", "otelcol_exporter_send_failed_metric_points${suffix}", + "otelcol_exporter_send_failed_spans${suffix}", "otelcol_exporter_sent_log_records${suffix}", - "otelcol_exporter_enqueue_failed_log_records${suffix}", - "otelcol_exporter_send_failed_log_records${suffix}", - "otelcol_process_cpu_seconds${suffix}", - "otelcol_process_uptime${suffix}", + "otelcol_exporter_sent_metric_points${suffix}", + "otelcol_exporter_sent_spans${suffix}", "otelcol_otelsvc_k8s_namespace_added${suffix}", "otelcol_otelsvc_k8s_namespace_updated${suffix}", "otelcol_otelsvc_k8s_pod_added${suffix}", - "otelcol_otelsvc_k8s_pod_updated${suffix}", "otelcol_otelsvc_k8s_pod_deleted${suffix}", + "otelcol_otelsvc_k8s_pod_updated${suffix}", + "otelcol_process_cpu_seconds${suffix}", + "otelcol_process_uptime${suffix}", + "otelcol_process_uptime.+", + "otelcol_processor_.+", + "otelcol_processor_accepted_log_records${suffix}", + "otelcol_processor_accepted_metric_points${suffix}", + "otelcol_processor_accepted_spans${suffix}", + "otelcol_processor_batch_batch_size_trigger_send${suffix}", + "otelcol_processor_batch_timeout_trigger_send${suffix}", + "otelcol_processor_dropped_log_records${suffix}", + "otelcol_processor_dropped_metric_points${suffix}", + "otelcol_processor_dropped_spans${suffix}", + "otelcol_processor_refused_log_records${suffix}", + "otelcol_processor_refused_metric_points${suffix}", + "otelcol_processor_refused_spans${suffix}", + "otelcol_receiver_.+", + "otelcol_receiver_accepted_log_records${suffix}", + "otelcol_receiver_accepted_metric_points${suffix}", + "otelcol_receiver_accepted_spans${suffix}", + "otelcol_receiver_refused_log_records${suffix}", + "otelcol_receiver_refused_metric_points${suffix}", + "otelcol_receiver_refused_spans${suffix}", }, }, } @@ -121,8 +122,12 @@ func TestAnalyze(t *testing.T) { t.Fatal(err) } metrics, invalidMetrics, errs := Analyze(dashboard) - assert.Equal(t, tt.resultMetrics, metrics) - assert.Equal(t, tt.invalidMetrics, invalidMetrics) + metricsAsSlice := metrics.TransformAsSlice() + invalidMetricsAsSlice := invalidMetrics.TransformAsSlice() + slices.Sort(metricsAsSlice) + slices.Sort(invalidMetricsAsSlice) + assert.Equal(t, tt.resultMetrics, metricsAsSlice) + assert.Equal(t, tt.invalidMetrics, invalidMetricsAsSlice) assert.Equal(t, tt.resultErrs, errs) }) } diff --git a/pkg/analyze/parser/parser.go b/pkg/analyze/parser/parser.go index eef2147..c5a7284 100644 --- a/pkg/analyze/parser/parser.go +++ b/pkg/analyze/parser/parser.go @@ -13,17 +13,21 @@ package parser -func ExtractMetricNameWithVariable(expr string) []string { - p := &parser{} +import modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" + +func ExtractMetricNameWithVariable(expr string) modelAPIV1.Set[string] { + p := &parser{ + metrics: modelAPIV1.Set[string]{}, + } return p.parse(expr) } type parser struct { - metrics []string + metrics modelAPIV1.Set[string] currentMetric string } -func (p *parser) parse(expr string) []string { +func (p *parser) parse(expr string) modelAPIV1.Set[string] { query := []rune(expr) for i := 0; i < len(query); i++ { char := query[i] @@ -64,7 +68,7 @@ func (p *parser) parse(expr string) []string { if char == '{' { if len(p.currentMetric) > 0 { // That means we reached the end of a metric, so we can save it - p.metrics = append(p.metrics, p.currentMetric) + p.metrics.Add(p.currentMetric) p.currentMetric = "" } } diff --git a/pkg/analyze/parser/parser_test.go b/pkg/analyze/parser/parser_test.go index 5e1f982..c69c9c2 100644 --- a/pkg/analyze/parser/parser_test.go +++ b/pkg/analyze/parser/parser_test.go @@ -44,7 +44,7 @@ func TestExtractMetricNameWithVariable(t *testing.T) { for _, test := range tests { t.Run(test.title, func(t *testing.T) { result := ExtractMetricNameWithVariable(test.expr) - assert.Equal(t, test.result, result) + assert.Equal(t, test.result, result.TransformAsSlice()) }) } } diff --git a/pkg/analyze/perses/perses.go b/pkg/analyze/perses/perses.go index 2eed99a..04f3753 100644 --- a/pkg/analyze/perses/perses.go +++ b/pkg/analyze/perses/perses.go @@ -21,7 +21,6 @@ import ( "github.com/perses/metrics-usage/pkg/analyze/parser" "github.com/perses/metrics-usage/pkg/analyze/prometheus" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" - "github.com/perses/metrics-usage/utils" "github.com/perses/perses/go-sdk/prometheus/query" "github.com/perses/perses/go-sdk/prometheus/variable/promql" v1 "github.com/perses/perses/pkg/model/api/v1" @@ -41,16 +40,18 @@ var variableReplacer = strings.NewReplacer( "$__project", "perses", ) -func Analyze(dashboard *v1.Dashboard) ([]string, []string, []*modelAPIV1.LogError) { +func Analyze(dashboard *v1.Dashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) { m1, inv1, err1 := extractMetricUsageFromVariables(dashboard.Spec.Variables, dashboard) m2, inv2, err2 := extractMetricUsageFromPanels(dashboard.Spec.Panels, dashboard) - return utils.Merge(m1, m2), utils.Merge(inv1, inv2), append(err1, err2...) + m1.Merge(m2) + inv1.Merge(inv2) + return m1, inv1, append(err1, err2...) } -func extractMetricUsageFromPanels(panels map[string]*v1.Panel, currentDashboard *v1.Dashboard) ([]string, []string, []*modelAPIV1.LogError) { +func extractMetricUsageFromPanels(panels map[string]*v1.Panel, currentDashboard *v1.Dashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError - var result []string - var invalidMetricsResult []string + result := modelAPIV1.Set[string]{} + invalidMetricsResult := modelAPIV1.Set[string]{} for panelName, panel := range panels { for i, q := range panel.Spec.Queries { if q.Spec.Plugin.Kind != query.PluginKind { @@ -73,11 +74,11 @@ func extractMetricUsageFromPanels(panels map[string]*v1.Panel, currentDashboard if err != nil { otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced) if len(otherMetrics) > 0 { - for _, m := range otherMetrics { + for m := range otherMetrics { if prometheus.IsValidMetricName(m) { - result = utils.InsertIfNotPresent(result, m) + result.Add(m) } else { - invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, m) + invalidMetricsResult.Add(m) } } } else { @@ -88,17 +89,17 @@ func extractMetricUsageFromPanels(panels map[string]*v1.Panel, currentDashboard continue } } - result = utils.Merge(result, metrics) - invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics) + result.Merge(metrics) + invalidMetricsResult.Merge(invalidMetrics) } } return result, invalidMetricsResult, errs } -func extractMetricUsageFromVariables(variables []dashboard.Variable, currentDashboard *v1.Dashboard) ([]string, []string, []*modelAPIV1.LogError) { +func extractMetricUsageFromVariables(variables []dashboard.Variable, currentDashboard *v1.Dashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) { var errs []*modelAPIV1.LogError - var result []string - var invalidMetricsResult []string + result := modelAPIV1.Set[string]{} + invalidMetricsResult := modelAPIV1.Set[string]{} for _, v := range variables { if v.Kind != variable.KindList { continue @@ -127,11 +128,11 @@ func extractMetricUsageFromVariables(variables []dashboard.Variable, currentDash if err != nil { otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced) if len(otherMetrics) > 0 { - for _, m := range otherMetrics { + for m := range otherMetrics { if prometheus.IsValidMetricName(m) { - result = utils.InsertIfNotPresent(result, m) + result.Add(m) } else { - invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, m) + invalidMetricsResult.Add(m) } } } else { @@ -142,8 +143,8 @@ func extractMetricUsageFromVariables(variables []dashboard.Variable, currentDash continue } } - result = utils.Merge(result, metrics) - invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics) + result.Merge(metrics) + invalidMetricsResult.Merge(invalidMetrics) } return result, invalidMetricsResult, errs } diff --git a/pkg/analyze/prometheus/prometheus.go b/pkg/analyze/prometheus/prometheus.go index f02c54a..2047342 100644 --- a/pkg/analyze/prometheus/prometheus.go +++ b/pkg/analyze/prometheus/prometheus.go @@ -18,7 +18,6 @@ import ( "regexp" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" - "github.com/perses/metrics-usage/utils" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" @@ -103,13 +102,13 @@ func Analyze(ruleGroups []v1.RuleGroup, source string) (map[string]*modelAPIV1.M // AnalyzePromQLExpression is returning a list of valid metric names extracted from the PromQL expression. // It also returned a list of invalid metric names that likely look like a regexp. -func AnalyzePromQLExpression(query string) ([]string, []string, error) { +func AnalyzePromQLExpression(query string) (modelAPIV1.Set[string], modelAPIV1.Set[string], error) { expr, err := parser.ParseExpr(query) if err != nil { return nil, nil, err } - var metricNames []string - var invalidMetricNames []string + metricNames := modelAPIV1.Set[string]{} + invalidMetricNames := modelAPIV1.Set[string]{} parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error { if n, ok := node.(*parser.VectorSelector); ok { // The metric name is only present when the node is a VectorSelector. @@ -117,15 +116,15 @@ func AnalyzePromQLExpression(query string) ([]string, []string, error) { // Otherwise, we need to look at the labelName __name__ to find it. // Note: we will need to change this rule with Prometheus 3.0 if n.Name != "" { - metricNames = append(metricNames, n.Name) + metricNames.Add(n.Name) return nil } for _, m := range n.LabelMatchers { if m.Name == labels.MetricName { if IsValidMetricName(m.Value) { - metricNames = append(metricNames, m.Value) + metricNames.Add(m.Value) } else { - invalidMetricNames = append(invalidMetricNames, m.Value) + invalidMetricNames.Add(m.Value) } return nil @@ -141,20 +140,22 @@ func IsValidMetricName(name string) bool { return validMetricName.MatchString(name) } -func populateUsage(metricUsage map[string]*modelAPIV1.MetricUsage, metricNames []string, item modelAPIV1.RuleUsage, isAlertingRules bool) { - for _, metricName := range metricNames { +func populateUsage(metricUsage map[string]*modelAPIV1.MetricUsage, metricNames modelAPIV1.Set[string], item modelAPIV1.RuleUsage, isAlertingRules bool) { + for metricName := range metricNames { if usage, ok := metricUsage[metricName]; ok { if isAlertingRules { - usage.AlertRules = utils.InsertIfNotPresent(usage.AlertRules, item) + usage.AlertRules.Add(item) } else { - usage.RecordingRules = utils.InsertIfNotPresent(usage.RecordingRules, item) + usage.RecordingRules.Add(item) } } else { u := &modelAPIV1.MetricUsage{} if isAlertingRules { - u.AlertRules = []modelAPIV1.RuleUsage{item} + u.AlertRules = modelAPIV1.NewSet(item) + u.RecordingRules = modelAPIV1.NewSet[modelAPIV1.RuleUsage]() } else { - u.RecordingRules = []modelAPIV1.RuleUsage{item} + u.RecordingRules = modelAPIV1.NewSet(item) + u.AlertRules = modelAPIV1.NewSet[modelAPIV1.RuleUsage]() } metricUsage[metricName] = u } diff --git a/pkg/analyze/prometheus/prometheus_test.go b/pkg/analyze/prometheus/prometheus_test.go index 218da95..021d2f3 100644 --- a/pkg/analyze/prometheus/prometheus_test.go +++ b/pkg/analyze/prometheus/prometheus_test.go @@ -9,5 +9,5 @@ import ( func TestAnalyzePromQLExpression(t *testing.T) { result, _, err := AnalyzePromQLExpression("service_status{env=~\"$env\",region=~\"$region\"}") assert.NoError(t, err) - assert.Equal(t, []string{"service_status"}, result) + assert.Equal(t, []string{"service_status"}, result.TransformAsSlice()) } diff --git a/pkg/api/v1/metric_usage.go b/pkg/api/v1/metric_usage.go index 08b69de..81b9124 100644 --- a/pkg/api/v1/metric_usage.go +++ b/pkg/api/v1/metric_usage.go @@ -13,6 +13,76 @@ package v1 +import "encoding/json" + +type Set[T comparable] map[T]struct{} + +func NewSet[T comparable](vals ...T) Set[T] { + s := Set[T]{} + for _, v := range vals { + s[v] = struct{}{} + } + return s +} + +func (s Set[T]) Add(vals ...T) { + for _, v := range vals { + s[v] = struct{}{} + } +} + +func (s Set[T]) Remove(value T) { + delete(s, value) +} + +func (s Set[T]) Contains(value T) bool { + _, ok := s[value] + return ok +} + +func (s Set[T]) Merge(other Set[T]) { + for v := range other { + s.Add(v) + } +} + +func (s Set[T]) TransformAsSlice() []T { + if s == nil { + return nil + } + var slice []T + for v := range s { + slice = append(slice, v) + } + return slice +} + +func (s Set[T]) MarshalJSON() ([]byte, error) { + if s == nil { + return []byte("[]"), nil + } + var slice []T + for v := range s { + slice = append(slice, v) + } + return json.Marshal(slice) +} + +func (s *Set[T]) UnmarshalJSON(b []byte) error { + var slice []T + if err := json.Unmarshal(b, &slice); err != nil { + return err + } + if len(slice) == 0 { + return nil + } + *s = make(map[T]struct{}, len(slice)) + for _, v := range slice { + s.Add(v) + } + return nil +} + type RuleUsage struct { PromLink string `json:"prom_link"` GroupName string `json:"group_name"` @@ -21,12 +91,12 @@ type RuleUsage struct { } type MetricUsage struct { - Dashboards []string `json:"dashboards,omitempty"` - RecordingRules []RuleUsage `json:"recordingRules,omitempty"` - AlertRules []RuleUsage `json:"alertRules,omitempty"` + Dashboards Set[string] `json:"dashboards,omitempty"` + RecordingRules Set[RuleUsage] `json:"recordingRules,omitempty"` + AlertRules Set[RuleUsage] `json:"alertRules,omitempty"` } type Metric struct { - Labels []string `json:"labels,omitempty"` + Labels Set[string] `json:"labels,omitempty"` Usage *MetricUsage `json:"usage,omitempty"` } diff --git a/source/grafana/grafana.go b/source/grafana/grafana.go index f7d4c6e..0b08714 100644 --- a/source/grafana/grafana.go +++ b/source/grafana/grafana.go @@ -29,7 +29,6 @@ import ( modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" "github.com/perses/metrics-usage/pkg/client" "github.com/perses/metrics-usage/usageclient" - "github.com/perses/metrics-usage/utils" "github.com/sirupsen/logrus" ) @@ -140,15 +139,15 @@ func (c *grafanaCollector) collectAllDashboardUID(ctx context.Context) ([]*grafa return result, nil } -func (c *grafanaCollector) generateUsage(metricNames []string, currentDashboard *grafana.SimplifiedDashboard) map[string]*modelAPIV1.MetricUsage { +func (c *grafanaCollector) generateUsage(metricNames modelAPIV1.Set[string], currentDashboard *grafana.SimplifiedDashboard) map[string]*modelAPIV1.MetricUsage { metricUsage := make(map[string]*modelAPIV1.MetricUsage) dashboardURL := fmt.Sprintf("%s/d/%s", c.grafanaURL, currentDashboard.UID) - for _, metricName := range metricNames { + for metricName := range metricNames { if usage, ok := metricUsage[metricName]; ok { - usage.Dashboards = utils.InsertIfNotPresent(usage.Dashboards, dashboardURL) + usage.Dashboards.Add(dashboardURL) } else { metricUsage[metricName] = &modelAPIV1.MetricUsage{ - Dashboards: []string{dashboardURL}, + Dashboards: modelAPIV1.NewSet(dashboardURL), } } } diff --git a/source/perses/perses.go b/source/perses/perses.go index 397fbdf..bdc4c07 100644 --- a/source/perses/perses.go +++ b/source/perses/perses.go @@ -24,7 +24,6 @@ import ( modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" "github.com/perses/metrics-usage/pkg/client" "github.com/perses/metrics-usage/usageclient" - "github.com/perses/metrics-usage/utils" persesClientV1 "github.com/perses/perses/pkg/client/api/v1" persesClientConfig "github.com/perses/perses/pkg/client/config" v1 "github.com/perses/perses/pkg/model/api/v1" @@ -86,15 +85,15 @@ func (c *persesCollector) Execute(_ context.Context, _ context.CancelFunc) error return nil } -func (c *persesCollector) generateUsage(metricNames []string, currentDashboard *v1.Dashboard) map[string]*modelAPIV1.MetricUsage { +func (c *persesCollector) generateUsage(metricNames modelAPIV1.Set[string], currentDashboard *v1.Dashboard) map[string]*modelAPIV1.MetricUsage { metricUsage := make(map[string]*modelAPIV1.MetricUsage) dashboardURL := fmt.Sprintf("%s/api/v1/projects/%s/dashboards/%s", c.persesURL, currentDashboard.Metadata.Project, currentDashboard.Metadata.Name) - for _, metricName := range metricNames { + for metricName := range metricNames { if usage, ok := metricUsage[metricName]; ok { - usage.Dashboards = utils.InsertIfNotPresent(usage.Dashboards, dashboardURL) + usage.Dashboards.Add(dashboardURL) } else { metricUsage[metricName] = &modelAPIV1.MetricUsage{ - Dashboards: []string{dashboardURL}, + Dashboards: modelAPIV1.NewSet(dashboardURL), } } } diff --git a/utils/array.go b/utils/array.go deleted file mode 100644 index 392f11e..0000000 --- a/utils/array.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Perses Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -func InsertIfNotPresent[T comparable](slice []T, item T) []T { - for _, s := range slice { - if item == s { - return slice - } - } - return append(slice, item) -} - -func Merge[T comparable](old, new []T) []T { - if len(old) == 0 { - return new - } - if len(new) == 0 { - return old - } - for _, a := range old { - found := false - for _, b := range new { - if a == b { - found = true - break - } - } - if !found { - new = append(new, a) - } - } - return new -}