Skip to content

Commit

Permalink
use set to optimize usage comparison
Browse files Browse the repository at this point in the history 
Signed-off-by: Augustin Husson <[email protected]>
  • Loading branch information
@Nexucis
Nexucis committed Nov 18, 2024
1 parent c0a0c95 commit c6f3147
Show file tree
Hide file tree
Showing 12 changed files with 214 additions and 175 deletions.
22 changes: 13 additions & 9 deletions database/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (

"github.com/perses/metrics-usage/config"
v1 "github.com/perses/metrics-usage/pkg/api/v1"
"github.com/perses/metrics-usage/utils"
"github.com/sirupsen/logrus"
)

Expand Down Expand Up @@ -145,7 +144,9 @@ func (d *db) watchMetricsQueue() {
for _, metricName := range metricsName {
if _, ok := d.metrics[metricName]; !ok {
// As this queue only serves the purpose of storing missing metrics, we are only looking for the one not already present in the database.
d.metrics[metricName] = &v1.Metric{}
d.metrics[metricName] = &v1.Metric{
Labels: make(v1.Set[string]),
}
// Since it's a new metric, potentially we already have a usage stored in the buffer.
if usage, usageExists := d.usage[metricName]; usageExists {
// TODO at some point we need to erase the usage map because it will cause a memory leak
Expand Down Expand Up @@ -204,10 +205,14 @@ func (d *db) watchLabelsQueue() {
if _, ok := d.metrics[metricName]; !ok {
// In this case, we should add the metric, because it means the metrics has been found from another source.
d.metrics[metricName] = &v1.Metric{
Labels: labels,
Labels: v1.NewSet(labels...),
}
} else {
d.metrics[metricName].Labels = utils.Merge(d.metrics[metricName].Labels, labels)
if d.metrics[metricName].Labels == nil {
d.metrics[metricName].Labels = v1.NewSet(labels...)
} else {
d.metrics[metricName].Labels.Add(labels...)
}
}
}
d.metricsMutex.Unlock()
Expand Down Expand Up @@ -249,9 +254,8 @@ func mergeUsage(old, new *v1.MetricUsage) *v1.MetricUsage {
if new == nil {
return old
}
return &v1.MetricUsage{
Dashboards: utils.Merge(old.Dashboards, new.Dashboards),
RecordingRules: utils.Merge(old.RecordingRules, new.RecordingRules),
AlertRules: utils.Merge(old.AlertRules, new.AlertRules),
}
old.Dashboards.Merge(new.Dashboards)
old.AlertRules.Merge(new.AlertRules)
old.RecordingRules.Merge(new.RecordingRules)
return old
}
55 changes: 28 additions & 27 deletions pkg/analyze/grafana/grafana.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"github.com/perses/metrics-usage/pkg/analyze/parser"
"github.com/perses/metrics-usage/pkg/analyze/prometheus"
modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
"github.com/perses/metrics-usage/utils"
)

type variableTuple struct {
Expand Down Expand Up @@ -146,24 +145,26 @@ var (
variableReplacer = strings.NewReplacer(generateGrafanaTupleVariableSyntaxReplacer(globalVariableList)...)
)

func Analyze(dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) {
func Analyze(dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) {
staticVariables := strings.NewReplacer(generateGrafanaVariableSyntaxReplacer(extractStaticVariables(dashboard.Templating.List))...)
allVariableNames := collectAllVariableName(dashboard.Templating.List)
m1, inv1, err1 := extractMetricsFromPanels(dashboard.Panels, staticVariables, allVariableNames, dashboard)
for _, r := range dashboard.Rows {
m2, inv2, err2 := extractMetricsFromPanels(r.Panels, staticVariables, allVariableNames, dashboard)
m1 = utils.Merge(m1, m2)
inv1 = utils.Merge(inv1, inv2)
m1.Merge(m2)
inv1.Merge(inv2)
err1 = append(err1, err2...)
}
m3, inv3, err3 := extractMetricsFromVariables(dashboard.Templating.List, staticVariables, allVariableNames, dashboard)
return utils.Merge(m1, m3), utils.Merge(inv1, inv3), append(err1, err3...)
m1.Merge(m3)
inv1.Merge(inv3)
return m1, inv1, append(err1, err3...)
}

func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, allVariableNames []string, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) {
func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer, allVariableNames modelAPIV1.Set[string], dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) {
var errs []*modelAPIV1.LogError
var result []string
var invalidMetricsResult []string
result := modelAPIV1.Set[string]{}
invalidMetricsResult := modelAPIV1.Set[string]{}
for _, p := range panels {
for _, t := range extractTarget(p) {
if len(t.Expr) == 0 {
Expand All @@ -174,11 +175,11 @@ func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer,
if err != nil {
otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced)
if len(otherMetrics) > 0 {
for _, m := range otherMetrics {
for m := range otherMetrics {
if prometheus.IsValidMetricName(m) {
result = utils.InsertIfNotPresent(result, m)
result.Add(m)
} else {
invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, formatVariableInMetricName(m, allVariableNames))
invalidMetricsResult.Add(formatVariableInMetricName(m, allVariableNames))
}
}
} else {
Expand All @@ -188,18 +189,18 @@ func extractMetricsFromPanels(panels []Panel, staticVariables *strings.Replacer,
})
}
} else {
result = utils.Merge(result, metrics)
invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics)
result.Merge(metrics)
invalidMetricsResult.Merge(invalidMetrics)
}
}
}
return result, invalidMetricsResult, errs
}

func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, allVariableNames []string, dashboard *SimplifiedDashboard) ([]string, []string, []*modelAPIV1.LogError) {
func extractMetricsFromVariables(variables []templateVar, staticVariables *strings.Replacer, allVariableNames modelAPIV1.Set[string], dashboard *SimplifiedDashboard) (modelAPIV1.Set[string], modelAPIV1.Set[string], []*modelAPIV1.LogError) {
var errs []*modelAPIV1.LogError
var result []string
var invalidMetricsResult []string
result := modelAPIV1.Set[string]{}
invalidMetricsResult := modelAPIV1.Set[string]{}
for _, v := range variables {
if v.Type != "query" {
continue
Expand Down Expand Up @@ -234,11 +235,11 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin
if err != nil {
otherMetrics := parser.ExtractMetricNameWithVariable(exprWithVariableReplaced)
if len(otherMetrics) > 0 {
for _, m := range otherMetrics {
for m := range otherMetrics {
if prometheus.IsValidMetricName(m) {
result = utils.InsertIfNotPresent(result, m)
result.Add(m)
} else {
invalidMetricsResult = utils.InsertIfNotPresent(invalidMetricsResult, formatVariableInMetricName(m, allVariableNames))
invalidMetricsResult.Add(formatVariableInMetricName(m, allVariableNames))
}
}
} else {
Expand All @@ -248,8 +249,8 @@ func extractMetricsFromVariables(variables []templateVar, staticVariables *strin
})
}
} else {
result = utils.Merge(result, metrics)
invalidMetricsResult = utils.Merge(invalidMetricsResult, invalidMetrics)
result.Merge(metrics)
invalidMetricsResult.Merge(invalidMetrics)
}
}
return result, invalidMetricsResult, errs
Expand All @@ -273,10 +274,10 @@ func extractStaticVariables(variables []templateVar) map[string]string {
return result
}

func collectAllVariableName(variables []templateVar) []string {
result := make([]string, len(variables))
for i, v := range variables {
result[i] = v.Name
func collectAllVariableName(variables []templateVar) modelAPIV1.Set[string] {
result := modelAPIV1.Set[string]{}
for _, v := range variables {
result.Add(v.Name)
}
return result
}
Expand All @@ -291,8 +292,8 @@ func replaceVariables(expr string, staticVariables *strings.Replacer) string {

// formatVariableInMetricName will replace the syntax of the variable by another one that can actually be parsed.
// It will be useful for later when we want to know which metrics, this metric with variable is covered.
func formatVariableInMetricName(metric string, variables []string) string {
for _, v := range variables {
func formatVariableInMetricName(metric string, variables modelAPIV1.Set[string]) string {
for v := range variables {
metric = strings.Replace(metric, fmt.Sprintf("$%s", v), fmt.Sprintf("${%s}", v), -1)
}
return metric
Expand Down
87 changes: 46 additions & 41 deletions pkg/analyze/grafana/grafana_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package grafana
import (
"encoding/json"
"os"
"slices"
"testing"

modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
Expand Down Expand Up @@ -58,59 +59,59 @@ func TestAnalyze(t *testing.T) {
name: "variable in metrics",
dashboardFile: "tests/d4.json",
resultMetrics: []string{
"otelcol_exporter_queue_capacity",
"otelcol_exporter_queue_size",
"otelcol_process_memory_rss",
"otelcol_rpc_client_request_size_bucket",
"otelcol_rpc_server_request_size_bucket",
"otelcol_rpc_client_duration_bucket",
"otelcol_rpc_server_duration_bucket",
"otelcol_rpc_client_responses_per_rpc_count",
"otelcol_rpc_server_responses_per_rpc_count",
"otelcol_process_runtime_heap_alloc_bytes",
"otelcol_process_runtime_total_sys_memory_bytes",
"otelcol_exporter_queue_size",
"otelcol_exporter_queue_capacity",
"otelcol_processor_batch_batch_send_size_sum",
"otelcol_processor_batch_batch_send_size_count",
"otelcol_processor_batch_batch_send_size_bucket",
"otelcol_processor_batch_batch_send_size_count",
"otelcol_processor_batch_batch_send_size_sum",
"otelcol_rpc_client_duration_bucket",
"otelcol_rpc_client_request_size_bucket",
"otelcol_rpc_client_responses_per_rpc_count",
"otelcol_rpc_server_duration_bucket",
"otelcol_rpc_server_request_size_bucket",
"otelcol_rpc_server_responses_per_rpc_count",
},
invalidMetrics: []string{
"otelcol_process_uptime.+",
"otelcol_exporter_.+",
"otelcol_processor_.+",
"otelcol_receiver_.+",
"otelcol_receiver_accepted_spans${suffix}",
"otelcol_receiver_refused_spans${suffix}",
"otelcol_receiver_accepted_metric_points${suffix}",
"otelcol_receiver_refused_metric_points${suffix}",
"otelcol_receiver_accepted_log_records${suffix}",
"otelcol_receiver_refused_log_records${suffix}",
"otelcol_processor_accepted_spans${suffix}",
"otelcol_processor_refused_spans${suffix}",
"otelcol_processor_dropped_spans${suffix}",
"otelcol_processor_accepted_metric_points${suffix}",
"otelcol_processor_refused_metric_points${suffix}",
"otelcol_processor_dropped_metric_points${suffix}",
"otelcol_processor_accepted_log_records${suffix}",
"otelcol_processor_refused_log_records${suffix}",
"otelcol_processor_dropped_log_records${suffix}",
"otelcol_processor_batch_batch_size_trigger_send${suffix}",
"otelcol_processor_batch_timeout_trigger_send${suffix}",
"otelcol_exporter_sent_spans${suffix}",
"otelcol_exporter_enqueue_failed_spans${suffix}",
"otelcol_exporter_send_failed_spans${suffix}",
"otelcol_exporter_sent_metric_points${suffix}",
"otelcol_exporter_enqueue_failed_log_records${suffix}",
"otelcol_exporter_enqueue_failed_metric_points${suffix}",
"otelcol_exporter_enqueue_failed_spans${suffix}",
"otelcol_exporter_send_failed_log_records${suffix}",
"otelcol_exporter_send_failed_metric_points${suffix}",
"otelcol_exporter_send_failed_spans${suffix}",
"otelcol_exporter_sent_log_records${suffix}",
"otelcol_exporter_enqueue_failed_log_records${suffix}",
"otelcol_exporter_send_failed_log_records${suffix}",
"otelcol_process_cpu_seconds${suffix}",
"otelcol_process_uptime${suffix}",
"otelcol_exporter_sent_metric_points${suffix}",
"otelcol_exporter_sent_spans${suffix}",
"otelcol_otelsvc_k8s_namespace_added${suffix}",
"otelcol_otelsvc_k8s_namespace_updated${suffix}",
"otelcol_otelsvc_k8s_pod_added${suffix}",
"otelcol_otelsvc_k8s_pod_updated${suffix}",
"otelcol_otelsvc_k8s_pod_deleted${suffix}",
"otelcol_otelsvc_k8s_pod_updated${suffix}",
"otelcol_process_cpu_seconds${suffix}",
"otelcol_process_uptime${suffix}",
"otelcol_process_uptime.+",
"otelcol_processor_.+",
"otelcol_processor_accepted_log_records${suffix}",
"otelcol_processor_accepted_metric_points${suffix}",
"otelcol_processor_accepted_spans${suffix}",
"otelcol_processor_batch_batch_size_trigger_send${suffix}",
"otelcol_processor_batch_timeout_trigger_send${suffix}",
"otelcol_processor_dropped_log_records${suffix}",
"otelcol_processor_dropped_metric_points${suffix}",
"otelcol_processor_dropped_spans${suffix}",
"otelcol_processor_refused_log_records${suffix}",
"otelcol_processor_refused_metric_points${suffix}",
"otelcol_processor_refused_spans${suffix}",
"otelcol_receiver_.+",
"otelcol_receiver_accepted_log_records${suffix}",
"otelcol_receiver_accepted_metric_points${suffix}",
"otelcol_receiver_accepted_spans${suffix}",
"otelcol_receiver_refused_log_records${suffix}",
"otelcol_receiver_refused_metric_points${suffix}",
"otelcol_receiver_refused_spans${suffix}",
},
},
}
Expand All @@ -121,8 +122,12 @@ func TestAnalyze(t *testing.T) {
t.Fatal(err)
}
metrics, invalidMetrics, errs := Analyze(dashboard)
assert.Equal(t, tt.resultMetrics, metrics)
assert.Equal(t, tt.invalidMetrics, invalidMetrics)
metricsAsSlice := metrics.TransformAsSlice()
invalidMetricsAsSlice := invalidMetrics.TransformAsSlice()
slices.Sort(metricsAsSlice)
slices.Sort(invalidMetricsAsSlice)
assert.Equal(t, tt.resultMetrics, metricsAsSlice)
assert.Equal(t, tt.invalidMetrics, invalidMetricsAsSlice)
assert.Equal(t, tt.resultErrs, errs)
})
}
Expand Down
14 changes: 9 additions & 5 deletions pkg/analyze/parser/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,21 @@

package parser

func ExtractMetricNameWithVariable(expr string) []string {
p := &parser{}
import modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"

func ExtractMetricNameWithVariable(expr string) modelAPIV1.Set[string] {
p := &parser{
metrics: modelAPIV1.Set[string]{},
}
return p.parse(expr)
}

type parser struct {
metrics []string
metrics modelAPIV1.Set[string]
currentMetric string
}

func (p *parser) parse(expr string) []string {
func (p *parser) parse(expr string) modelAPIV1.Set[string] {
query := []rune(expr)
for i := 0; i < len(query); i++ {
char := query[i]
Expand Down Expand Up @@ -64,7 +68,7 @@ func (p *parser) parse(expr string) []string {
if char == '{' {
if len(p.currentMetric) > 0 {
// That means we reached the end of a metric, so we can save it
p.metrics = append(p.metrics, p.currentMetric)
p.metrics.Add(p.currentMetric)
p.currentMetric = ""
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/analyze/parser/parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestExtractMetricNameWithVariable(t *testing.T) {
for _, test := range tests {
t.Run(test.title, func(t *testing.T) {
result := ExtractMetricNameWithVariable(test.expr)
assert.Equal(t, test.result, result)
assert.Equal(t, test.result, result.TransformAsSlice())
})
}
}
Loading

0 comments on commit c6f3147

Please sign in to comment.