From d52fa5397bba2b934fde5a94021e9c439b4e5817 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Tue, 3 Sep 2024 14:27:27 -0400 Subject: [PATCH 1/2] feat: grafana import should allow rewriting cluster label Fixes: #3131 --- cmd/tools/grafana/grafana.go | 69 ++++++++++++++++++++++++++++++++++-- cmd/tools/grafana/metrics.go | 7 ++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go index c1dee2b9d..789e94440 100644 --- a/cmd/tools/grafana/grafana.go +++ b/cmd/tools/grafana/grafana.go @@ -62,6 +62,7 @@ type options struct { svmRegex string customizeDir string customAllValue string + customCluster string } type Folder struct { @@ -516,6 +517,11 @@ func importFiles(dir string, folder *Folder) { data = addSvmRegex(data, file.Name(), opts.svmRegex) } + // change cluster label if needed + if opts.customCluster != "" { + data = changeClusterLabel(data, opts.customCluster) + } + // labelMap is used to ensure we don't modify the query of one of the new labels we're adding labelMap := make(map[string]string) caser := cases.Title(language.Und) @@ -587,6 +593,60 @@ func importFiles(dir string, folder *Folder) { } } +// This function will rewrite all panel expressions in the dashboard to use the new cluster label. +// Example: +// sum(write_data{datacenter=~"$Datacenter",cluster=~"$Cluster",svm=~"$SVM"}) +// with --cluster-label=na_cluster will become +// sum(write_data{datacenter=~"$Datacenter",na_cluster=~"$Cluster",svm=~"$SVM"}) +// See https://github.com/NetApp/harvest/issues/3131 +func changeClusterLabel(data []byte, cluster string) []byte { + + // Change all panel expressions + VisitAllPanels(data, func(path string, _, value gjson.Result) { + value.Get("targets").ForEach(func(targetKey, target gjson.Result) bool { + expr := target.Get("expr") + if expr.Exists() { + newExpr := strings.ReplaceAll(expr.String(), "cluster", cluster) + + destPath := path + ".targets." + targetKey.String() + ".expr" + data, _ = sjson.SetBytes(data, destPath, []byte(newExpr)) + } + + legendFormat := target.Get("legendFormat") + if legendFormat.Exists() { + newLegendFormat := strings.ReplaceAll(legendFormat.String(), "cluster", cluster) + + destPath := path + ".targets." + targetKey.String() + ".legendFormat" + data, _ = sjson.SetBytes(data, destPath, []byte(newLegendFormat)) + } + + return true + }) + }) + + // Change all templating variables that contain cluster + gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool { + + // Change definition + definition := value.Get("definition") + if definition.Exists() { + newDefinition := strings.ReplaceAll(definition.String(), "cluster", cluster) + data, _ = sjson.SetBytes(data, "templating.list."+key.String()+".definition", []byte(newDefinition)) + } + + // Change query + query := value.Get("query.query") + if query.Exists() { + newQuery := strings.ReplaceAll(query.String(), "cluster", cluster) + data, _ = sjson.SetBytes(data, "templating.list."+key.String()+".query.query", []byte(newQuery)) + } + + return true + }) + + return data +} + func writeCustomDashboard(dashboard map[string]any, dir string, file os.DirEntry) error { data, err := json.Marshal(dashboard) if err != nil { @@ -617,8 +677,8 @@ func formatJSON(data []byte) ([]byte, error) { return prettyJSON.Bytes(), nil } -// addGlobalPrefix adds the given prefix to all metric names in the -// dashboards. It assumes that metrics are in Prometheus-format. +// addGlobalPrefix adds the given prefix to all metric names in the dashboards. +// It assumes that metrics are in Prometheus format. // // A more reliable implementation of this feature would be, to // add a constant prefix to all metrics, before they are pushed @@ -717,7 +777,7 @@ func handlingPanels(p interface{}, prefix string) { // addPrefixToMetricNames adds prefix to metric names in expr or leaves it // unchanged if no metric names are identified. -// Note that this function will only work with the Prometheus-dashboards of Harvest. +// Note that this function will only work with the Prometheus dashboards of Harvest. // It will use a number of patterns in which metrics might be used in queries. // (E.g. a single metric, multiple metrics used in addition, etc. -- for examples // see the test). If we change queries of our dashboards, we have to review @@ -1153,6 +1213,9 @@ func addImportCustomizeFlags(commands ...*cobra.Command) { "Modify the dashboards to add multi-select dropdowns for each variable") cmd.PersistentFlags().BoolVar(&opts.forceImport, "force", false, "Import even if the datasource name is not defined in Grafana") + cmd.PersistentFlags().StringVar(&opts.customCluster, "cluster-label", "", + "Rewrite all panel expressions to use the specified cluster label instead of the default 'cluster'") + _ = cmd.PersistentFlags().MarkHidden("multi") _ = cmd.PersistentFlags().MarkHidden("force") } diff --git a/cmd/tools/grafana/metrics.go b/cmd/tools/grafana/metrics.go index 53166b961..ae5909226 100644 --- a/cmd/tools/grafana/metrics.go +++ b/cmd/tools/grafana/metrics.go @@ -190,8 +190,11 @@ func VisitAllPanels(data []byte, handle func(path string, key gjson.Result, valu func visitPanels(data []byte, panelPath string, pathPrefix string, handle func(path string, key gjson.Result, value gjson.Result)) { gjson.GetBytes(data, panelPath).ForEach(func(key, value gjson.Result) bool { - path := fmt.Sprintf("%s[%d]", panelPath, key.Int()) - fullPath := fmt.Sprintf("%s%s", pathPrefix, path) + path := panelPath + "." + key.String() + fullPath := path + if pathPrefix != "" { + fullPath = pathPrefix + "." + path + } handle(fullPath, key, value) visitPanels([]byte(value.Raw), "panels", fullPath, handle) return true From dd82969a2929d57611e6bee57d7c625bf632183e Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Thu, 5 Sep 2024 08:39:32 -0400 Subject: [PATCH 2/2] feat: grafana import should allow rewriting cluster label Fixes: #3131 --- cmd/tools/grafana/grafana.go | 116 +++++++++++++++++++++++++++--- cmd/tools/grafana/grafana_test.go | 57 +++++++++++++++ 2 files changed, 165 insertions(+), 8 deletions(-) diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go index 789e94440..be5dd7506 100644 --- a/cmd/tools/grafana/grafana.go +++ b/cmd/tools/grafana/grafana.go @@ -603,25 +603,105 @@ func changeClusterLabel(data []byte, cluster string) []byte { // Change all panel expressions VisitAllPanels(data, func(path string, _, value gjson.Result) { + + // Rewrite expressions and legends value.Get("targets").ForEach(func(targetKey, target gjson.Result) bool { expr := target.Get("expr") if expr.Exists() { - newExpr := strings.ReplaceAll(expr.String(), "cluster", cluster) + newExpression := rewriteCluster(expr.String(), cluster) - destPath := path + ".targets." + targetKey.String() + ".expr" - data, _ = sjson.SetBytes(data, destPath, []byte(newExpr)) + data, _ = sjson.SetBytes(data, path+".targets."+targetKey.String()+".expr", []byte(newExpression)) } legendFormat := target.Get("legendFormat") if legendFormat.Exists() { - newLegendFormat := strings.ReplaceAll(legendFormat.String(), "cluster", cluster) + newLegendFormat := rewriteCluster(legendFormat.String(), cluster) - destPath := path + ".targets." + targetKey.String() + ".legendFormat" - data, _ = sjson.SetBytes(data, destPath, []byte(newLegendFormat)) + data, _ = sjson.SetBytes(data, path+".targets."+targetKey.String()+".legendFormat", []byte(newLegendFormat)) } return true }) + + // Rewrite tables columns + panelType := value.Get("type") + if panelType.String() == "table" { + value.Get("transformations").ForEach(func(transKey, value gjson.Result) bool { + id := value.Get("id") + if id.String() == "organize" { + + // Check if the cluster exists in renameByName, and if so, rename it to the new cluster label + clusterTrans := value.Get("options.renameByName.cluster") + if clusterTrans.Exists() { + data, _ = sjson.SetBytes(data, path+".transformations."+transKey.String()+".options.renameByName."+cluster, []byte(clusterTrans.String())) + } + + // If the cluster column exists, remove the column, and add the new cluster label at the same index + clusterIndex := value.Get("options.indexByName.cluster") + if clusterIndex.Exists() { + data, _ = sjson.SetBytes(data, path+".transformations."+transKey.String()+".options.indexByName."+cluster, clusterIndex.Int()) + data, _ = sjson.DeleteBytes(data, path+".transformations."+transKey.String()+".options.indexByName.cluster") + } + // Handle the case where the cluster column is named "cluster 1", "cluster 2", etc. + for i := range 10 { + clusterN := "cluster " + strconv.Itoa(i) + clusterIndexI := value.Get("options.indexByName." + clusterN) + if clusterIndexI.Exists() { + data, _ = sjson.SetBytes(data, path+".transformations."+transKey.String()+".options.indexByName."+cluster+" "+strconv.Itoa(i), clusterIndexI.Int()) + data, _ = sjson.DeleteBytes(data, path+".transformations."+transKey.String()+".options.indexByName."+clusterN) + } + } + + // If cluster is excluded from the table, exclude the new cluster label too + excludeByName := value.Get("options.excludeByName") + if excludeByName.Exists() { + clusterIndex := value.Get("options.excludeByName.cluster") + if clusterIndex.Exists() { + data, _ = sjson.SetBytes(data, path+".transformations."+transKey.String()+".options.excludeByName."+cluster, clusterIndex.Int()) + data, _ = sjson.DeleteBytes(data, path+".transformations."+transKey.String()+".options.excludeByName.cluster") + } + // Handle the case where the cluster column is named "cluster 1", "cluster 2", etc. + for i := range 10 { + clusterN := "cluster " + strconv.Itoa(i) + clusterIndexI := value.Get("options.excludeByName." + clusterN) + if clusterIndexI.Exists() { + data, _ = sjson.SetBytes(data, path+".transformations."+transKey.String()+".options.excludeByName."+cluster+" "+strconv.Itoa(i), true) + data, _ = sjson.DeleteBytes(data, path+".transformations."+transKey.String()+".options.excludeByName."+clusterN) + } + } + } + } else if id.String() == "filterFieldsByName" { + // Check if the cluster exists in filterFieldsByName, and if so, rename it to the new cluster label + names := value.Get("options.include.names") + if names.Exists() { + var nameValues []string + hasCluster := false + for _, name := range names.Array() { + if name.String() == "cluster" { + hasCluster = true + nameValues = append(nameValues, cluster) + continue + } + nameValues = append(nameValues, name.String()) + } + + if hasCluster { + data, _ = sjson.SetBytes(data, path+".transformations."+transKey.String()+".options.include.names", nameValues) + } + } + } + + return true + }) + + // Change all fieldConfig overrides that contain cluster + value.Get("fieldConfig.overrides").ForEach(func(overrideKey, override gjson.Result) bool { + if override.Get("matcher.id").String() == "byName" && override.Get("matcher.options").String() == "cluster" { + data, _ = sjson.SetBytes(data, path+".fieldConfig.overrides."+overrideKey.String()+".matcher.options", cluster) + } + return true + }) + } }) // Change all templating variables that contain cluster @@ -630,14 +710,15 @@ func changeClusterLabel(data []byte, cluster string) []byte { // Change definition definition := value.Get("definition") if definition.Exists() { - newDefinition := strings.ReplaceAll(definition.String(), "cluster", cluster) + newDefinition := rewriteCluster(definition.String(), cluster) + data, _ = sjson.SetBytes(data, "templating.list."+key.String()+".definition", []byte(newDefinition)) } // Change query query := value.Get("query.query") if query.Exists() { - newQuery := strings.ReplaceAll(query.String(), "cluster", cluster) + newQuery := rewriteCluster(query.String(), cluster) data, _ = sjson.SetBytes(data, "templating.list."+key.String()+".query.query", []byte(newQuery)) } @@ -647,6 +728,25 @@ func changeClusterLabel(data []byte, cluster string) []byte { return data } +func rewriteCluster(input string, cluster string) string { + const marker = `!!^` + hiddenNames := []string{"cluster_new_status", "source_cluster"} + for _, name := range hiddenNames { + if strings.Contains(input, name) { + // hide name + repl := strings.ReplaceAll(name, "cluster", marker) + input = strings.ReplaceAll(input, name, repl) + } + } + + result := strings.ReplaceAll(input, "cluster", cluster) + + // Restore hidden names + result = strings.ReplaceAll(result, marker, "cluster") + + return result +} + func writeCustomDashboard(dashboard map[string]any, dir string, file os.DirEntry) error { data, err := json.Marshal(dashboard) if err != nil { diff --git a/cmd/tools/grafana/grafana_test.go b/cmd/tools/grafana/grafana_test.go index bf9c756b0..548fdcea2 100644 --- a/cmd/tools/grafana/grafana_test.go +++ b/cmd/tools/grafana/grafana_test.go @@ -631,3 +631,60 @@ func TestAddLabel(t *testing.T) { }) } } + +func TestClusterRewrite(t *testing.T) { + type test struct { + name string + input string + want string + cluster string + } + + tests := []test{ + { + name: "no cluster label", + input: `abc{datacenter=~\"$Datacenter\"})`, + want: `abc{datacenter=~\"$Datacenter\"})`, + cluster: "netapp_cluster", + }, + { + name: "multiple cluster labels", + input: `sum by(site_name,cluster,datacenter)(storagegrid_storage_utilization_data_bytes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})`, + want: `sum by(site_name,netapp_cluster,datacenter)(storagegrid_storage_utilization_data_bytes{datacenter=~\"$Datacenter\",netapp_cluster=~\"$Cluster\"})`, + cluster: "netapp_cluster", + }, + { + name: "by cluster label", + input: `sum by (cluster) (abc{cluster="$Cluster"}[2m)`, + want: `sum by (netapp_cluster) (abc{netapp_cluster="$Cluster"}[2m)`, + cluster: "netapp_cluster", + }, + { + name: "cluster_new_status should not change", + input: `"label_values(cluster_new_status{}, cluster)`, + want: `"label_values(cluster_new_status{}, netapp_cluster)`, + cluster: "netapp_cluster", + }, + { + name: "cluster_new_status should not change 2", + input: `cluster_new_status{}, cluster`, + want: `cluster_new_status{}, netapp_cluster`, + cluster: "netapp_cluster", + }, + { + name: "snapmirror var", + input: `snapmirror_labels{source_cluster) "source_cluster", "$1", "cluster", "(.*)")`, + want: `snapmirror_labels{source_cluster) "source_cluster", "$1", "netapp_cluster", "(.*)")`, + cluster: "netapp_cluster", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := rewriteCluster(tt.input, tt.cluster) + if got != tt.want { + t.Errorf("TestClusterLabel\n got=%v\nwant=%v", got, tt.want) + } + }) + } +}