From 5756e781260b8769157148a167d64c3fee2a85aa Mon Sep 17 00:00:00 2001 From: hardikl Date: Tue, 17 Sep 2024 22:07:59 +0530 Subject: [PATCH] fix: adding ci test for alert rules-2 --- cmd/collectors/alert_rule_test.go | 274 +++++++----------------- cmd/collectors/commonutils.go | 31 +++ cmd/collectors/ems/ems_test.go | 4 +- cmd/tools/generate/counter.go | 70 ++++-- cmd/tools/generate/generate.go | 29 +-- conf/ems/9.6.0/ems.yaml | 37 +++- integration/test/dashboard_json_test.go | 35 +-- 7 files changed, 215 insertions(+), 265 deletions(-) diff --git a/cmd/collectors/alert_rule_test.go b/cmd/collectors/alert_rule_test.go index 3965fb8c7..61682c589 100644 --- a/cmd/collectors/alert_rule_test.go +++ b/cmd/collectors/alert_rule_test.go @@ -2,18 +2,13 @@ package collectors import ( "fmt" - template2 "github.com/netapp/harvest/v2/cmd/tools/template" - "github.com/netapp/harvest/v2/pkg/conf" + "github.com/netapp/harvest/v2/cmd/tools/generate" "github.com/netapp/harvest/v2/pkg/tree" "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" - "log" - "maps" - "os" "path/filepath" "regexp" "slices" - "strconv" "strings" "testing" ) @@ -25,7 +20,6 @@ type AlertRule struct { } var labelRegex = regexp.MustCompile(`\{{.*?}}`) -var isStringAlphabetic = regexp.MustCompile(`^[a-zA-Z0-9_]*$`).MatchString var pluginGeneratedMetric = map[string][]string{ "change_log": {"svm", "state", "type", "anti_ransomware_state", "object", "node", "location", "healthy", "volume", "style", "aggr", "status"}, } @@ -34,29 +28,50 @@ var exceptionMetrics = []string{ "node_nfs_latency", // need to check why it was skipped in restperf } -func TestParseAlertRules(t *testing.T) { - metrics := getRestMetrics(t, "../../") - maps.Copy(metrics, pluginGeneratedMetric) +func TestAlertRules(t *testing.T) { + metrics, _ := generate.GeneratedMetrics("../../", "harvest.yml") + for pluginMetric, pluginLabels := range pluginGeneratedMetric { + metrics[pluginMetric] = generate.Counter{Name: pluginMetric, Labels: pluginLabels} + } - alertRules := GetAllAlertRules("../../container/prometheus/", "alert_rules.yml") + alertRules := GetAllAlertRules("../../container/prometheus/", "alert_rules.yml", false) for _, alertRule := range alertRules { - if strings.Contains(strings.Join(alertRule.exprs, ","), "_labels") { - continue - } - for _, expr := range alertRule.exprs { - if !slices.Contains(exceptionMetrics, expr) { - metricLabels := metrics[expr] - for _, label := range alertRule.labels { - if !slices.Contains(metricLabels, label) { - t.Errorf("%s is not available in %s metric", label, expr) + for _, label := range alertRule.labels { + found := false + for _, expr := range alertRule.exprs { + if !slices.Contains(exceptionMetrics, expr) { + metricCounters := metrics[expr] + if slices.Contains(metricCounters.Labels, label) { + found = true + break } + } else { + found = true } } + if !found { + t.Errorf("%s is not available in %s metric", label, alertRule.exprs) + } } } } -func GetAllAlertRules(dir string, fileName string) []AlertRule { +func TestEmsAlertRules(t *testing.T) { + templateEmsLabels := getEmsLabels("../../conf/ems/9.6.0/ems.yaml") + emsAlertRules := GetAllAlertRules("../../container/prometheus/", "ems_alert_rules.yml", true) + for _, alertRule := range emsAlertRules { + for _, ems := range alertRule.exprs { + emsLabels := templateEmsLabels[ems] + for _, label := range alertRule.labels { + if !slices.Contains(emsLabels, label) { + t.Errorf("%s is not available in %s ems", label, ems) + } + } + } + } +} + +func GetAllAlertRules(dir string, fileName string, isEms bool) []AlertRule { alertRules := make([]AlertRule, 0) alertNames := make([]string, 0) exprList := make([]string, 0) @@ -87,15 +102,19 @@ func GetAllAlertRules(dir string, fileName string) []AlertRule { } for i := range alertNames { - alertRules = append(alertRules, AlertRule{name: alertNames[i], exprs: getAllExpressions(exprList[i]), labels: getAllLabels(summaryList[i])}) + alertRules = append(alertRules, AlertRule{name: alertNames[i], exprs: getAllExpressions(exprList[i], isEms), labels: getAllLabels(summaryList[i])}) } return alertRules } -func getAllExpressions(expression string) []string { - all := FindStringBetweenTwoChar(expression, "{", "(") +func getAllExpressions(expression string, isEms bool) []string { filtered := make([]string, 0) - + var all []string + if isEms { + all = FindEms(expression, "{", "}") + } else { + all = FindStringBetweenTwoChar(expression, "{", "(") + } for _, counter := range all { if counter == "" { continue @@ -105,32 +124,17 @@ func getAllExpressions(expression string) []string { return filtered } -func FindStringBetweenTwoChar(stringValue string, startChar string, endChar string) []string { - var counters = make([]string, 0) +func FindEms(stringValue string, startChar string, endChar string) []string { + var emsSlice = make([]string, 0) firstSet := strings.Split(stringValue, startChar) - for _, actualString := range firstSet { - counterArray := strings.Split(actualString, endChar) - switch { - case strings.Contains(actualString, ")"): // check for inner expression such as top: - counterArray = strings.Split(actualString, ")") - case strings.Contains(actualString, "+"): // check for inner expression such as top: - counterArray = strings.Split(actualString, "+") - case strings.Contains(actualString, "/"): // check for inner expression such as top: - counterArray = strings.Split(actualString, "/") - case strings.Contains(actualString, ","): // check for inner expression such as top: - counterArray = strings.Split(actualString, ",") - } - counter := strings.TrimSpace(counterArray[len(counterArray)-1]) - counterArray = strings.Split(counter, endChar) - counter = strings.TrimSpace(counterArray[len(counterArray)-1]) - if _, err := strconv.Atoi(counter); err == nil { - continue - } - if isStringAlphabetic(counter) && counter != "" { - counters = append(counters, counter) - } - } - return counters + actualString := strings.TrimSpace(firstSet[1]) + counterArray := strings.Split(actualString, endChar) + ems := strings.TrimSpace(counterArray[0]) + if ems != "" { + counterArray = strings.Split(ems, "=") + emsSlice = append(emsSlice, strings.ReplaceAll(counterArray[1], "\"", "")) + } + return emsSlice } func getAllLabels(summary string) []string { @@ -147,168 +151,38 @@ func getAllLabels(summary string) []string { return labels } -func getRestMetrics(t *testing.T, path string) map[string][]string { - var ( - err error - ) - - harvestYmlPath := filepath.Join(path, "harvest.yml") - _, err = conf.LoadHarvestConfig(harvestYmlPath) - if err != nil { - t.Fatalf("Unable to load harvest config file %s. Error: %v", harvestYmlPath, err) - } - - restCounters := processRestCounters(path) - return restCounters -} - -func processRestCounters(path string) map[string][]string { - restPerfCounters := visitRestTemplates(filepath.Join(path, "conf", "restperf"), processRestPerfCounters) - restCounters := visitRestTemplates(filepath.Join(path, "conf", "rest"), processRestConfigCounters) - - maps.Copy(restCounters, restPerfCounters) - - return restCounters -} - -func visitRestTemplates(dir string, eachTemp func(path string) map[string][]string) map[string][]string { - result := make(map[string][]string) - err := filepath.WalkDir(dir, func(path string, _ os.DirEntry, err error) error { - if err != nil { - log.Fatal("failed to read directory:", err) - } - ext := filepath.Ext(path) - if ext != ".yaml" { - return nil - } - if strings.HasSuffix(path, "default.yaml") { - return nil - } - r := eachTemp(path) - maps.Copy(result, r) - return nil - }) - - if err != nil { - log.Fatal("failed to read template:", err) - return nil - } - return result -} - -func processRestPerfCounters(path string) map[string][]string { - var ( - counters = make(map[string][]string) - ) - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) - return nil - } - model, err := template2.ReadTemplate(path) - if err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - if model.ExportData == "false" && noExtraMetrics { - return nil - } - if templateCounters == nil { - return nil - } - counterMap := make(map[string]string) - counterMapNoPrefix := make(map[string]string) - labels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) - for _, c := range templateCounters.GetAllChildContentS() { - if c != "" { - name, display, m, _ := util.ParseMetric(c) - if m == "float" { - counterMap[name] = model.Object + "_" + display - counterMapNoPrefix[name] = display - counters[name] = labels - } - } - } - - return counters -} - -func processRestConfigCounters(path string) map[string][]string { +func getEmsLabels(path string) map[string][]string { var ( - counters = make(map[string][]string) + emsLabels = make(map[string][]string) ) - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - - model, err := template2.ReadTemplate(path) - if err != nil { + emsNames := make([]string, 0) + labels := make([]string, 0) + data, err := tree.ImportYaml(path) + if data == nil || err != nil { fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) return nil } - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - if model.ExportData == "false" && noExtraMetrics { - return nil - } - - if templateCounters != nil { - processCounters(t, templateCounters.GetAllChildContentS(), &model, counters) - } - endpoints := t.GetChildS("endpoints") - if endpoints != nil { - for _, endpoint := range endpoints.GetChildren() { - for _, line := range endpoint.GetChildren() { - if line.GetNameS() == "counters" { - processCounters(line, line.GetAllChildContentS(), &model, counters) - } - } - } + for _, e := range data.GetChildS("events").GetChildren() { + emsNames = append(emsNames, e.GetChildContentS("name")) + labels = append(labels, parseEmsLabels(e.GetChildS("exports"))) } - return counters -} -func processCounters(t *node.Node, counterContents []string, model *template2.Model, counters map[string][]string) { - labels := getAllExportedLabels(t, counterContents) - for _, c := range counterContents { - if c == "" { - continue - } - _, display, m, _ := util.ParseMetric(c) - harvestName := model.Object + "_" + display - if m == "float" { - counters[harvestName] = labels - } + for i := range emsNames { + emsLabels[emsNames[i]] = strings.Split(labels[i], ",") } - harvestName := model.Object + "_" + "labels" - counters[harvestName] = labels + return emsLabels } -func getAllExportedLabels(t *node.Node, counterContents []string) []string { - labels := make([]string, 0) - if exportOptions := t.GetChildS("export_options"); exportOptions != nil { - if iAllLabels := exportOptions.GetChildS("include_all_labels"); iAllLabels != nil { - if iAllLabels.GetContentS() == "true" { - for _, c := range counterContents { - if c == "" { - continue - } - if _, display, m, _ := util.ParseMetric(c); m == "key" || m == "label" { - labels = append(labels, display) - } - } - return labels +func parseEmsLabels(exports *node.Node) string { + var labels []string + if exports != nil { + for _, export := range exports.GetAllChildContentS() { + name, _, _, _ := util.ParseMetric(export) + if strings.HasPrefix(name, "parameters") { + labels = append(labels, strings.Split(name, ".")[1]) } } - - if iKeys := exportOptions.GetChildS("instance_keys"); iKeys != nil { - labels = append(labels, iKeys.GetAllChildContentS()...) - } } - return labels + return strings.Join(labels, ",") } diff --git a/cmd/collectors/commonutils.go b/cmd/collectors/commonutils.go index fc371fd07..a6a2d51e1 100644 --- a/cmd/collectors/commonutils.go +++ b/cmd/collectors/commonutils.go @@ -9,6 +9,7 @@ import ( "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "os" + "regexp" "sort" "strconv" "strings" @@ -20,6 +21,8 @@ const ( MaxAllowedTimeDrift = 10 * time.Second ) +var isStringAlphabetic = regexp.MustCompile(`^[a-zA-Z0-9_]*$`).MatchString + var validUnits = map[string]bool{ "mW": true, "W": true, @@ -514,3 +517,31 @@ func PopulateIfgroupMetrics(portIfgroupMap map[string]string, portDataMap map[st } return nil } + +func FindStringBetweenTwoChar(stringValue string, startChar string, endChar string) []string { + var counters = make([]string, 0) + firstSet := strings.Split(stringValue, startChar) + for _, actualString := range firstSet { + counterArray := strings.Split(actualString, endChar) + switch { + case strings.Contains(actualString, ")"): // check for inner expression such as top: + counterArray = strings.Split(actualString, ")") + case strings.Contains(actualString, "+"): // check for inner expression such as top: + counterArray = strings.Split(actualString, "+") + case strings.Contains(actualString, "/"): // check for inner expression such as top: + counterArray = strings.Split(actualString, "/") + case strings.Contains(actualString, ","): // check for inner expression such as top: + counterArray = strings.Split(actualString, ",") + } + counter := strings.TrimSpace(counterArray[len(counterArray)-1]) + counterArray = strings.Split(counter, endChar) + counter = strings.TrimSpace(counterArray[len(counterArray)-1]) + if _, err := strconv.Atoi(counter); err == nil { + continue + } + if isStringAlphabetic(counter) && counter != "" { + counters = append(counters, counter) + } + } + return counters +} diff --git a/cmd/collectors/ems/ems_test.go b/cmd/collectors/ems/ems_test.go index 50af08a01..810b92985 100644 --- a/cmd/collectors/ems/ems_test.go +++ b/cmd/collectors/ems/ems_test.go @@ -17,8 +17,8 @@ import ( // Bookend EMS testing: Simulated bookend issuing ems "wafl.vvol.offline" and ems "hm.alert.raised" with alert_id value as "RaidLeftBehindAggrAlert" var issuingEmsNames = []string{"wafl.vvol.offline", "hm.alert.raised"} -// Default labels per ems is 5, "hm.alert.raised" ems has 10 labels and "wafl.vvol.offline" has 4 labels, total instance labels would be 24 -const expectedInstanceLabelCount = 24 +// Default labels per ems is 5, "hm.alert.raised" ems has 11 labels and "wafl.vvol.offline" has 4 labels, total instance labels would be 24 +const expectedInstanceLabelCount = 25 // Auto resolve EMS testing: Simulated bookend issuing ems "LUN.offline" and ems "monitor.fan.critical" var autoresolveEmsNames = []string{"LUN.offline", "monitor.fan.critical"} diff --git a/cmd/tools/generate/counter.go b/cmd/tools/generate/counter.go index b95fb642e..d02383341 100644 --- a/cmd/tools/generate/counter.go +++ b/cmd/tools/generate/counter.go @@ -118,6 +118,7 @@ type Counter struct { Name string `yaml:"Name"` Description string `yaml:"Description"` APIs []MetricDef `yaml:"APIs"` + Labels []string `yaml:"Labels"` } func (c Counter) Header() string { @@ -167,15 +168,15 @@ func searchDescriptionSwagger(objName string, ontapCounterName string) string { } // processRestCounters parse rest and restperf templates -func processRestCounters(client *rest.Client) map[string]Counter { - restPerfCounters := visitRestTemplates("conf/restperf", client, func(path string, client *rest.Client) map[string]Counter { +func processRestCounters(dir string, client *rest.Client) map[string]Counter { + restPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "restperf"), client, func(path string, client *rest.Client) map[string]Counter { if _, ok := excludePerfTemplates[filepath.Base(path)]; ok { return nil } return processRestPerfCounters(path, client) }) - restCounters := visitRestTemplates("conf/rest", client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter + restCounters := visitRestTemplates(filepath.Join(dir, "conf", "rest"), client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter return processRestConfigCounters(path) }) @@ -186,11 +187,11 @@ func processRestCounters(client *rest.Client) map[string]Counter { } // processZapiCounters parse zapi and zapiperf templates -func processZapiCounters(client *zapi.Client) map[string]Counter { - zapiCounters := visitZapiTemplates("conf/zapi/cdot", client, func(path string, client *zapi.Client) map[string]Counter { // revive:disable-line:unused-parameter +func processZapiCounters(dir string, client *zapi.Client) map[string]Counter { + zapiCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapi", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { // revive:disable-line:unused-parameter return processZapiConfigCounters(path) }) - zapiPerfCounters := visitZapiTemplates("conf/zapiperf/cdot", client, func(path string, client *zapi.Client) map[string]Counter { + zapiPerfCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapiperf", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { if _, ok := excludePerfTemplates[filepath.Base(path)]; ok { return nil } @@ -260,6 +261,8 @@ func processRestConfigCounters(path string) map[string]Counter { var ( counters = make(map[string]Counter) ) + var metricLabels []string + var labels []string t, err := tree.ImportYaml(path) if t == nil || err != nil { fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) @@ -278,7 +281,11 @@ func processRestConfigCounters(path string) map[string]Counter { } if templateCounters != nil { - processCounters(templateCounters.GetAllChildContentS(), &model, path, model.Query, counters) + metricLabels, labels = getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + processCounters(templateCounters.GetAllChildContentS(), &model, path, model.Query, counters, metricLabels) + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + counters[harvestName] = Counter{Name: harvestName, Labels: labels} } endpoints := t.GetChildS("endpoints") @@ -290,7 +297,7 @@ func processRestConfigCounters(path string) map[string]Counter { query = line.GetContentS() } if line.GetNameS() == "counters" { - processCounters(line.GetAllChildContentS(), &model, path, query, counters) + processCounters(line.GetAllChildContentS(), &model, path, query, counters, metricLabels) } } } @@ -315,7 +322,7 @@ func processRestConfigCounters(path string) map[string]Counter { return counters } -func processCounters(counterContents []string, model *template2.Model, path, query string, counters map[string]Counter) { +func processCounters(counterContents []string, model *template2.Model, path, query string, counters map[string]Counter, metricLabels []string) { for _, c := range counterContents { if c == "" { continue @@ -338,6 +345,7 @@ func processCounters(counterContents []string, model *template2.Model, path, que ONTAPCounter: name, }, }, + Labels: metricLabels, } counters[harvestName] = co @@ -427,6 +435,7 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte } } + metricLabels, _ := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) for _, c := range templateCounters.GetAllChildContentS() { if c != "" { name, display, m, _ := util.ParseMetric(c) @@ -454,6 +463,7 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte BaseCounter: zapiBaseCounterMap[name], }, }, + Labels: metricLabels, } counters[harvestName] = co @@ -515,6 +525,8 @@ func processZapiConfigCounters(path string) map[string]Counter { var ( counters = make(map[string]Counter) ) + var metricLabels []string + var labels []string t, err := tree.ImportYaml(path) if t == nil || err != nil { fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) @@ -535,7 +547,10 @@ func processZapiConfigCounters(path string) map[string]Counter { } zc := make(map[string]string) - + metricLabels, labels = getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + counters[harvestName] = Counter{Name: harvestName, Labels: labels} for _, c := range templateCounters.GetChildren() { parseZapiCounters(c, []string{}, model.Object, zc) } @@ -554,6 +569,7 @@ func processZapiConfigCounters(path string) map[string]Counter { ONTAPCounter: v, }, }, + Labels: metricLabels, } counters[k] = co @@ -823,6 +839,7 @@ func processRestPerfCounters(path string, client *rest.Client) map[string]Counte } counterMap := make(map[string]string) counterMapNoPrefix := make(map[string]string) + metricLabels, _ := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) for _, c := range templateCounters.GetAllChildContentS() { if c != "" { name, display, m, _ := util.ParseMetric(c) @@ -882,6 +899,7 @@ func processRestPerfCounters(path string, client *rest.Client) map[string]Counte BaseCounter: r.Get("denominator.name").String(), }, }, + Labels: metricLabels, } counters[c.Name] = c @@ -942,8 +960,8 @@ func addAggregatedCounter(c *Counter, metric plugin.DerivedMetric, withPrefix st } } -func processExternalCounters(counters map[string]Counter) map[string]Counter { - dat, err := os.ReadFile("cmd/tools/generate/counter.yaml") +func processExternalCounters(dir string, counters map[string]Counter) map[string]Counter { + dat, err := os.ReadFile(filepath.Join(dir, "cmd", "tools", "generate", "counter.yaml")) if err != nil { fmt.Printf("error while reading file %v", err) return nil @@ -1005,3 +1023,31 @@ func findAPI(apis []MetricDef, other MetricDef) []int { } return indices } + +func getAllExportedLabels(t *node.Node, counterContents []string) ([]string, []string) { + metricLabels := make([]string, 0) + labels := make([]string, 0) + if exportOptions := t.GetChildS("export_options"); exportOptions != nil { + if iAllLabels := exportOptions.GetChildS("include_all_labels"); iAllLabels != nil { + if iAllLabels.GetContentS() == "true" { + for _, c := range counterContents { + if c == "" { + continue + } + if _, display, m, _ := util.ParseMetric(c); m == "key" || m == "label" { + metricLabels = append(metricLabels, display) + } + } + return metricLabels, metricLabels + } + } + + if iKeys := exportOptions.GetChildS("instance_keys"); iKeys != nil { + metricLabels = append(metricLabels, iKeys.GetAllChildContentS()...) + } + if iLabels := exportOptions.GetChildS("instance_labels"); iLabels != nil { + labels = append(labels, iLabels.GetAllChildContentS()...) + } + } + return metricLabels, append(labels, metricLabels...) +} diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go index db4137314..c0da35b3b 100644 --- a/cmd/tools/generate/generate.go +++ b/cmd/tools/generate/generate.go @@ -136,13 +136,13 @@ func doDockerCompose(cmd *cobra.Command, _ []string) { func doGenerateMetrics(cmd *cobra.Command, _ []string) { addRootOptions(cmd) - counters, cluster := generateMetrics() + counters, cluster := GeneratedMetrics("", "") generateCounterTemplate(counters, cluster.Version) } func doDescription(cmd *cobra.Command, _ []string) { addRootOptions(cmd) - counters, _ := generateMetrics() + counters, _ := GeneratedMetrics("", "") grafana.VisitDashboards( []string{"grafana/dashboards/cmode"}, func(path string, data []byte) { @@ -553,19 +553,24 @@ func writeAdminSystemd(configFp string) { println(color.Colorize("✓", color.Green) + " HTTP SD file: " + harvestAdminService + " created") } -func generateMetrics() (map[string]Counter, rest.Cluster) { +func GeneratedMetrics(dir, configPath string) (map[string]Counter, rest.Cluster) { var ( - poller *conf.Poller - err error - restClient *rest.Client - zapiClient *zapi.Client + poller *conf.Poller + err error + restClient *rest.Client + zapiClient *zapi.Client + harvestYmlPath string ) - _, err = conf.LoadHarvestConfig(opts.configPath) + if opts.configPath != "" { + harvestYmlPath = filepath.Join(dir, opts.configPath) + } else { + harvestYmlPath = filepath.Join(dir, configPath) + } + _, err = conf.LoadHarvestConfig(harvestYmlPath) if err != nil { logErrAndExit(err) } - if poller, _, err = rest.GetPollerAndAddr(opts.Poller); err != nil { logErrAndExit(err) } @@ -587,10 +592,10 @@ func generateMetrics() (map[string]Counter, rest.Cluster) { } swaggerBytes = readSwaggerJSON() - restCounters := processRestCounters(restClient) - zapiCounters := processZapiCounters(zapiClient) + restCounters := processRestCounters(dir, restClient) + zapiCounters := processZapiCounters(dir, zapiClient) counters := mergeCounters(restCounters, zapiCounters) - counters = processExternalCounters(counters) + counters = processExternalCounters(dir, counters) return counters, restClient.Cluster() } diff --git a/conf/ems/9.6.0/ems.yaml b/conf/ems/9.6.0/ems.yaml index 887716f00..c8b1014d4 100644 --- a/conf/ems/9.6.0/ems.yaml +++ b/conf/ems/9.6.0/ems.yaml @@ -138,18 +138,21 @@ events: - name: Nblade.vscanVirusDetected exports: - - parameters.SID => sid - - parameters.clientIp => client_ip - - parameters.filePath => file_path - - parameters.object_type => object_type - - parameters.object_uuid => object_uuid - - parameters.vscanServerIp => vscan_server_ip - - parameters.vserverName => svm + - parameters.SID => sid + - parameters.clientIp => client_ip + - parameters.filePath => file_path + - parameters.object_type => object_type + - parameters.object_uuid => object_uuid + - parameters.vscanServerIp => vscan_server_ip + - parameters.vserverName => svm + - parameters.vscanEngineStatus => vscanEngineStatus + - parameters.vscanEngineResultString => vscanEngineResultString - name: arl.netra.ca.check.failed exports: - parameters.aggr_uuid => aggr_uuid - parameters.vol => volume + - parameters.reason => reason - name: arw.volume.state matches: @@ -269,6 +272,7 @@ events: - parameters.object_uuid => object_uuid - parameters.operation => operation - parameters.shareName => share + - parameters.errMsg => errMsg - name: cloud.aws.iamNotInitialized exports: @@ -319,6 +323,7 @@ events: exports: - parameters.aggr_uuid => aggr_uuid - parameters.vol => volume + - parameters.reason => reason - name: hm.alert.raised matches: @@ -335,6 +340,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -354,6 +360,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -373,6 +380,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -392,6 +400,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -411,6 +420,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -430,6 +440,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -449,6 +460,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -468,6 +480,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -487,6 +500,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -506,6 +520,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -525,6 +540,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -544,6 +560,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -563,6 +580,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -582,6 +600,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -601,6 +620,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -620,6 +640,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -639,6 +660,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h @@ -658,6 +680,7 @@ events: - parameters.possible_effect => possible_effect - parameters.suppress => suppress - parameters.suppressor => suppressor + - parameters.detailed_info => detailed_info resolve_when_ems: - name: hm.alert.cleared resolve_after: 672h diff --git a/integration/test/dashboard_json_test.go b/integration/test/dashboard_json_test.go index 6698fbbbc..f843a7fbf 100644 --- a/integration/test/dashboard_json_test.go +++ b/integration/test/dashboard_json_test.go @@ -4,14 +4,13 @@ import ( "fmt" "github.com/Netapp/harvest-automation/test/dashboard" "github.com/Netapp/harvest-automation/test/utils" + "github.com/netapp/harvest/v2/cmd/collectors" "github.com/rs/zerolog/log" "github.com/tidwall/gjson" "golang.org/x/text/cases" "golang.org/x/text/language" "os" "path/filepath" - "regexp" - "strconv" "strings" "testing" "time" @@ -340,7 +339,7 @@ func getAllExpr(record gjson.Result) []string { } func getAllCounters(expression string) []string { - all := FindStringBetweenTwoChar(expression, "{", "(") + all := collectors.FindStringBetweenTwoChar(expression, "{", "(") var filtered []string allLoop: @@ -361,7 +360,7 @@ allLoop: func validateExpr(expression string) (bool, string) { if expression != "" { - counters := FindStringBetweenTwoChar(expression, "{", "(") + counters := collectors.FindStringBetweenTwoChar(expression, "{", "(") newExpression := expression if len(counters) > 0 { for _, counter := range counters { @@ -395,34 +394,6 @@ func GetAllJsons(dir string) []string { return fileSet } -func FindStringBetweenTwoChar(stringValue string, startChar string, endChar string) []string { - var counters = make([]string, 0) - var isStringAlphabetic = regexp.MustCompile(`^[a-zA-Z0-9_]*$`).MatchString - firstSet := strings.Split(stringValue, startChar) - for _, actualString := range firstSet { - counterArray := strings.Split(actualString, endChar) - switch { - case strings.Contains(actualString, "+"): // check for inner expression such as top: - counterArray = strings.Split(actualString, "+") - case strings.Contains(actualString, "/"): // check for inner expression such as top: - counterArray = strings.Split(actualString, "/") - case strings.Contains(actualString, ","): // check for inner expression such as top: - counterArray = strings.Split(actualString, ",") - } - - counter := strings.TrimSpace(counterArray[len(counterArray)-1]) - counterArray = strings.Split(counter, endChar) - counter = strings.TrimSpace(counterArray[len(counterArray)-1]) - if _, err := strconv.Atoi(counter); err == nil { - continue - } - if isStringAlphabetic(counter) && counter != "" { - counters = append(counters, counter) - } - } - return counters -} - func hasDataInDB(query string, waitFor time.Duration) bool { now := time.Now() for {