Skip to content

Commit

Permalink
fix: correcting the alert rule expression for required labels (#3143)
Browse files Browse the repository at this point in the history 
* fix: correcting the alert rule expression for required labels

* fix: adding ci test for alert rules-1

* fix: alert rules should include required labels (#3153)

* fix: correcting the ems alert rule expression for required labels-2

* fix: adding ci test for alert rules-2

* fix: minor changes

* fix: minor changes

* fix: moved test to ci

* fix: minor changes

* fix: handled review comments

---------

Co-authored-by: Chris Grindstaff <[email protected]>
  • Loading branch information
@Hardikl @cgrinds
Hardikl and cgrinds authored Sep 23, 2024
1 parent 2e627e0 commit d24598f
Show file tree
Hide file tree
Showing 7 changed files with 334 additions and 46 deletions.
4 changes: 2 additions & 2 deletions cmd/collectors/ems/ems_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import (
// Bookend EMS testing: Simulated bookend issuing ems "wafl.vvol.offline" and ems "hm.alert.raised" with alert_id value as "RaidLeftBehindAggrAlert"
var issuingEmsNames = []string{"wafl.vvol.offline", "hm.alert.raised"}

// Default labels per ems is 5, "hm.alert.raised" ems has 10 labels and "wafl.vvol.offline" has 4 labels, total instance labels would be 24
const expectedInstanceLabelCount = 24
// Default labels per ems is 5, "hm.alert.raised" ems has 11 labels and "wafl.vvol.offline" has 4 labels, total instance labels would be 24
const expectedInstanceLabelCount = 25

// Auto resolve EMS testing: Simulated bookend issuing ems "LUN.offline" and ems "monitor.fan.critical"
var autoresolveEmsNames = []string{"LUN.offline", "monitor.fan.critical"}
Expand Down
70 changes: 58 additions & 12 deletions cmd/tools/generate/counter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ type Counter struct {
Name string `yaml:"Name"`
Description string `yaml:"Description"`
APIs []MetricDef `yaml:"APIs"`
Labels []string `yaml:"Labels"`
}

func (c Counter) Header() string {
Expand Down Expand Up @@ -209,15 +210,15 @@ func searchDescriptionSwagger(objName string, ontapCounterName string) string {
}

// processRestCounters parse rest and restperf templates
func processRestCounters(client *rest.Client) map[string]Counter {
restPerfCounters := visitRestTemplates("conf/restperf", client, func(path string, client *rest.Client) map[string]Counter {
func processRestCounters(dir string, client *rest.Client) map[string]Counter {
restPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "restperf"), client, func(path string, client *rest.Client) map[string]Counter {
if _, ok := excludePerfTemplates[filepath.Base(path)]; ok {
return nil
}
return processRestPerfCounters(path, client)
})

restCounters := visitRestTemplates("conf/rest", client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter
restCounters := visitRestTemplates(filepath.Join(dir, "conf", "rest"), client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter
return processRestConfigCounters(path)
})

Expand All @@ -228,11 +229,11 @@ func processRestCounters(client *rest.Client) map[string]Counter {
}

// processZapiCounters parse zapi and zapiperf templates
func processZapiCounters(client *zapi.Client) map[string]Counter {
zapiCounters := visitZapiTemplates("conf/zapi/cdot", client, func(path string, client *zapi.Client) map[string]Counter { // revive:disable-line:unused-parameter
func processZapiCounters(dir string, client *zapi.Client) map[string]Counter {
zapiCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapi", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { // revive:disable-line:unused-parameter
return processZapiConfigCounters(path)
})
zapiPerfCounters := visitZapiTemplates("conf/zapiperf/cdot", client, func(path string, client *zapi.Client) map[string]Counter {
zapiPerfCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapiperf", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter {
if _, ok := excludePerfTemplates[filepath.Base(path)]; ok {
return nil
}
Expand Down Expand Up @@ -302,6 +303,8 @@ func processRestConfigCounters(path string) map[string]Counter {
var (
counters = make(map[string]Counter)
)
var metricLabels []string
var labels []string
t, err := tree.ImportYaml(path)
if t == nil || err != nil {
fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err)
Expand All @@ -320,7 +323,11 @@ func processRestConfigCounters(path string) map[string]Counter {
}

if templateCounters != nil {
processCounters(templateCounters.GetAllChildContentS(), &model, path, model.Query, counters)
metricLabels, labels = getAllExportedLabels(t, templateCounters.GetAllChildContentS())
processCounters(templateCounters.GetAllChildContentS(), &model, path, model.Query, counters, metricLabels)
// This is for object_labels metrics
harvestName := model.Object + "_" + "labels"
counters[harvestName] = Counter{Name: harvestName, Labels: labels}
}

endpoints := t.GetChildS("endpoints")
Expand All @@ -332,7 +339,7 @@ func processRestConfigCounters(path string) map[string]Counter {
query = line.GetContentS()
}
if line.GetNameS() == "counters" {
processCounters(line.GetAllChildContentS(), &model, path, query, counters)
processCounters(line.GetAllChildContentS(), &model, path, query, counters, metricLabels)
}
}
}
Expand All @@ -357,7 +364,7 @@ func processRestConfigCounters(path string) map[string]Counter {
return counters
}

func processCounters(counterContents []string, model *template2.Model, path, query string, counters map[string]Counter) {
func processCounters(counterContents []string, model *template2.Model, path, query string, counters map[string]Counter, metricLabels []string) {
for _, c := range counterContents {
if c == "" {
continue
Expand All @@ -380,6 +387,7 @@ func processCounters(counterContents []string, model *template2.Model, path, que
ONTAPCounter: name,
},
},
Labels: metricLabels,
}
counters[harvestName] = co

Expand Down Expand Up @@ -469,6 +477,7 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte
}
}

metricLabels, _ := getAllExportedLabels(t, templateCounters.GetAllChildContentS())
for _, c := range templateCounters.GetAllChildContentS() {
if c != "" {
name, display, m, _ := util.ParseMetric(c)
Expand Down Expand Up @@ -496,6 +505,7 @@ func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counte
BaseCounter: zapiBaseCounterMap[name],
},
},
Labels: metricLabels,
}
counters[harvestName] = co

Expand Down Expand Up @@ -557,6 +567,8 @@ func processZapiConfigCounters(path string) map[string]Counter {
var (
counters = make(map[string]Counter)
)
var metricLabels []string
var labels []string
t, err := tree.ImportYaml(path)
if t == nil || err != nil {
fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path)
Expand All @@ -577,7 +589,10 @@ func processZapiConfigCounters(path string) map[string]Counter {
}

zc := make(map[string]string)

metricLabels, labels = getAllExportedLabels(t, templateCounters.GetAllChildContentS())
// This is for object_labels metrics
harvestName := model.Object + "_" + "labels"
counters[harvestName] = Counter{Name: harvestName, Labels: labels}
for _, c := range templateCounters.GetChildren() {
parseZapiCounters(c, []string{}, model.Object, zc)
}
Expand All @@ -596,6 +611,7 @@ func processZapiConfigCounters(path string) map[string]Counter {
ONTAPCounter: v,
},
},
Labels: metricLabels,
}
counters[k] = co

Expand Down Expand Up @@ -875,6 +891,7 @@ func processRestPerfCounters(path string, client *rest.Client) map[string]Counte
}
counterMap := make(map[string]string)
counterMapNoPrefix := make(map[string]string)
metricLabels, _ := getAllExportedLabels(t, templateCounters.GetAllChildContentS())
for _, c := range templateCounters.GetAllChildContentS() {
if c != "" {
name, display, m, _ := util.ParseMetric(c)
Expand Down Expand Up @@ -934,6 +951,7 @@ func processRestPerfCounters(path string, client *rest.Client) map[string]Counte
BaseCounter: r.Get("denominator.name").String(),
},
},
Labels: metricLabels,
}
counters[c.Name] = c

Expand Down Expand Up @@ -994,8 +1012,8 @@ func addAggregatedCounter(c *Counter, metric plugin.DerivedMetric, withPrefix st
}
}

func processExternalCounters(counters map[string]Counter) map[string]Counter {
dat, err := os.ReadFile("cmd/tools/generate/counter.yaml")
func processExternalCounters(dir string, counters map[string]Counter) map[string]Counter {
dat, err := os.ReadFile(filepath.Join(dir, "cmd", "tools", "generate", "counter.yaml"))
if err != nil {
fmt.Printf("error while reading file %v", err)
return nil
Expand Down Expand Up @@ -1057,3 +1075,31 @@ func findAPI(apis []MetricDef, other MetricDef) []int {
}
return indices
}

func getAllExportedLabels(t *node.Node, counterContents []string) ([]string, []string) {
metricLabels := make([]string, 0)
labels := make([]string, 0)
if exportOptions := t.GetChildS("export_options"); exportOptions != nil {
if iAllLabels := exportOptions.GetChildS("include_all_labels"); iAllLabels != nil {
if iAllLabels.GetContentS() == "true" {
for _, c := range counterContents {
if c == "" {
continue
}
if _, display, m, _ := util.ParseMetric(c); m == "key" || m == "label" {
metricLabels = append(metricLabels, display)
}
}
return metricLabels, metricLabels
}
}

if iKeys := exportOptions.GetChildS("instance_keys"); iKeys != nil {
metricLabels = append(metricLabels, iKeys.GetAllChildContentS()...)
}
if iLabels := exportOptions.GetChildS("instance_labels"); iLabels != nil {
labels = append(labels, iLabels.GetAllChildContentS()...)
}
}
return metricLabels, append(labels, metricLabels...)
}
34 changes: 20 additions & 14 deletions cmd/tools/generate/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,13 +136,13 @@ func doDockerCompose(cmd *cobra.Command, _ []string) {

func doGenerateMetrics(cmd *cobra.Command, _ []string) {
addRootOptions(cmd)
counters, cluster := generateMetrics()
counters, cluster := BuildMetrics("", "", opts.Poller)
generateCounterTemplate(counters, cluster.Version)
}

func doDescription(cmd *cobra.Command, _ []string) {
addRootOptions(cmd)
counters, _ := generateMetrics()
counters, _ := BuildMetrics("", "", opts.Poller)
grafana.VisitDashboards(
[]string{"grafana/dashboards/cmode"},
func(path string, data []byte) {
Expand Down Expand Up @@ -553,20 +553,26 @@ func writeAdminSystemd(configFp string) {
println(color.Colorize("✓", color.Green) + " HTTP SD file: " + harvestAdminService + " created")
}

func generateMetrics() (map[string]Counter, rest.Cluster) {
func BuildMetrics(dir, configPath, pollerName string) (map[string]Counter, rest.Cluster) {
var (
poller *conf.Poller
err error
restClient *rest.Client
zapiClient *zapi.Client
poller *conf.Poller
err error
restClient *rest.Client
zapiClient *zapi.Client
harvestYmlPath string
)

_, err = conf.LoadHarvestConfig(opts.configPath)
if opts.configPath != "" {
harvestYmlPath = filepath.Join(dir, opts.configPath)
} else {
harvestYmlPath = filepath.Join(dir, configPath)
}
_, err = conf.LoadHarvestConfig(harvestYmlPath)
if err != nil {
logErrAndExit(err)
}

if poller, _, err = rest.GetPollerAndAddr(opts.Poller); err != nil {
if poller, _, err = rest.GetPollerAndAddr(pollerName); err != nil {
logErrAndExit(err)
}

Expand All @@ -587,10 +593,10 @@ func generateMetrics() (map[string]Counter, rest.Cluster) {
}

swaggerBytes = readSwaggerJSON()
restCounters := processRestCounters(restClient)
zapiCounters := processZapiCounters(zapiClient)
restCounters := processRestCounters(dir, restClient)
zapiCounters := processZapiCounters(dir, zapiClient)
counters := mergeCounters(restCounters, zapiCounters)
counters = processExternalCounters(counters)
counters = processExternalCounters(dir, counters)
return counters, restClient.Cluster()
}

Expand Down Expand Up @@ -670,11 +676,11 @@ func init() {
fFlags := fullCmd.PersistentFlags()

flags := metricCmd.PersistentFlags()
flags.StringVarP(&opts.Poller, "poller", "p", "sar", "name of poller, e.g. 10.193.48.154")
flags.StringVarP(&opts.Poller, "poller", "p", "dc1", "name of poller, e.g. 10.193.48.154")
_ = metricCmd.MarkPersistentFlagRequired("poller")

flag := descCmd.PersistentFlags()
flag.StringVarP(&opts.Poller, "poller", "p", "sar", "name of poller, e.g. 10.193.48.154")
flag.StringVarP(&opts.Poller, "poller", "p", "dc1", "name of poller, e.g. 10.193.48.154")
_ = descCmd.MarkPersistentFlagRequired("poller")

dFlags.IntVarP(&opts.loglevel, "loglevel", "l", 2,
Expand Down
Loading

0 comments on commit d24598f

Please sign in to comment.