Merge remote-tracking branch 'origin/main' into hl_vol_flexgroup

cmd/collectors/collectorstest.go

@@ -50,13 +50,11 @@ func JSONToGson(path string, flatten bool) []gjson.Result {
 		return nil
 	}
 	bb := b.Bytes()
-	output := gjson.GetManyBytes(bb, "records", "num_records", "_links.next.href")
+	output := gjson.ParseBytes(bb)
+	data := output.Get("records")
+	numRecords := output.Get("num_records")
 
-	data := output[0]
-	numRecords := output[1]
-	isNonIterRestCall := !data.Exists()
-
-	if isNonIterRestCall {
+	if !data.Exists() {
 		contentJSON := `{"records":[]}`
 		response, err := sjson.SetRawBytes([]byte(contentJSON), "records.-1", bb)
 		if err != nil {

cmd/collectors/restperf/restperf.go

@@ -476,18 +476,12 @@ func parseMetricResponse(instanceData gjson.Result, metric string) *metricRespon
 	for _, name := range t.Array() {
 		if name.String() == metric {
 			metricPath := "counters.#(name=" + metric + ")"
-			many := gjson.GetMany(instanceDataS,
-				metricPath+".value",
-				metricPath+".values",
-				metricPath+".labels",
-				metricPath+".counters.#.label",
-				metricPath+".counters.#.values",
-			)
-			value := many[0]
-			values := many[1]
-			labels := many[2]
-			subLabels := many[3]
-			subValues := many[4]
+			many := gjson.Parse(instanceDataS)
+			value := many.Get(metricPath + ".value")
+			values := many.Get(metricPath + ".values")
+			labels := many.Get(metricPath + ".labels")
+			subLabels := many.Get(metricPath + ".counters.#.label")
+			subValues := many.Get(metricPath + ".counters.#.values")
 			if value.String() != "" {
 				return &metricResponse{value: strings.Clone(value.String()), label: ""}
 			}

cmd/collectors/storagegrid/plugins/joinrest/joinrest.go

@@ -131,9 +131,9 @@ func (t *JoinRest) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err
 }
 
 func (t *JoinRest) updateCache(model join, bytes *[]byte) {
-	results := gjson.GetManyBytes(*bytes, "data.#."+model.JoinRest, "data.#."+model.LabelRest)
-	keys := results[0].Array()
-	vals := results[1].Array()
+	results := gjson.ParseBytes(*bytes)
+	keys := results.Get("data.#." + model.JoinRest).Array()
+	vals := results.Get("data.#." + model.LabelRest).Array()
 	if len(keys) != len(vals) {
 		t.Logger.Error().
 			Str("restKey", model.JoinRest).

cmd/collectors/storagegrid/rest/client.go

@@ -143,8 +143,8 @@ func (c *Client) Fetch(request string, result *[]gjson.Result) error {
 		return fmt.Errorf("error making request %w", err)
 	}
 
-	output := gjson.GetManyBytes(fetched, "data")
-	data = output[0]
+	output := gjson.ParseBytes(fetched)
+	data = output.Get("data")
 	for _, r := range data.Array() {
 		*result = append(*result, r.Array()...)
 	}
@@ -172,8 +172,8 @@ func (c *Client) GetMetricQuery(metric string, result *[]gjson.Result) error {
 	if err != nil {
 		return err
 	}
-	output := gjson.GetManyBytes(fetched, "data")
-	data := output[0]
+	output := gjson.ParseBytes(fetched)
+	data := output.Get("data")
 	for _, r := range data.Array() {
 		*result = append(*result, r.Array()...)
 	}
@@ -280,23 +280,24 @@ func (c *Client) Init(retries int) error {
 		if content, err = c.GetGridRest("grid/config/product-version"); err != nil {
 			continue
 		}
-		results := gjson.GetManyBytes(content, "data.productVersion")
-		err = c.SetVersion(results[0].String())
+		results := gjson.ParseBytes(content)
+		err = c.SetVersion(results.Get("data.productVersion").String())
 		if err != nil {
 			return err
 		}
 
 		if content, err = c.GetGridRest("grid/health/topology?depth=grid"); err != nil {
 			continue
 		}
-		results = gjson.GetManyBytes(content, "data.name")
-		c.Cluster.Name = strings.ReplaceAll(results[0].String(), " ", "_")
+
+		results = gjson.ParseBytes(content)
+		c.Cluster.Name = strings.ReplaceAll(results.Get("data.name").String(), " ", "_")
 
 		if content, err = c.GetGridRest("grid/license"); err != nil {
 			continue
 		}
-		results = gjson.GetManyBytes(content, "data.systemId")
-		c.Cluster.UUID = results[0].String()
+		results = gjson.ParseBytes(content)
+		c.Cluster.UUID = results.Get("data.systemId").String()
 		return nil
 	}
 
@@ -377,9 +378,9 @@ func (c *Client) fetchTokenWithAuthRetry() error {
 			return errs.NewStorageGridErr(response.StatusCode, body)
 		}
 
-		results := gjson.GetManyBytes(body, "data", "message.text")
-		token := results[0]
-		errorMsg := results[1]
+		results := gjson.ParseBytes(body)
+		token := results.Get("data")
+		errorMsg := results.Get("message.text")
 
 		if token.Exists() {
 			c.token = token.String()

cmd/collectors/zapi/plugins/aggregate/aggregate.go

@@ -52,18 +52,47 @@ func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er
 	if err := a.getCloudStores(); err != nil {
 		if errors.Is(err, errs.ErrNoInstance) {
 			a.Logger.Debug().Err(err).Msg("Failed to collect cloud store data")
-			return nil, nil
 		}
-		return nil, err
+	}
+
+	aggrFootprintMap, err := a.getAggrFootprint()
+	if err != nil {
+		a.Logger.Error().Err(err).Msg("Failed to update footprint data")
+		// clean the map in case of the error
+		clear(aggrFootprintMap)
 	}
 
 	// update aggregate instance label with cloud stores info
-	if len(a.aggrCloudStoresMap) > 0 {
-		for aggrUUID, aggr := range data.GetInstances() {
-			if !aggr.IsExportable() {
-				continue
+	for aggrUUID, aggr := range data.GetInstances() {
+		if !aggr.IsExportable() {
+			continue
+		}
+		aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[aggrUUID], ","))
+
+		// Handling aggr footprint metrics
+		aggrName := aggr.GetLabel("aggr")
+		if af, ok := aggrFootprintMap[aggrName]; ok {
+			for afKey, afVal := range af {
+				vfMetric := data.GetMetric(afKey)
+				if vfMetric == nil {
+					if vfMetric, err = data.NewMetricFloat64(afKey); err != nil {
+						a.Logger.Error().Err(err).Str("metric", afKey).Msg("add metric")
+						continue
+					}
+				}
+
+				if afVal != "" {
+					vfMetricVal, err := strconv.ParseFloat(afVal, 64)
+					if err != nil {
+						a.Logger.Error().Err(err).Str(afKey, afVal).Msg("parse")
+						continue
+					}
+					if err = vfMetric.SetValueFloat64(aggr, vfMetricVal); err != nil {
+						a.Logger.Error().Err(err).Str(afKey, afVal).Msg("set")
+						continue
+					}
+				}
 			}
-			aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[aggrUUID], ","))
 		}
 	}
 	return nil, nil
@@ -124,3 +153,44 @@ func (a *Aggregate) getCloudStores() error {
 	}
 	return nil
 }
+
+func (a *Aggregate) getAggrFootprint() (map[string]map[string]string, error) {
+	var (
+		result           []*node.Node
+		aggrFootprintMap map[string]map[string]string
+		err              error
+	)
+
+	aggrFootprintMap = make(map[string]map[string]string)
+	request := node.NewXMLS("aggr-space-get-iter")
+	request.NewChildS("max-records", collectors.DefaultBatchSize)
+	desired := node.NewXMLS("desired-attributes")
+	spaceInfo := node.NewXMLS("space-information")
+	spaceInfo.NewChildS("aggregate", "")
+	spaceInfo.NewChildS("volume-footprints", "")
+	spaceInfo.NewChildS("volume-footprints-percent", "")
+	desired.AddChild(spaceInfo)
+	request.AddChild(desired)
+
+	if result, err = a.client.InvokeZapiCall(request); err != nil {
+		return nil, err
+	}
+
+	if len(result) == 0 {
+		return aggrFootprintMap, nil
+	}
+
+	for _, footprint := range result {
+		footprintMetrics := make(map[string]string)
+		aggr := footprint.GetChildContentS("aggregate")
+		performanceTierUsed := footprint.GetChildContentS("volume-footprints")
+		performanceTierUsedPerc := footprint.GetChildContentS("volume-footprints-percent")
+		if performanceTierUsed != "" || performanceTierUsedPerc != "" {
+			footprintMetrics["space_performance_tier_used"] = performanceTierUsed
+			footprintMetrics["space_performance_tier_used_percent"] = performanceTierUsedPerc
+			aggrFootprintMap[aggr] = footprintMetrics
+		}
+	}
+
+	return aggrFootprintMap, nil
+}

cmd/collectors/zapi/plugins/volume/volume.go

@@ -92,19 +92,25 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
 	}
 
 	volumeCloneMap, err := v.getVolumeCloneInfo()
-
 	if err != nil {
 		v.Logger.Error().Err(err).Msg("Failed to update clone data")
 	}
 
+	volumeFootprintMap, err := v.getVolumeFootprint()
+	if err != nil {
+		v.Logger.Error().Err(err).Msg("Failed to update footprint data")
+		// clean the map in case of the error
+		clear(volumeFootprintMap)
+	}
+
 	// update volume instance labels
-	v.updateVolumeLabels(data, volumeCloneMap)
+	v.updateVolumeLabels(data, volumeCloneMap, volumeFootprintMap)
 
 	v.currentVal++
 	return nil, nil
 }
 
-func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeCloneMap map[string]volumeClone) {
+func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeCloneMap map[string]volumeClone, volumeFootprintMap map[string]map[string]string) {
 	var err error
 	for _, volume := range data.GetInstances() {
 		if !volume.IsExportable() {
@@ -143,6 +149,31 @@ func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeCloneMap map[stri
 				continue
 			}
 		}
+
+		// Handling volume footprint metrics
+		if vf, ok := volumeFootprintMap[key]; ok {
+			for vfKey, vfVal := range vf {
+				vfMetric := data.GetMetric(vfKey)
+				if vfMetric == nil {
+					if vfMetric, err = data.NewMetricFloat64(vfKey); err != nil {
+						v.Logger.Error().Err(err).Str("metric", vfKey).Msg("add metric")
+						continue
+					}
+				}
+
+				if vfVal != "" {
+					vfMetricVal, err := strconv.ParseFloat(vfVal, 64)
+					if err != nil {
+						v.Logger.Error().Err(err).Str(vfKey, vfVal).Msg("parse")
+						continue
+					}
+					if err = vfMetric.SetValueFloat64(volume, vfMetricVal); err != nil {
+						v.Logger.Error().Err(err).Str(vfKey, vfVal).Msg("set")
+						continue
+					}
+				}
+			}
+		}
 	}
 }
 
@@ -186,6 +217,53 @@ func (v *Volume) getVolumeCloneInfo() (map[string]volumeClone, error) {
 	return volumeCloneMap, nil
 }
 
+func (v *Volume) getVolumeFootprint() (map[string]map[string]string, error) {
+	var (
+		result             []*node.Node
+		volumeFootprintMap map[string]map[string]string
+		err                error
+	)
+
+	volumeFootprintMap = make(map[string]map[string]string)
+	request := node.NewXMLS("volume-footprint-get-iter")
+	request.NewChildS("max-records", collectors.DefaultBatchSize)
+	desired := node.NewXMLS("desired-attributes")
+	footprintInfo := node.NewXMLS("footprint-info")
+	footprintInfo.NewChildS("volume", "")
+	footprintInfo.NewChildS("vserver", "")
+	footprintInfo.NewChildS("volume-blocks-footprint-bin0", "")
+	footprintInfo.NewChildS("volume-blocks-footprint-bin0-percent", "")
+	footprintInfo.NewChildS("volume-blocks-footprint-bin1", "")
+	footprintInfo.NewChildS("volume-blocks-footprint-bin1-percent", "")
+	desired.AddChild(footprintInfo)
+	request.AddChild(desired)
+
+	if result, err = v.client.InvokeZapiCall(request); err != nil {
+		return nil, err
+	}
+
+	if len(result) == 0 {
+		return volumeFootprintMap, nil
+	}
+
+	for _, footprint := range result {
+		footprintMetrics := make(map[string]string)
+		volume := footprint.GetChildContentS("volume")
+		svm := footprint.GetChildContentS("vserver")
+		performanceTierFootprint := footprint.GetChildContentS("volume-blocks-footprint-bin0")
+		performanceTierFootprintPerc := footprint.GetChildContentS("volume-blocks-footprint-bin0-percent")
+		capacityTierFootprint := footprint.GetChildContentS("volume-blocks-footprint-bin1")
+		capacityTierFootprintPerc := footprint.GetChildContentS("volume-blocks-footprint-bin1-percent")
+		footprintMetrics["performance_tier_footprint"] = performanceTierFootprint
+		footprintMetrics["performance_tier_footprint_percent"] = performanceTierFootprintPerc
+		footprintMetrics["capacity_tier_footprint"] = capacityTierFootprint
+		footprintMetrics["capacity_tier_footprint_percent"] = capacityTierFootprintPerc
+		volumeFootprintMap[volume+svm] = footprintMetrics
+	}
+
+	return volumeFootprintMap, nil
+}
+
 func (v *Volume) getEncryptedDisks() ([]string, error) {
 	var (
 		result    []*node.Node

cmd/tools/doctor/doctor.go

@@ -96,39 +96,39 @@ func doDoctorCmd(cmd *cobra.Command, _ []string) {
 }
 
 func doDoctor(path string, confPath string) {
-	contents, err := os.ReadFile(path)
-	if err != nil {
-		fmt.Printf("error reading config file. err=%+v\n", err)
-		return
-	}
 	if opts.ShouldPrintConfig {
+		contents, err := os.ReadFile(path)
+		if err != nil {
+			fmt.Printf("error reading config file. err=%+v\n", err)
+			return
+		}
 		printRedactedConfig(path, contents)
 	}
-	checkAll(path, contents, confPath)
+	checkAll(path, confPath)
 }
 
 // checkAll runs all doctor checks
 // If all checks succeed, print nothing and exit with a return code of 0
 // Otherwise, print what failed and exit with a return code of 1
-func checkAll(path string, contents []byte, confPath string) {
+func checkAll(path string, confPath string) {
 	// See https://github.com/NetApp/harvest/issues/16 for more checks to add
 	color.DetectConsole(opts.Color)
-	// Validate that the config file can be parsed
-	harvestConfig := &conf.HarvestConfig{}
-	err := yaml.Unmarshal(contents, harvestConfig)
+
+	_, err := conf.LoadHarvestConfig(path)
 	if err != nil {
 		fmt.Printf("error reading config file=[%s] %+v\n", path, err)
 		os.Exit(1)
 		return
 	}
 
+	cfg := conf.Config
 	confPaths := filepath.SplitList(confPath)
 	anyFailed := false
-	anyFailed = !checkUniquePromPorts(*harvestConfig).isValid || anyFailed
-	anyFailed = !checkPollersExportToUniquePromPorts(*harvestConfig).isValid || anyFailed
-	anyFailed = !checkExporterTypes(*harvestConfig).isValid || anyFailed
+	anyFailed = !checkUniquePromPorts(cfg).isValid || anyFailed
+	anyFailed = !checkPollersExportToUniquePromPorts(cfg).isValid || anyFailed
+	anyFailed = !checkExporterTypes(cfg).isValid || anyFailed
 	anyFailed = !checkConfTemplates(confPaths).isValid || anyFailed
-	anyFailed = !checkCollectorName(*harvestConfig).isValid || anyFailed
+	anyFailed = !checkCollectorName(cfg).isValid || anyFailed
 
 	if anyFailed {
 		os.Exit(1)