feat: support flexgroup constituents in template

cmd/collectors/commonutils.go

@@ -7,6 +7,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/tidwall/gjson"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -234,3 +235,12 @@ func UpdateLagTime(instance *matrix.Instance, lastTransferSize *matrix.Metric, l
 func IsValidUnit(unit string) bool {
 	return validUnits[unit]
 }
+
+func ReadPluginKey(param *node.Node, key string) bool {
+	if val := param.GetChildContentS(key); val != "" {
+		if boolValue, err := strconv.ParseBool(val); err == nil {
+			return boolValue
+		}
+	}
+	return false
+}

cmd/collectors/rest/plugins/volume/volume.go

@@ -5,6 +5,7 @@
 package volume
 
 import (
+	"github.com/hashicorp/go-version"
 	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/tools/rest"
@@ -18,113 +19,154 @@ import (
 )
 
 const HoursInMonth = 24 * 30
+const ARWSupportedVersion = "9.10.0"
 
 type Volume struct {
 	*plugin.AbstractPlugin
-	currentVal int
-	client     *rest.Client
-	aggrsMap   map[string]string // aggregate-uuid -> aggregate-name map
-	arw        *matrix.Matrix
+	currentVal            int
+	client                *rest.Client
+	aggrsMap              map[string]bool // aggregate-name -> exist map
+	arw                   *matrix.Matrix
+	includeConstituents   bool
+	isArwSupportedVersion bool
+}
+
+type volumeInfo struct {
+	arwStartTime             string
+	arwState                 string
+	snapshotAutodelete       string
+	cloneSnapshotName        string
+	cloneSplitEstimateMetric float64
 }
 
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &Volume{AbstractPlugin: p}
 }
 
-func (my *Volume) Init() error {
+func (v *Volume) Init() error {
 
 	var err error
 
-	if err = my.InitAbc(); err != nil {
+	if err = v.InitAbc(); err != nil {
 		return err
 	}
 
-	my.aggrsMap = make(map[string]string)
+	v.aggrsMap = make(map[string]bool)
 
 	// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
-	my.currentVal = my.SetPluginInterval()
+	v.currentVal = v.SetPluginInterval()
 
-	if my.Options.IsTest {
+	if v.Options.IsTest {
 		return nil
 	}
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
-	if my.client, err = rest.New(conf.ZapiPoller(my.ParentParams), timeout, my.Auth); err != nil {
-		my.Logger.Error().Stack().Err(err).Msg("connecting")
+	if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
+		v.Logger.Error().Stack().Err(err).Msg("connecting")
 		return err
 	}
 
-	if err = my.client.Init(5); err != nil {
+	if err = v.client.Init(5); err != nil {
 		return err
 	}
 
-	my.arw = matrix.New(my.Parent+".Volume", "volume_arw", "volume_arw")
+	v.arw = matrix.New(v.Parent+".Volume", "volume_arw", "volume_arw")
 	exportOptions := node.NewS("export_options")
 	instanceKeys := exportOptions.NewChildS("instance_keys", "")
 	instanceKeys.NewChildS("", "ArwStatus")
-	my.arw.SetExportOptions(exportOptions)
-	_, err = my.arw.NewMetricFloat64("status", "status")
+	v.arw.SetExportOptions(exportOptions)
+	_, err = v.arw.NewMetricFloat64("status", "status")
 	if err != nil {
-		my.Logger.Error().Stack().Err(err).Msg("add metric")
+		v.Logger.Error().Stack().Err(err).Msg("add metric")
 		return err
 	}
+
+	// Read template to decide inclusion of flexgroup constituents
+	v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
+	// ARW feature is supported from 9.10 onwards, If we ask this field in Rest call in plugin, then it will be failed.
+	v.isArwSupportedVersion = v.versionHigherThan(ARWSupportedVersion)
 	return nil
 }
 
-func (my *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
-	data := dataMap[my.Object]
-	if my.currentVal >= my.PluginInvocationRate {
-		my.currentVal = 0
+func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+	data := dataMap[v.Object]
+	if v.currentVal >= v.PluginInvocationRate {
+		v.currentVal = 0
 
 		// invoke disk rest and populate info in aggrsMap
-		if disks, err := my.getEncryptedDisks(); err != nil {
+		if disks, err := v.getEncryptedDisks(); err != nil {
 			if errs.IsRestErr(err, errs.APINotFound) {
-				my.Logger.Debug().Err(err).Msg("Failed to collect disk data")
+				v.Logger.Debug().Err(err).Msg("Failed to collect disk data")
 			} else {
-				my.Logger.Error().Err(err).Msg("Failed to collect disk data")
+				v.Logger.Error().Err(err).Msg("Failed to collect disk data")
 			}
 		} else {
 			// update aggrsMap based on disk data
-			my.updateAggrMap(disks)
+			v.updateAggrMap(disks)
 		}
 	}
 
+	volumeMap, err := v.getVolumeInfo()
+	if err != nil {
+		v.Logger.Error().Err(err).Msg("Failed to collect volume info data")
+	}
+
 	// update volume instance labels
-	my.updateVolumeLabels(data)
+	v.updateVolumeLabels(data, volumeMap)
 
 	// parse anti_ransomware_start_time, antiRansomwareState for all volumes and export at cluster level
-	my.handleARWProtection(data)
+	v.handleARWProtection(data)
 
-	my.currentVal++
-	return []*matrix.Matrix{my.arw}, nil
+	v.currentVal++
+	return []*matrix.Matrix{v.arw}, nil
 }
 
-func (my *Volume) updateVolumeLabels(data *matrix.Matrix) {
+func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeMap map[string]volumeInfo) {
+	var err error
+	cloneSplitEstimateMetric := data.GetMetric("clone_split_estimate")
+	if cloneSplitEstimateMetric == nil {
+		if cloneSplitEstimateMetric, err = data.NewMetricFloat64("clone_split_estimate"); err != nil {
+			v.Logger.Error().Stack().Msg("error while creating clone split estimate metric")
+		}
+	}
 	for _, volume := range data.GetInstances() {
-		// For flexgroup, aggrUuid in Rest should be empty for parity with Zapi response
-		if volumeStyle := volume.GetLabel("style"); volumeStyle == "flexgroup" {
-			volume.SetLabel("aggrUuid", "")
+		if !volume.IsExportable() {
+			continue
+		}
+
+		if volume.GetLabel("style") == "flexgroup_constituent" {
+			volume.SetExportable(v.includeConstituents)
 		}
-		aggrUUID := volume.GetLabel("aggrUuid")
 
-		_, exist := my.aggrsMap[aggrUUID]
-		volume.SetLabel("isHardwareEncrypted", strconv.FormatBool(exist))
+		volume.SetLabel("isHardwareEncrypted", strconv.FormatBool(v.aggrsMap[volume.GetLabel("aggr")]))
+
+		if vInfo, ok := volumeMap[volume.GetLabel("volume")+volume.GetLabel("svm")]; ok {
+			volume.SetLabel("anti_ransomware_start_time", vInfo.arwStartTime)
+			volume.SetLabel("antiRansomwareState", vInfo.arwState)
+			volume.SetLabel("snapshot_autodelete", vInfo.snapshotAutodelete)
+			if volume.GetLabel("is_flexclone") == "true" {
+				volume.SetLabel("clone_parent_snapshot", vInfo.cloneSnapshotName)
+				if err = cloneSplitEstimateMetric.SetValueFloat64(volume, vInfo.cloneSplitEstimateMetric); err != nil {
+					v.Logger.Error().Err(err).Str("metric", "cloneSplitEstimateMetric").Msg("Unable to set value on metric")
+				}
+			}
+		}
 	}
 }
 
-func (my *Volume) handleARWProtection(data *matrix.Matrix) {
+func (v *Volume) handleARWProtection(data *matrix.Matrix) {
 	var (
 		arwInstance       *matrix.Instance
 		arwStartTimeValue time.Time
 		err               error
 	)
 
 	// Purge and reset data
-	my.arw.PurgeInstances()
-	my.arw.Reset()
+	v.arw.PurgeInstances()
+	v.arw.Reset()
 
 	// Set all global labels
-	my.arw.SetGlobalLabels(data.GetGlobalLabels())
+	v.arw.SetGlobalLabels(data.GetGlobalLabels())
 	arwStatusValue := "Active Mode"
 	// Case where cluster don't have any volumes, arwStatus show as 'Not Monitoring'
 	if len(data.GetInstances()) == 0 {
@@ -153,7 +195,7 @@ func (my *Volume) handleARWProtection(data *matrix.Matrix) {
 			}
 			// If ARW startTime is more than 30 days old, which indicates that learning mode has been finished.
 			if arwStartTimeValue, err = time.Parse(time.RFC3339, arwStartTime); err != nil {
-				my.Logger.Error().Err(err).Msg("Failed to parse arw start time")
+				v.Logger.Error().Err(err).Msg("Failed to parse arw start time")
 				arwStartTimeValue = time.Now()
 			}
 			if time.Since(arwStartTimeValue).Hours() > HoursInMonth {
@@ -165,53 +207,106 @@ func (my *Volume) handleARWProtection(data *matrix.Matrix) {
 	}
 
 	arwInstanceKey := data.GetGlobalLabels()["cluster"] + data.GetGlobalLabels()["datacenter"]
-	if arwInstance, err = my.arw.NewInstance(arwInstanceKey); err != nil {
-		my.Logger.Error().Err(err).Str("arwInstanceKey", arwInstanceKey).Msg("Failed to create arw instance")
+	if arwInstance, err = v.arw.NewInstance(arwInstanceKey); err != nil {
+		v.Logger.Error().Err(err).Str("arwInstanceKey", arwInstanceKey).Msg("Failed to create arw instance")
 		return
 	}
 
 	arwInstance.SetLabel("ArwStatus", arwStatusValue)
-	m := my.arw.GetMetric("status")
+	m := v.arw.GetMetric("status")
 	// populate numeric data
 	value := 1.0
 	if err = m.SetValueFloat64(arwInstance, value); err != nil {
-		my.Logger.Error().Stack().Err(err).Float64("value", value).Msg("Failed to parse value")
+		v.Logger.Error().Stack().Err(err).Float64("value", value).Msg("Failed to parse value")
 	} else {
-		my.Logger.Debug().Float64("value", value).Msg("added value")
+		v.Logger.Debug().Float64("value", value).Msg("added value")
 	}
 }
 
-func (my *Volume) getEncryptedDisks() ([]gjson.Result, error) {
+func (v *Volume) getEncryptedDisks() ([]gjson.Result, error) {
 	var (
 		result []gjson.Result
 		err    error
 	)
-
-	fields := []string{"aggregates.name", "aggregates.uuid"}
+	fields := []string{"aggregates.name"}
 	query := "api/storage/disks"
 	href := rest.NewHrefBuilder().
 		APIPath(query).
 		Fields(fields).
 		Filter([]string{"protection_mode=!data|full"}).
 		Build()
 
-	if result, err = collectors.InvokeRestCall(my.client, href, my.Logger); err != nil {
+	if result, err = collectors.InvokeRestCall(v.client, href, v.Logger); err != nil {
 		return nil, err
 	}
 	return result, nil
 }
 
-func (my *Volume) updateAggrMap(disks []gjson.Result) {
+func (v *Volume) getVolumeInfo() (map[string]volumeInfo, error) {
+	volumeMap := make(map[string]volumeInfo)
+	fields := []string{"name", "svm.name", "space.snapshot.autodelete_enabled", "clone.parent_snapshot.name", "clone.split_estimate"}
+	if !v.isArwSupportedVersion {
+		return v.getVolume("", fields, volumeMap)
+	}
+
+	// Only ask this field when ARW would be supported, is_constituent is supported from 9.10 onwards in public api same as ARW
+	fields = append(fields, "anti_ransomware.dry_run_start_time", "anti_ransomware.state")
+	if _, err := v.getVolume("is_constituent=false", fields, volumeMap); err != nil {
+		return nil, err
+	}
+	return v.getVolume("is_constituent=true", fields, volumeMap)
+}
+
+func (v *Volume) getVolume(field string, fields []string, volumeMap map[string]volumeInfo) (map[string]volumeInfo, error) {
+	var (
+		result []gjson.Result
+		err    error
+	)
+	query := "api/storage/volumes"
+	href := rest.NewHrefBuilder().
+		APIPath(query).
+		Fields(fields).
+		Filter([]string{field}).
+		Build()
+
+	if result, err = collectors.InvokeRestCall(v.client, href, v.Logger); err != nil {
+		return nil, err
+	}
+
+	for _, volume := range result {
+		volName := volume.Get("name").String()
+		svmName := volume.Get("svm.name").String()
+		arwStartTime := volume.Get("anti_ransomware.dry_run_start_time").String()
+		arwState := volume.Get("anti_ransomware.state").String()
+		snapshotAutodelete := volume.Get("space.snapshot.autodelete_enabled").String()
+		cloneSnapshotName := volume.Get("clone.parent_snapshot.name").String()
+		cloneSplitEstimate := volume.Get("clone.split_estimate").Float()
+		volumeMap[volName+svmName] = volumeInfo{arwStartTime: arwStartTime, arwState: arwState, snapshotAutodelete: snapshotAutodelete, cloneSnapshotName: cloneSnapshotName, cloneSplitEstimateMetric: cloneSplitEstimate}
+	}
+	return volumeMap, nil
+}
+
+func (v *Volume) updateAggrMap(disks []gjson.Result) {
 	if disks != nil {
 		// Clean aggrsMap map
-		my.aggrsMap = make(map[string]string)
-
+		clear(v.aggrsMap)
 		for _, disk := range disks {
-			aggrName := disk.Get("aggregates.name").String()
-			aggrUUID := disk.Get("aggregates.uuid").String()
-			if aggrUUID != "" {
-				my.aggrsMap[aggrUUID] = aggrName
+			aggrName := disk.Get("aggregates.#.name").Array()
+			for _, aggr := range aggrName {
+				v.aggrsMap[aggr.String()] = true
 			}
 		}
 	}
 }
+
+func (v *Volume) versionHigherThan(minVersion string) bool {
+	currentVersion, err := version.NewVersion(v.client.Cluster().GetVersion())
+	if err != nil {
+		return false
+	}
+	minSupportedVersion, err := version.NewVersion(minVersion)
+	if err != nil {
+		return false
+	}
+	return currentVersion.GreaterThanOrEqual(minSupportedVersion)
+}

cmd/collectors/restperf/plugins/volume/volume.go

@@ -1,6 +1,7 @@
 package volume
 
 import (
+	"github.com/netapp/harvest/v2/cmd/collectors"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
@@ -12,7 +13,8 @@ import (
 
 type Volume struct {
 	*plugin.AbstractPlugin
-	styleType string
+	styleType           string
+	includeConstituents bool
 }
 
 func New(p *plugin.AbstractPlugin) plugin.Plugin {
@@ -31,6 +33,9 @@ func (v *Volume) Init() error {
 	if v.Params.HasChildS("historicalLabels") {
 		v.styleType = "type"
 	}
+
+	// Read template to decide inclusion of flexgroup constituents
+	v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
 	return nil
 }
 
@@ -92,7 +97,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
 			fgAggrMap[key].Add(i.GetLabel("aggr"))
 			flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
 			i.SetLabel(style, "flexgroup_constituent")
-			i.SetExportable(false)
+			i.SetExportable(v.includeConstituents)
 		} else {
 			i.SetLabel(style, "flexvol")
 			key := i.GetLabel("svm") + "." + i.GetLabel("volume")