From 805f750162f17b25c8bcc340b99304241d09e297 Mon Sep 17 00:00:00 2001 From: hardikl Date: Mon, 7 Oct 2024 20:49:22 +0530 Subject: [PATCH 1/4] fix: handled flexgroup based on volume config call --- .../restperf/plugins/volume/volume.go | 56 +++++++++++++- .../restperf/plugins/volume/volume_test.go | 27 +++++-- cmd/collectors/volume.go | 16 ++-- .../zapiperf/plugins/volume/volume.go | 73 ++++++++++++++++++- 4 files changed, 153 insertions(+), 19 deletions(-) diff --git a/cmd/collectors/restperf/plugins/volume/volume.go b/cmd/collectors/restperf/plugins/volume/volume.go index 43691d608..1b0357361 100644 --- a/cmd/collectors/restperf/plugins/volume/volume.go +++ b/cmd/collectors/restperf/plugins/volume/volume.go @@ -3,14 +3,19 @@ package volume import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/cmd/tools/rest" + "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/util" + "log/slog" + "time" ) type Volume struct { *plugin.AbstractPlugin styleType string includeConstituents bool + client *rest.Client } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -18,7 +23,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } func (v *Volume) Init() error { - + var err error if err := v.InitAbc(); err != nil { return err } @@ -31,13 +36,58 @@ func (v *Volume) Init() error { // Read template to decide inclusion of flexgroup constituents v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents") - return nil + + if v.Options.IsTest { + v.client = &rest.Client{Metadata: &util.Metadata{}} + return nil + } + + timeout, _ := time.ParseDuration(rest.DefaultTimeout) + if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil { + v.SLogger.Error("connecting", slog.Any("err", err)) + return err + } + + return v.client.Init(5) } func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] style := v.styleType opsKeyPrefix := "temp_" + volumesMap := v.fetchVolumes() + + return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, volumesMap) +} + +func (v *Volume) fetchVolumes() map[string]string { + volumesMap := make(map[string]string) + query := "api/storage/volumes" + + href := rest.NewHrefBuilder(). + APIPath(query). + Fields([]string{"volume", "volume_style_extended"}). + Build() + + records, err := rest.FetchAll(v.client, href) + if err != nil { + v.SLogger.Error("Failed to fetch data", slog.Any("err", err), slog.String("href", href)) + return nil + } + + if len(records) == 0 { + return nil + } + + for _, volume := range records { + if !volume.IsObject() { + v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String())) + continue + } + styleExtended := volume.Get("volume_style_extended").String() + name := volume.Get("volume").String() + volumesMap[name] = styleExtended + } - return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix) + return volumesMap } diff --git a/cmd/collectors/restperf/plugins/volume/volume_test.go b/cmd/collectors/restperf/plugins/volume/volume_test.go index 6f5d19ab1..513232584 100644 --- a/cmd/collectors/restperf/plugins/volume/volume_test.go +++ b/cmd/collectors/restperf/plugins/volume/volume_test.go @@ -1,9 +1,12 @@ package volume_test import ( + "github.com/netapp/harvest/v2/cmd/collectors" volume2 "github.com/netapp/harvest/v2/cmd/collectors/restperf/plugins/volume" "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/volume" + "github.com/netapp/harvest/v2/cmd/poller/options" "log/slog" + "strconv" "testing" "github.com/netapp/harvest/v2/cmd/poller/plugin" @@ -11,11 +14,16 @@ import ( "github.com/netapp/harvest/v2/pkg/tree/node" ) +const OpsKeyPrefix = "temp_" +const StyleType = "style" +const PollerName = "test" + // Common test logic for RestPerf/ZapiPerf Volume plugin func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) { params := node.NewS("Volume") params.NewChildS("include_constituents", includeConstituents) v := createVolume(params) + volumesMap := make(map[string]string) // Initialize the plugin if err := v.Init(); err != nil { @@ -28,22 +36,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu instance1.SetLabel("volume", "RahulTest__0001") instance1.SetLabel("svm", "svm1") instance1.SetLabel("aggr", "aggr1") + volumesMap["RahulTest__0001"] = "flexgroup_constituent" instance2, _ := data.NewInstance("RahulTest__0002") instance2.SetLabel("volume", "RahulTest__0002") instance2.SetLabel("svm", "svm1") instance2.SetLabel("aggr", "aggr2") + volumesMap["RahulTest__0002"] = "flexgroup_constituent" instance3, _ := data.NewInstance("RahulTest__0003") instance3.SetLabel("volume", "RahulTest__0003") instance3.SetLabel("svm", "svm1") instance3.SetLabel("aggr", "aggr3") + volumesMap["RahulTest__0003"] = "flexgroup_constituent" // Create a simple volume instance simpleInstance, _ := data.NewInstance("SimpleVolume") simpleInstance.SetLabel("volume", "SimpleVolume") simpleInstance.SetLabel("svm", "svm1") simpleInstance.SetLabel("aggr", "aggr4") + volumesMap["SimpleVolume"] = "flexvol" // Create latency and ops metrics latencyMetric, _ := data.NewMetricFloat64("read_latency") @@ -73,12 +85,9 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu _ = latencyMetric.SetValueFloat64(simpleInstance, 50) _ = opsMetric.SetValueFloat64(simpleInstance, 5) - dataMap := map[string]*matrix.Matrix{ - "volume": data, - } - // Run the plugin - output, _, err := v.Run(dataMap) + boolValue, _ := strconv.ParseBool(includeConstituents) + output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap) if err != nil { t.Fatalf("Run method failed: %v", err) } @@ -230,13 +239,17 @@ func TestRunForAllImplementations(t *testing.T) { } func createRestVolume(params *node.Node) plugin.Plugin { - v := &volume2.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)} + opts := options.New(options.WithConfPath("testdata/conf")) + opts.IsTest = true + v := &volume2.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)} v.SLogger = slog.Default() return v } func createZapiVolume(params *node.Node) plugin.Plugin { - v := &volume.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)} + opts := options.New(options.WithConfPath("testdata/conf")) + opts.IsTest = true + v := &volume.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)} v.SLogger = slog.Default() return v } diff --git a/cmd/collectors/volume.go b/cmd/collectors/volume.go index 3831ab37a..c2fb943b3 100644 --- a/cmd/collectors/volume.go +++ b/cmd/collectors/volume.go @@ -13,7 +13,7 @@ import ( var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`) -func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string) ([]*matrix.Matrix, *util.Metadata, error) { +func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) { var err error fgAggrMap := make(map[string]*set.Set) @@ -33,7 +33,10 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string cache.UUID += ".Volume" for _, i := range data.GetInstances() { - if match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 { + volName := i.GetLabel("volume") + switch volumesMap[volName] { + case "flexgroup_constituent": + match := flexgroupRegex.FindStringSubmatch(volName) key := i.GetLabel("svm") + "." + match[1] if cache.GetInstance(key) == nil { fg, _ := cache.NewInstance(key) @@ -61,9 +64,9 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string flexgroupAggrsMap[key].Add(i.GetLabel("aggr")) i.SetLabel(style, "flexgroup_constituent") i.SetExportable(includeConstituents) - } else { + case "flexvol": i.SetLabel(style, "flexvol") - key := i.GetLabel("svm") + "." + i.GetLabel("volume") + key := i.GetLabel("svm") + "." + volName flexvolInstance, err := volumeAggrmetric.NewInstance(key) if err != nil { logger.Error("Failed to create new instance", slog.Any("err", err), slog.String("key", key)) @@ -81,10 +84,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string recordFGFalse := make(map[string]*set.Set) for _, i := range data.GetInstances() { - match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")) - if len(match) != 3 { + volName := i.GetLabel("volume") + if volumesMap[volName] != "flexgroup_constituent" { continue } + match := flexgroupRegex.FindStringSubmatch(volName) key := i.GetLabel("svm") + "." + match[1] flexgroupInstance := volumeAggrmetric.GetInstance(key) diff --git a/cmd/collectors/zapiperf/plugins/volume/volume.go b/cmd/collectors/zapiperf/plugins/volume/volume.go index b9810fa63..8d5a10119 100644 --- a/cmd/collectors/zapiperf/plugins/volume/volume.go +++ b/cmd/collectors/zapiperf/plugins/volume/volume.go @@ -7,14 +7,21 @@ package volume import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi" + "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" + "log/slog" ) +const batchSize = "500" + type Volume struct { *plugin.AbstractPlugin styleType string includeConstituents bool + client *zapi.Client } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -22,7 +29,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } func (v *Volume) Init() error { - + var err error if err := v.InitAbc(); err != nil { return err } @@ -33,15 +40,75 @@ func (v *Volume) Init() error { v.styleType = "type" } + if v.Options.IsTest { + return nil + } + // Read template to decide inclusion of flexgroup constituents v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents") - return nil + + if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams), v.Auth); err != nil { + v.SLogger.Error("connecting", slog.Any("err", err)) + return err + } + return v.client.Init(5) } func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] style := v.styleType opsKeyPrefix := "temp_" + volumesMap := v.fetchVolumes() + + return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, volumesMap) +} + +func (v *Volume) fetchVolumes() map[string]string { + var ( + result *node.Node + volumes []*node.Node + volumesMap map[string]string + ) + + volumesMap = make(map[string]string) + query := "volume-get-iter" + tag := "initial" + request := node.NewXMLS(query) + request.NewChildS("max-records", batchSize) + desired := node.NewXMLS("desired-attributes") + volumeAttributes := node.NewXMLS("desired-attributes") + volumeIDAttributes := node.NewXMLS("volume-id-attributes") + volumeIDAttributes.NewChildS("name", "") + volumeIDAttributes.NewChildS("style-extended", "") + volumeAttributes.AddChild(volumeIDAttributes) + desired.AddChild(volumeAttributes) + request.AddChild(desired) + + for { + responseData, err := v.client.InvokeBatchRequest(request, tag, "") + if err != nil { + return nil + } + result = responseData.Result + tag = responseData.Tag + + if result == nil { + break + } + + if x := result.GetChildS("attributes-list"); x != nil { + volumes = x.GetChildren() + } + if len(volumes) == 0 { + return nil + } + + for _, volume := range volumes { + styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended") + name := volume.GetChildS("volume-id-attributes").GetChildContentS("name") + volumesMap[name] = styleExtended + } + } - return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix) + return volumesMap } From eb7dd269c9486ce8c5fe15b7aa5d245852f43b66 Mon Sep 17 00:00:00 2001 From: hardikl Date: Tue, 8 Oct 2024 16:04:59 +0530 Subject: [PATCH 2/4] fix: handled review comments --- .../restperf/plugins/volume/volume.go | 18 ++++++++++++++++-- cmd/collectors/volume.go | 5 +++++ .../zapiperf/plugins/volume/volume.go | 17 +++++++++++++++-- cmd/poller/plugin/plugin.go | 15 +++++++++------ 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/cmd/collectors/restperf/plugins/volume/volume.go b/cmd/collectors/restperf/plugins/volume/volume.go index 1b0357361..6efa204f9 100644 --- a/cmd/collectors/restperf/plugins/volume/volume.go +++ b/cmd/collectors/restperf/plugins/volume/volume.go @@ -13,9 +13,11 @@ import ( type Volume struct { *plugin.AbstractPlugin + currentVal int styleType string includeConstituents bool client *rest.Client + volumesMap map[string]string // volume-name -> volume-extended-style map } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -34,6 +36,11 @@ func (v *Volume) Init() error { v.styleType = "type" } + v.volumesMap = make(map[string]string) + + // Assigned the value to currentVal so that plugin would be invoked first time to populate cache. + v.currentVal = v.SetPluginInterval() + // Read template to decide inclusion of flexgroup constituents v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents") @@ -55,9 +62,15 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util data := dataMap[v.Object] style := v.styleType opsKeyPrefix := "temp_" - volumesMap := v.fetchVolumes() + if v.currentVal >= v.PluginInvocationRate { + v.currentVal = 0 + // Clean volumesMap map + clear(v.volumesMap) + v.volumesMap = v.fetchVolumes() + } - return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, volumesMap) + v.currentVal++ + return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap) } func (v *Volume) fetchVolumes() map[string]string { @@ -67,6 +80,7 @@ func (v *Volume) fetchVolumes() map[string]string { href := rest.NewHrefBuilder(). APIPath(query). Fields([]string{"volume", "volume_style_extended"}). + MaxRecords(collectors.DefaultBatchSize). Build() records, err := rest.FetchAll(v.client, href) diff --git a/cmd/collectors/volume.go b/cmd/collectors/volume.go index c2fb943b3..6284f2297 100644 --- a/cmd/collectors/volume.go +++ b/cmd/collectors/volume.go @@ -16,6 +16,11 @@ var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`) func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) { var err error + if volumesMap == nil { + logger.Info("volumes config data not found") + return nil, nil, nil + } + fgAggrMap := make(map[string]*set.Set) flexgroupAggrsMap := make(map[string]*set.Set) diff --git a/cmd/collectors/zapiperf/plugins/volume/volume.go b/cmd/collectors/zapiperf/plugins/volume/volume.go index 8d5a10119..5af6eaea5 100644 --- a/cmd/collectors/zapiperf/plugins/volume/volume.go +++ b/cmd/collectors/zapiperf/plugins/volume/volume.go @@ -19,9 +19,11 @@ const batchSize = "500" type Volume struct { *plugin.AbstractPlugin + currentVal int styleType string includeConstituents bool client *zapi.Client + volumesMap map[string]string // volume-name -> volume-extended-style map } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -44,6 +46,11 @@ func (v *Volume) Init() error { return nil } + v.volumesMap = make(map[string]string) + + // Assigned the value to currentVal so that plugin would be invoked first time to populate cache. + v.currentVal = v.SetPluginInterval() + // Read template to decide inclusion of flexgroup constituents v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents") @@ -58,9 +65,15 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util data := dataMap[v.Object] style := v.styleType opsKeyPrefix := "temp_" - volumesMap := v.fetchVolumes() + if v.currentVal >= v.PluginInvocationRate { + v.currentVal = 0 + // Clean volumesMap map + clear(v.volumesMap) + v.volumesMap = v.fetchVolumes() + } - return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, volumesMap) + v.currentVal++ + return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap) } func (v *Volume) fetchVolumes() map[string]string { diff --git a/cmd/poller/plugin/plugin.go b/cmd/poller/plugin/plugin.go index ccd4ae738..ea69207ae 100644 --- a/cmd/poller/plugin/plugin.go +++ b/cmd/poller/plugin/plugin.go @@ -167,15 +167,18 @@ func (p *AbstractPlugin) SetPluginInterval() int { } func GetInterval(param *node.Node, defaultInterval time.Duration) float64 { - schedule := param.GetChildS("schedule") - if schedule != nil { - dataInterval := schedule.GetChildContentS("data") - if dataInterval != "" { - if durationVal, err := time.ParseDuration(dataInterval); err == nil { - return durationVal.Seconds() + if param != nil { + schedule := param.GetChildS("schedule") + if schedule != nil { + dataInterval := schedule.GetChildContentS("data") + if dataInterval != "" { + if durationVal, err := time.ParseDuration(dataInterval); err == nil { + return durationVal.Seconds() + } } } } + return defaultInterval.Seconds() } From 2859e46291ee7e6bf077bbf0c3b5e5656799104f Mon Sep 17 00:00:00 2001 From: hardikl Date: Tue, 8 Oct 2024 17:29:46 +0530 Subject: [PATCH 3/4] fix: handled review comments --- cmd/collectors/restperf/plugins/volume/volume.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/collectors/restperf/plugins/volume/volume.go b/cmd/collectors/restperf/plugins/volume/volume.go index 6efa204f9..b1ef35bbd 100644 --- a/cmd/collectors/restperf/plugins/volume/volume.go +++ b/cmd/collectors/restperf/plugins/volume/volume.go @@ -75,7 +75,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util func (v *Volume) fetchVolumes() map[string]string { volumesMap := make(map[string]string) - query := "api/storage/volumes" + query := "api/private/cli/volume" href := rest.NewHrefBuilder(). APIPath(query). From 05d037976bb69d0d40956ca8462cb3659ddd1641 Mon Sep 17 00:00:00 2001 From: hardikl Date: Wed, 9 Oct 2024 12:11:30 +0530 Subject: [PATCH 4/4] fix: handled review comments --- cmd/collectors/restperf/plugins/volume/volume.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/collectors/restperf/plugins/volume/volume.go b/cmd/collectors/restperf/plugins/volume/volume.go index b1ef35bbd..c66186073 100644 --- a/cmd/collectors/restperf/plugins/volume/volume.go +++ b/cmd/collectors/restperf/plugins/volume/volume.go @@ -80,6 +80,7 @@ func (v *Volume) fetchVolumes() map[string]string { href := rest.NewHrefBuilder(). APIPath(query). Fields([]string{"volume", "volume_style_extended"}). + Filter([]string{"is_constituent=*"}). MaxRecords(collectors.DefaultBatchSize). Build()