From 391b6b748112ef2cee63e5668c7472d23eafa3ae Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:16:47 +0530 Subject: [PATCH] fix: handled flexgroup based on volume config call (#3199) * fix: handled flexgroup based on volume config call --- .../restperf/plugins/volume/volume.go | 71 ++++++++++++++- .../restperf/plugins/volume/volume_test.go | 27 ++++-- cmd/collectors/volume.go | 21 +++-- .../zapiperf/plugins/volume/volume.go | 86 ++++++++++++++++++- cmd/poller/plugin/plugin.go | 15 ++-- 5 files changed, 195 insertions(+), 25 deletions(-) diff --git a/cmd/collectors/restperf/plugins/volume/volume.go b/cmd/collectors/restperf/plugins/volume/volume.go index 43691d608..c66186073 100644 --- a/cmd/collectors/restperf/plugins/volume/volume.go +++ b/cmd/collectors/restperf/plugins/volume/volume.go @@ -3,14 +3,21 @@ package volume import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/cmd/tools/rest" + "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/util" + "log/slog" + "time" ) type Volume struct { *plugin.AbstractPlugin + currentVal int styleType string includeConstituents bool + client *rest.Client + volumesMap map[string]string // volume-name -> volume-extended-style map } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -18,7 +25,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } func (v *Volume) Init() error { - + var err error if err := v.InitAbc(); err != nil { return err } @@ -29,15 +36,73 @@ func (v *Volume) Init() error { v.styleType = "type" } + v.volumesMap = make(map[string]string) + + // Assigned the value to currentVal so that plugin would be invoked first time to populate cache. + v.currentVal = v.SetPluginInterval() + // Read template to decide inclusion of flexgroup constituents v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents") - return nil + + if v.Options.IsTest { + v.client = &rest.Client{Metadata: &util.Metadata{}} + return nil + } + + timeout, _ := time.ParseDuration(rest.DefaultTimeout) + if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil { + v.SLogger.Error("connecting", slog.Any("err", err)) + return err + } + + return v.client.Init(5) } func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] style := v.styleType opsKeyPrefix := "temp_" + if v.currentVal >= v.PluginInvocationRate { + v.currentVal = 0 + // Clean volumesMap map + clear(v.volumesMap) + v.volumesMap = v.fetchVolumes() + } + + v.currentVal++ + return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap) +} + +func (v *Volume) fetchVolumes() map[string]string { + volumesMap := make(map[string]string) + query := "api/private/cli/volume" + + href := rest.NewHrefBuilder(). + APIPath(query). + Fields([]string{"volume", "volume_style_extended"}). + Filter([]string{"is_constituent=*"}). + MaxRecords(collectors.DefaultBatchSize). + Build() + + records, err := rest.FetchAll(v.client, href) + if err != nil { + v.SLogger.Error("Failed to fetch data", slog.Any("err", err), slog.String("href", href)) + return nil + } + + if len(records) == 0 { + return nil + } + + for _, volume := range records { + if !volume.IsObject() { + v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String())) + continue + } + styleExtended := volume.Get("volume_style_extended").String() + name := volume.Get("volume").String() + volumesMap[name] = styleExtended + } - return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix) + return volumesMap } diff --git a/cmd/collectors/restperf/plugins/volume/volume_test.go b/cmd/collectors/restperf/plugins/volume/volume_test.go index 6f5d19ab1..513232584 100644 --- a/cmd/collectors/restperf/plugins/volume/volume_test.go +++ b/cmd/collectors/restperf/plugins/volume/volume_test.go @@ -1,9 +1,12 @@ package volume_test import ( + "github.com/netapp/harvest/v2/cmd/collectors" volume2 "github.com/netapp/harvest/v2/cmd/collectors/restperf/plugins/volume" "github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/volume" + "github.com/netapp/harvest/v2/cmd/poller/options" "log/slog" + "strconv" "testing" "github.com/netapp/harvest/v2/cmd/poller/plugin" @@ -11,11 +14,16 @@ import ( "github.com/netapp/harvest/v2/pkg/tree/node" ) +const OpsKeyPrefix = "temp_" +const StyleType = "style" +const PollerName = "test" + // Common test logic for RestPerf/ZapiPerf Volume plugin func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) { params := node.NewS("Volume") params.NewChildS("include_constituents", includeConstituents) v := createVolume(params) + volumesMap := make(map[string]string) // Initialize the plugin if err := v.Init(); err != nil { @@ -28,22 +36,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu instance1.SetLabel("volume", "RahulTest__0001") instance1.SetLabel("svm", "svm1") instance1.SetLabel("aggr", "aggr1") + volumesMap["RahulTest__0001"] = "flexgroup_constituent" instance2, _ := data.NewInstance("RahulTest__0002") instance2.SetLabel("volume", "RahulTest__0002") instance2.SetLabel("svm", "svm1") instance2.SetLabel("aggr", "aggr2") + volumesMap["RahulTest__0002"] = "flexgroup_constituent" instance3, _ := data.NewInstance("RahulTest__0003") instance3.SetLabel("volume", "RahulTest__0003") instance3.SetLabel("svm", "svm1") instance3.SetLabel("aggr", "aggr3") + volumesMap["RahulTest__0003"] = "flexgroup_constituent" // Create a simple volume instance simpleInstance, _ := data.NewInstance("SimpleVolume") simpleInstance.SetLabel("volume", "SimpleVolume") simpleInstance.SetLabel("svm", "svm1") simpleInstance.SetLabel("aggr", "aggr4") + volumesMap["SimpleVolume"] = "flexvol" // Create latency and ops metrics latencyMetric, _ := data.NewMetricFloat64("read_latency") @@ -73,12 +85,9 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu _ = latencyMetric.SetValueFloat64(simpleInstance, 50) _ = opsMetric.SetValueFloat64(simpleInstance, 5) - dataMap := map[string]*matrix.Matrix{ - "volume": data, - } - // Run the plugin - output, _, err := v.Run(dataMap) + boolValue, _ := strconv.ParseBool(includeConstituents) + output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap) if err != nil { t.Fatalf("Run method failed: %v", err) } @@ -230,13 +239,17 @@ func TestRunForAllImplementations(t *testing.T) { } func createRestVolume(params *node.Node) plugin.Plugin { - v := &volume2.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)} + opts := options.New(options.WithConfPath("testdata/conf")) + opts.IsTest = true + v := &volume2.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)} v.SLogger = slog.Default() return v } func createZapiVolume(params *node.Node) plugin.Plugin { - v := &volume.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)} + opts := options.New(options.WithConfPath("testdata/conf")) + opts.IsTest = true + v := &volume.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)} v.SLogger = slog.Default() return v } diff --git a/cmd/collectors/volume.go b/cmd/collectors/volume.go index 3e894d268..69567f0b9 100644 --- a/cmd/collectors/volume.go +++ b/cmd/collectors/volume.go @@ -14,9 +14,14 @@ import ( var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`) -func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string) ([]*matrix.Matrix, *util.Metadata, error) { +func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) { var err error + if volumesMap == nil { + logger.Info("volumes config data not found") + return nil, nil, nil + } + fgAggrMap := make(map[string]*set.Set) flexgroupAggrsMap := make(map[string]*set.Set) @@ -34,7 +39,10 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string cache.UUID += ".Volume" for _, i := range data.GetInstances() { - if match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 { + volName := i.GetLabel("volume") + switch volumesMap[volName] { + case "flexgroup_constituent": + match := flexgroupRegex.FindStringSubmatch(volName) key := i.GetLabel("svm") + "." + match[1] if cache.GetInstance(key) == nil { fg, _ := cache.NewInstance(key) @@ -62,9 +70,9 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string flexgroupAggrsMap[key].Add(i.GetLabel("aggr")) i.SetLabel(style, "flexgroup_constituent") i.SetExportable(includeConstituents) - } else { + case "flexvol": i.SetLabel(style, "flexvol") - key := i.GetLabel("svm") + "." + i.GetLabel("volume") + key := i.GetLabel("svm") + "." + volName flexvolInstance, err := volumeAggrmetric.NewInstance(key) if err != nil { logger.Error("Failed to create new instance", slogx.Err(err), slog.String("key", key)) @@ -82,10 +90,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string recordFGFalse := make(map[string]*set.Set) for _, i := range data.GetInstances() { - match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")) - if len(match) != 3 { + volName := i.GetLabel("volume") + if volumesMap[volName] != "flexgroup_constituent" { continue } + match := flexgroupRegex.FindStringSubmatch(volName) key := i.GetLabel("svm") + "." + match[1] flexgroupInstance := volumeAggrmetric.GetInstance(key) diff --git a/cmd/collectors/zapiperf/plugins/volume/volume.go b/cmd/collectors/zapiperf/plugins/volume/volume.go index b9810fa63..5af6eaea5 100644 --- a/cmd/collectors/zapiperf/plugins/volume/volume.go +++ b/cmd/collectors/zapiperf/plugins/volume/volume.go @@ -7,14 +7,23 @@ package volume import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi" + "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" + "log/slog" ) +const batchSize = "500" + type Volume struct { *plugin.AbstractPlugin + currentVal int styleType string includeConstituents bool + client *zapi.Client + volumesMap map[string]string // volume-name -> volume-extended-style map } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -22,7 +31,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } func (v *Volume) Init() error { - + var err error if err := v.InitAbc(); err != nil { return err } @@ -33,15 +42,86 @@ func (v *Volume) Init() error { v.styleType = "type" } + if v.Options.IsTest { + return nil + } + + v.volumesMap = make(map[string]string) + + // Assigned the value to currentVal so that plugin would be invoked first time to populate cache. + v.currentVal = v.SetPluginInterval() + // Read template to decide inclusion of flexgroup constituents v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents") - return nil + + if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams), v.Auth); err != nil { + v.SLogger.Error("connecting", slog.Any("err", err)) + return err + } + return v.client.Init(5) } func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] style := v.styleType opsKeyPrefix := "temp_" + if v.currentVal >= v.PluginInvocationRate { + v.currentVal = 0 + // Clean volumesMap map + clear(v.volumesMap) + v.volumesMap = v.fetchVolumes() + } + + v.currentVal++ + return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap) +} + +func (v *Volume) fetchVolumes() map[string]string { + var ( + result *node.Node + volumes []*node.Node + volumesMap map[string]string + ) + + volumesMap = make(map[string]string) + query := "volume-get-iter" + tag := "initial" + request := node.NewXMLS(query) + request.NewChildS("max-records", batchSize) + desired := node.NewXMLS("desired-attributes") + volumeAttributes := node.NewXMLS("desired-attributes") + volumeIDAttributes := node.NewXMLS("volume-id-attributes") + volumeIDAttributes.NewChildS("name", "") + volumeIDAttributes.NewChildS("style-extended", "") + volumeAttributes.AddChild(volumeIDAttributes) + desired.AddChild(volumeAttributes) + request.AddChild(desired) + + for { + responseData, err := v.client.InvokeBatchRequest(request, tag, "") + if err != nil { + return nil + } + result = responseData.Result + tag = responseData.Tag + + if result == nil { + break + } + + if x := result.GetChildS("attributes-list"); x != nil { + volumes = x.GetChildren() + } + if len(volumes) == 0 { + return nil + } + + for _, volume := range volumes { + styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended") + name := volume.GetChildS("volume-id-attributes").GetChildContentS("name") + volumesMap[name] = styleExtended + } + } - return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix) + return volumesMap } diff --git a/cmd/poller/plugin/plugin.go b/cmd/poller/plugin/plugin.go index ccd4ae738..ea69207ae 100644 --- a/cmd/poller/plugin/plugin.go +++ b/cmd/poller/plugin/plugin.go @@ -167,15 +167,18 @@ func (p *AbstractPlugin) SetPluginInterval() int { } func GetInterval(param *node.Node, defaultInterval time.Duration) float64 { - schedule := param.GetChildS("schedule") - if schedule != nil { - dataInterval := schedule.GetChildContentS("data") - if dataInterval != "" { - if durationVal, err := time.ParseDuration(dataInterval); err == nil { - return durationVal.Seconds() + if param != nil { + schedule := param.GetChildS("schedule") + if schedule != nil { + dataInterval := schedule.GetChildContentS("data") + if dataInterval != "" { + if durationVal, err := time.ParseDuration(dataInterval); err == nil { + return durationVal.Seconds() + } } } } + return defaultInterval.Seconds() }