Skip to content

Commit

Permalink
fix: handled flexgroup based on volume config call (#3199)
Browse files Browse the repository at this point in the history 
* fix: handled flexgroup based on volume config call
  • Loading branch information
@Hardikl
Hardikl authored Oct 9, 2024
1 parent f0e2c6a commit 391b6b7
Show file tree
Hide file tree
Showing 5 changed files with 195 additions and 25 deletions.
71 changes: 68 additions & 3 deletions cmd/collectors/restperf/plugins/volume/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,29 @@ package volume
import (
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
"time"
)

type Volume struct {
*plugin.AbstractPlugin
currentVal int
styleType string
includeConstituents bool
client *rest.Client
volumesMap map[string]string // volume-name -> volume-extended-style map
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Volume{AbstractPlugin: p}
}

func (v *Volume) Init() error {

var err error
if err := v.InitAbc(); err != nil {
return err
}
Expand All @@ -29,15 +36,73 @@ func (v *Volume) Init() error {
v.styleType = "type"
}

v.volumesMap = make(map[string]string)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
return nil

if v.Options.IsTest {
v.client = &rest.Client{Metadata: &util.Metadata{}}
return nil
}

timeout, _ := time.ParseDuration(rest.DefaultTimeout)
if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
v.SLogger.Error("connecting", slog.Any("err", err))
return err
}

return v.client.Init(5)
}

func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
opsKeyPrefix := "temp_"
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0
// Clean volumesMap map
clear(v.volumesMap)
v.volumesMap = v.fetchVolumes()
}

v.currentVal++
return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
}

func (v *Volume) fetchVolumes() map[string]string {
volumesMap := make(map[string]string)
query := "api/private/cli/volume"

href := rest.NewHrefBuilder().
APIPath(query).
Fields([]string{"volume", "volume_style_extended"}).
Filter([]string{"is_constituent=*"}).
MaxRecords(collectors.DefaultBatchSize).
Build()

records, err := rest.FetchAll(v.client, href)
if err != nil {
v.SLogger.Error("Failed to fetch data", slog.Any("err", err), slog.String("href", href))
return nil
}

if len(records) == 0 {
return nil
}

for _, volume := range records {
if !volume.IsObject() {
v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String()))
continue
}
styleExtended := volume.Get("volume_style_extended").String()
name := volume.Get("volume").String()
volumesMap[name] = styleExtended
}

return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix)
return volumesMap
}
27 changes: 20 additions & 7 deletions cmd/collectors/restperf/plugins/volume/volume_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,29 @@
package volume_test

import (
"github.com/netapp/harvest/v2/cmd/collectors"
volume2 "github.com/netapp/harvest/v2/cmd/collectors/restperf/plugins/volume"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/volume"
"github.com/netapp/harvest/v2/cmd/poller/options"
"log/slog"
"strconv"
"testing"

"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
)

const OpsKeyPrefix = "temp_"
const StyleType = "style"
const PollerName = "test"

// Common test logic for RestPerf/ZapiPerf Volume plugin
func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) {
params := node.NewS("Volume")
params.NewChildS("include_constituents", includeConstituents)
v := createVolume(params)
volumesMap := make(map[string]string)

// Initialize the plugin
if err := v.Init(); err != nil {
Expand All @@ -28,22 +36,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
instance1.SetLabel("volume", "RahulTest__0001")
instance1.SetLabel("svm", "svm1")
instance1.SetLabel("aggr", "aggr1")
volumesMap["RahulTest__0001"] = "flexgroup_constituent"

instance2, _ := data.NewInstance("RahulTest__0002")
instance2.SetLabel("volume", "RahulTest__0002")
instance2.SetLabel("svm", "svm1")
instance2.SetLabel("aggr", "aggr2")
volumesMap["RahulTest__0002"] = "flexgroup_constituent"

instance3, _ := data.NewInstance("RahulTest__0003")
instance3.SetLabel("volume", "RahulTest__0003")
instance3.SetLabel("svm", "svm1")
instance3.SetLabel("aggr", "aggr3")
volumesMap["RahulTest__0003"] = "flexgroup_constituent"

// Create a simple volume instance
simpleInstance, _ := data.NewInstance("SimpleVolume")
simpleInstance.SetLabel("volume", "SimpleVolume")
simpleInstance.SetLabel("svm", "svm1")
simpleInstance.SetLabel("aggr", "aggr4")
volumesMap["SimpleVolume"] = "flexvol"

// Create latency and ops metrics
latencyMetric, _ := data.NewMetricFloat64("read_latency")
Expand Down Expand Up @@ -73,12 +85,9 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
_ = latencyMetric.SetValueFloat64(simpleInstance, 50)
_ = opsMetric.SetValueFloat64(simpleInstance, 5)

dataMap := map[string]*matrix.Matrix{
"volume": data,
}

// Run the plugin
output, _, err := v.Run(dataMap)
boolValue, _ := strconv.ParseBool(includeConstituents)
output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap)
if err != nil {
t.Fatalf("Run method failed: %v", err)
}
Expand Down Expand Up @@ -230,13 +239,17 @@ func TestRunForAllImplementations(t *testing.T) {
}

func createRestVolume(params *node.Node) plugin.Plugin {
v := &volume2.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)}
opts := options.New(options.WithConfPath("testdata/conf"))
opts.IsTest = true
v := &volume2.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)}
v.SLogger = slog.Default()
return v
}

func createZapiVolume(params *node.Node) plugin.Plugin {
v := &volume.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)}
opts := options.New(options.WithConfPath("testdata/conf"))
opts.IsTest = true
v := &volume.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)}
v.SLogger = slog.Default()
return v
}
21 changes: 15 additions & 6 deletions cmd/collectors/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,14 @@ import (

var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`)

func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string) ([]*matrix.Matrix, *util.Metadata, error) {
func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) {
var err error

if volumesMap == nil {
logger.Info("volumes config data not found")
return nil, nil, nil
}

fgAggrMap := make(map[string]*set.Set)
flexgroupAggrsMap := make(map[string]*set.Set)

Expand All @@ -34,7 +39,10 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
cache.UUID += ".Volume"

for _, i := range data.GetInstances() {
if match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
volName := i.GetLabel("volume")
switch volumesMap[volName] {
case "flexgroup_constituent":
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]
if cache.GetInstance(key) == nil {
fg, _ := cache.NewInstance(key)
Expand Down Expand Up @@ -62,9 +70,9 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
i.SetLabel(style, "flexgroup_constituent")
i.SetExportable(includeConstituents)
} else {
case "flexvol":
i.SetLabel(style, "flexvol")
key := i.GetLabel("svm") + "." + i.GetLabel("volume")
key := i.GetLabel("svm") + "." + volName
flexvolInstance, err := volumeAggrmetric.NewInstance(key)
if err != nil {
logger.Error("Failed to create new instance", slogx.Err(err), slog.String("key", key))
Expand All @@ -82,10 +90,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string

recordFGFalse := make(map[string]*set.Set)
for _, i := range data.GetInstances() {
match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume"))
if len(match) != 3 {
volName := i.GetLabel("volume")
if volumesMap[volName] != "flexgroup_constituent" {
continue
}
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]

flexgroupInstance := volumeAggrmetric.GetInstance(key)
Expand Down
86 changes: 83 additions & 3 deletions cmd/collectors/zapiperf/plugins/volume/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,31 @@ package volume
import (
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
)

const batchSize = "500"

type Volume struct {
*plugin.AbstractPlugin
currentVal int
styleType string
includeConstituents bool
client *zapi.Client
volumesMap map[string]string // volume-name -> volume-extended-style map
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Volume{AbstractPlugin: p}
}

func (v *Volume) Init() error {

var err error
if err := v.InitAbc(); err != nil {
return err
}
Expand All @@ -33,15 +42,86 @@ func (v *Volume) Init() error {
v.styleType = "type"
}

if v.Options.IsTest {
return nil
}

v.volumesMap = make(map[string]string)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
return nil

if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams), v.Auth); err != nil {
v.SLogger.Error("connecting", slog.Any("err", err))
return err
}
return v.client.Init(5)
}

func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
opsKeyPrefix := "temp_"
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0
// Clean volumesMap map
clear(v.volumesMap)
v.volumesMap = v.fetchVolumes()
}

v.currentVal++
return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
}

func (v *Volume) fetchVolumes() map[string]string {
var (
result *node.Node
volumes []*node.Node
volumesMap map[string]string
)

volumesMap = make(map[string]string)
query := "volume-get-iter"
tag := "initial"
request := node.NewXMLS(query)
request.NewChildS("max-records", batchSize)
desired := node.NewXMLS("desired-attributes")
volumeAttributes := node.NewXMLS("desired-attributes")
volumeIDAttributes := node.NewXMLS("volume-id-attributes")
volumeIDAttributes.NewChildS("name", "")
volumeIDAttributes.NewChildS("style-extended", "")
volumeAttributes.AddChild(volumeIDAttributes)
desired.AddChild(volumeAttributes)
request.AddChild(desired)

for {
responseData, err := v.client.InvokeBatchRequest(request, tag, "")
if err != nil {
return nil
}
result = responseData.Result
tag = responseData.Tag

if result == nil {
break
}

if x := result.GetChildS("attributes-list"); x != nil {
volumes = x.GetChildren()
}
if len(volumes) == 0 {
return nil
}

for _, volume := range volumes {
styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
volumesMap[name] = styleExtended
}
}

return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix)
return volumesMap
}
15 changes: 9 additions & 6 deletions cmd/poller/plugin/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,15 +167,18 @@ func (p *AbstractPlugin) SetPluginInterval() int {
}

func GetInterval(param *node.Node, defaultInterval time.Duration) float64 {
schedule := param.GetChildS("schedule")
if schedule != nil {
dataInterval := schedule.GetChildContentS("data")
if dataInterval != "" {
if durationVal, err := time.ParseDuration(dataInterval); err == nil {
return durationVal.Seconds()
if param != nil {
schedule := param.GetChildS("schedule")
if schedule != nil {
dataInterval := schedule.GetChildContentS("data")
if dataInterval != "" {
if durationVal, err := time.ParseDuration(dataInterval); err == nil {
return durationVal.Seconds()
}
}
}
}

return defaultInterval.Seconds()
}

Expand Down

0 comments on commit 391b6b7

Please sign in to comment.