Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: handled flexgroup based on volume config call #3199

Merged
merged 4 commits into from
Oct 9, 2024
Merged

fix: handled flexgroup based on volume config call #3199

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
805f750
fix: handled flexgroup based on volume config call
Hardikl Oct 7, 2024
eb7dd26
fix: handled review comments
Hardikl Oct 8, 2024
2859e46
fix: handled review comments
Hardikl Oct 8, 2024
05d0379
fix: handled review comments
Hardikl Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 53 additions & 3 deletions cmd/collectors/restperf/plugins/volume/volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,27 @@ package volume
import (
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
"time"
)

type Volume struct {
*plugin.AbstractPlugin
styleType string
includeConstituents bool
client *rest.Client
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Volume{AbstractPlugin: p}
}

func (v *Volume) Init() error {

var err error
if err := v.InitAbc(); err != nil {
return err
}
Expand All @@ -31,13 +36,58 @@ func (v *Volume) Init() error {

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
return nil

if v.Options.IsTest {
v.client = &rest.Client{Metadata: &util.Metadata{}}
return nil
}

timeout, _ := time.ParseDuration(rest.DefaultTimeout)
if v.client, err = rest.New(conf.ZapiPoller(v.ParentParams), timeout, v.Auth); err != nil {
v.SLogger.Error("connecting", slog.Any("err", err))
return err
}

return v.client.Init(5)
}

func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
opsKeyPrefix := "temp_"
volumesMap := v.fetchVolumes()

return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, volumesMap)
}

func (v *Volume) fetchVolumes() map[string]string {
volumesMap := make(map[string]string)
query := "api/storage/volumes"

href := rest.NewHrefBuilder().
APIPath(query).
Fields([]string{"volume", "volume_style_extended"}).
Hardikl marked this conversation as resolved.
Show resolved Hide resolved
Build()

records, err := rest.FetchAll(v.client, href)
if err != nil {
v.SLogger.Error("Failed to fetch data", slog.Any("err", err), slog.String("href", href))
return nil
}

if len(records) == 0 {
return nil
}

for _, volume := range records {
if !volume.IsObject() {
v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String()))
continue
}
styleExtended := volume.Get("volume_style_extended").String()
name := volume.Get("volume").String()
volumesMap[name] = styleExtended
}

return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix)
return volumesMap
}
27 changes: 20 additions & 7 deletions cmd/collectors/restperf/plugins/volume/volume_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,29 @@
package volume_test

import (
"github.com/netapp/harvest/v2/cmd/collectors"
volume2 "github.com/netapp/harvest/v2/cmd/collectors/restperf/plugins/volume"
"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/volume"
"github.com/netapp/harvest/v2/cmd/poller/options"
"log/slog"
"strconv"
"testing"

"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
)

const OpsKeyPrefix = "temp_"
const StyleType = "style"
const PollerName = "test"

// Common test logic for RestPerf/ZapiPerf Volume plugin
func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) {
params := node.NewS("Volume")
params.NewChildS("include_constituents", includeConstituents)
v := createVolume(params)
volumesMap := make(map[string]string)

// Initialize the plugin
if err := v.Init(); err != nil {
Expand All @@ -28,22 +36,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
instance1.SetLabel("volume", "RahulTest__0001")
instance1.SetLabel("svm", "svm1")
instance1.SetLabel("aggr", "aggr1")
volumesMap["RahulTest__0001"] = "flexgroup_constituent"

instance2, _ := data.NewInstance("RahulTest__0002")
instance2.SetLabel("volume", "RahulTest__0002")
instance2.SetLabel("svm", "svm1")
instance2.SetLabel("aggr", "aggr2")
volumesMap["RahulTest__0002"] = "flexgroup_constituent"

instance3, _ := data.NewInstance("RahulTest__0003")
instance3.SetLabel("volume", "RahulTest__0003")
instance3.SetLabel("svm", "svm1")
instance3.SetLabel("aggr", "aggr3")
volumesMap["RahulTest__0003"] = "flexgroup_constituent"

// Create a simple volume instance
simpleInstance, _ := data.NewInstance("SimpleVolume")
simpleInstance.SetLabel("volume", "SimpleVolume")
simpleInstance.SetLabel("svm", "svm1")
simpleInstance.SetLabel("aggr", "aggr4")
volumesMap["SimpleVolume"] = "flexvol"

// Create latency and ops metrics
latencyMetric, _ := data.NewMetricFloat64("read_latency")
Expand Down Expand Up @@ -73,12 +85,9 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
_ = latencyMetric.SetValueFloat64(simpleInstance, 50)
_ = opsMetric.SetValueFloat64(simpleInstance, 5)

dataMap := map[string]*matrix.Matrix{
"volume": data,
}

// Run the plugin
output, _, err := v.Run(dataMap)
boolValue, _ := strconv.ParseBool(includeConstituents)
output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap)
if err != nil {
t.Fatalf("Run method failed: %v", err)
}
Expand Down Expand Up @@ -230,13 +239,17 @@ func TestRunForAllImplementations(t *testing.T) {
}

func createRestVolume(params *node.Node) plugin.Plugin {
v := &volume2.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)}
opts := options.New(options.WithConfPath("testdata/conf"))
opts.IsTest = true
v := &volume2.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)}
v.SLogger = slog.Default()
return v
}

func createZapiVolume(params *node.Node) plugin.Plugin {
v := &volume.Volume{AbstractPlugin: plugin.New("volume", nil, params, nil, "volume", nil)}
opts := options.New(options.WithConfPath("testdata/conf"))
opts.IsTest = true
v := &volume.Volume{AbstractPlugin: plugin.New("volume", opts, params, nil, "volume", nil)}
v.SLogger = slog.Default()
return v
}
16 changes: 10 additions & 6 deletions cmd/collectors/volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (

var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`)

func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string) ([]*matrix.Matrix, *util.Metadata, error) {
func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) {
var err error

fgAggrMap := make(map[string]*set.Set)
Expand All @@ -33,7 +33,10 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
cache.UUID += ".Volume"

for _, i := range data.GetInstances() {
if match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume")); len(match) == 3 {
volName := i.GetLabel("volume")
switch volumesMap[volName] {
Hardikl marked this conversation as resolved.
Show resolved Hide resolved
case "flexgroup_constituent":
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]
if cache.GetInstance(key) == nil {
fg, _ := cache.NewInstance(key)
Expand Down Expand Up @@ -61,9 +64,9 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
flexgroupAggrsMap[key].Add(i.GetLabel("aggr"))
i.SetLabel(style, "flexgroup_constituent")
i.SetExportable(includeConstituents)
} else {
case "flexvol":
i.SetLabel(style, "flexvol")
key := i.GetLabel("svm") + "." + i.GetLabel("volume")
key := i.GetLabel("svm") + "." + volName
flexvolInstance, err := volumeAggrmetric.NewInstance(key)
if err != nil {
logger.Error("Failed to create new instance", slog.Any("err", err), slog.String("key", key))
Expand All @@ -81,10 +84,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string

recordFGFalse := make(map[string]*set.Set)
for _, i := range data.GetInstances() {
match := flexgroupRegex.FindStringSubmatch(i.GetLabel("volume"))
if len(match) != 3 {
volName := i.GetLabel("volume")
if volumesMap[volName] != "flexgroup_constituent" {
continue
}
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]

flexgroupInstance := volumeAggrmetric.GetInstance(key)
Expand Down
73 changes: 70 additions & 3 deletions cmd/collectors/zapiperf/plugins/volume/volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,29 @@ package volume
import (
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
)

const batchSize = "500"

type Volume struct {
*plugin.AbstractPlugin
styleType string
includeConstituents bool
client *zapi.Client
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Volume{AbstractPlugin: p}
}

func (v *Volume) Init() error {

var err error
if err := v.InitAbc(); err != nil {
return err
}
Expand All @@ -33,15 +40,75 @@ func (v *Volume) Init() error {
v.styleType = "type"
}

if v.Options.IsTest {
return nil
}

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
return nil

if v.client, err = zapi.New(conf.ZapiPoller(v.ParentParams), v.Auth); err != nil {
v.SLogger.Error("connecting", slog.Any("err", err))
return err
}
return v.client.Init(5)
}

func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
opsKeyPrefix := "temp_"
volumesMap := v.fetchVolumes()

return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, volumesMap)
}

func (v *Volume) fetchVolumes() map[string]string {
var (
result *node.Node
volumes []*node.Node
volumesMap map[string]string
)

volumesMap = make(map[string]string)
query := "volume-get-iter"
tag := "initial"
request := node.NewXMLS(query)
request.NewChildS("max-records", batchSize)
desired := node.NewXMLS("desired-attributes")
volumeAttributes := node.NewXMLS("desired-attributes")
volumeIDAttributes := node.NewXMLS("volume-id-attributes")
volumeIDAttributes.NewChildS("name", "")
volumeIDAttributes.NewChildS("style-extended", "")
volumeAttributes.AddChild(volumeIDAttributes)
desired.AddChild(volumeAttributes)
request.AddChild(desired)

for {
responseData, err := v.client.InvokeBatchRequest(request, tag, "")
if err != nil {
return nil
}
result = responseData.Result
tag = responseData.Tag

if result == nil {
break
}

if x := result.GetChildS("attributes-list"); x != nil {
volumes = x.GetChildren()
}
if len(volumes) == 0 {
return nil
}

for _, volume := range volumes {
styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
volumesMap[name] = styleExtended
}
}

return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix)
return volumesMap
}