Skip to content

Commit

Permalink
feat: update volume dashboard with tags
Browse files Browse the repository at this point in the history
  • Loading branch information
Hardikl committed Nov 12, 2024
1 parent e1e4c17 commit f3dd31f
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 84 deletions.
30 changes: 23 additions & 7 deletions cmd/collectors/restperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
"sort"
"strings"
"time"
)

Expand All @@ -17,7 +19,7 @@ type Volume struct {
styleType string
includeConstituents bool
client *rest.Client
volumesMap map[string]string // volume-name -> volume-extended-style map
volumesMap map[string]collectors.VolumeData // volume-name -> {volume-extended-style, tags} map
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
Expand All @@ -36,7 +38,7 @@ func (v *Volume) Init() error {
v.styleType = "type"
}

v.volumesMap = make(map[string]string)
v.volumesMap = make(map[string]collectors.VolumeData)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()
Expand All @@ -61,6 +63,7 @@ func (v *Volume) Init() error {
func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
tags := "tags"
opsKeyPrefix := "temp_"
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0
Expand All @@ -70,16 +73,16 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
}

v.currentVal++
return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
return collectors.ProcessFlexGroupData(v.SLogger, data, style, tags, v.includeConstituents, opsKeyPrefix, v.volumesMap)
}

func (v *Volume) fetchVolumes() map[string]string {
volumesMap := make(map[string]string)
func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
volumesMap := make(map[string]collectors.VolumeData)
query := "api/private/cli/volume"

href := rest.NewHrefBuilder().
APIPath(query).
Fields([]string{"volume", "volume_style_extended"}).
Fields([]string{"volume", "volume_style_extended", "tags"}).
Filter([]string{"is_constituent=*"}).
MaxRecords(collectors.DefaultBatchSize).
Build()
Expand All @@ -95,13 +98,26 @@ func (v *Volume) fetchVolumes() map[string]string {
}

for _, volume := range records {
var tags string
if !volume.IsObject() {
v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String()))
continue
}
styleExtended := volume.Get("volume_style_extended").String()
name := volume.Get("volume").String()
volumesMap[name] = styleExtended
tagData := volume.Get("tags")
if tagData.IsArray() {
var labelArray []string
for _, r := range tagData.Array() {
labelString := r.String()
labelArray = append(labelArray, labelString)
}
sort.Strings(labelArray)
tags = strings.Join(labelArray, ",")
} else {
tags = tagData.String()
}
volumesMap[name] = collectors.VolumeData{Style: styleExtended, Tags: tags}
}

return volumesMap
Expand Down
13 changes: 7 additions & 6 deletions cmd/collectors/restperf/plugins/volume/volume_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@ import (

const OpsKeyPrefix = "temp_"
const StyleType = "style"
const Tags = "tags"
const PollerName = "test"

// Common test logic for RestPerf/ZapiPerf Volume plugin
func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) {
params := node.NewS("Volume")
params.NewChildS("include_constituents", includeConstituents)
v := createVolume(params)
volumesMap := make(map[string]string)
volumesMap := make(map[string]collectors.VolumeData)

// Initialize the plugin
if err := v.Init(); err != nil {
Expand All @@ -36,26 +37,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
instance1.SetLabel("volume", "RahulTest__0001")
instance1.SetLabel("svm", "svm1")
instance1.SetLabel("aggr", "aggr1")
volumesMap["RahulTest__0001"] = "flexgroup_constituent"
volumesMap["RahulTest__0001"] = collectors.VolumeData{Style: "flexgroup_constituent"}

instance2, _ := data.NewInstance("RahulTest__0002")
instance2.SetLabel("volume", "RahulTest__0002")
instance2.SetLabel("svm", "svm1")
instance2.SetLabel("aggr", "aggr2")
volumesMap["RahulTest__0002"] = "flexgroup_constituent"
volumesMap["RahulTest__0002"] = collectors.VolumeData{Style: "flexgroup_constituent"}

instance3, _ := data.NewInstance("RahulTest__0003")
instance3.SetLabel("volume", "RahulTest__0003")
instance3.SetLabel("svm", "svm1")
instance3.SetLabel("aggr", "aggr3")
volumesMap["RahulTest__0003"] = "flexgroup_constituent"
volumesMap["RahulTest__0003"] = collectors.VolumeData{Style: "flexgroup_constituent"}

// Create a simple volume instance
simpleInstance, _ := data.NewInstance("SimpleVolume")
simpleInstance.SetLabel("volume", "SimpleVolume")
simpleInstance.SetLabel("svm", "svm1")
simpleInstance.SetLabel("aggr", "aggr4")
volumesMap["SimpleVolume"] = "flexvol"
volumesMap["SimpleVolume"] = collectors.VolumeData{Style: "flexvol"}

// Create latency and ops metrics
latencyMetric, _ := data.NewMetricFloat64("read_latency")
Expand Down Expand Up @@ -87,7 +88,7 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu

// Run the plugin
boolValue, _ := strconv.ParseBool(includeConstituents)
output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap)
output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, Tags, boolValue, OpsKeyPrefix, volumesMap)
if err != nil {
t.Fatalf("Run method failed: %v", err)
}
Expand Down
13 changes: 10 additions & 3 deletions cmd/collectors/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ import (

var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`)

func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) {
type VolumeData struct {
Style string
Tags string
}

func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, tags string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]VolumeData) ([]*matrix.Matrix, *util.Metadata, error) {
var err error

if volumesMap == nil {
Expand All @@ -40,7 +45,9 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string

for _, i := range data.GetInstances() {
volName := i.GetLabel("volume")
switch volumesMap[volName] {
volData := volumesMap[volName]
i.SetLabel(tags, volData.Tags)
switch volData.Style {
case "flexgroup_constituent":
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]
Expand Down Expand Up @@ -91,7 +98,7 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
recordFGFalse := make(map[string]*set.Set)
for _, i := range data.GetInstances() {
volName := i.GetLabel("volume")
if volumesMap[volName] != "flexgroup_constituent" {
if volumesMap[volName].Style != "flexgroup_constituent" {
continue
}
match := flexgroupRegex.FindStringSubmatch(volName)
Expand Down
15 changes: 8 additions & 7 deletions cmd/collectors/zapiperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type Volume struct {
styleType string
includeConstituents bool
client *zapi.Client
volumesMap map[string]string // volume-name -> volume-extended-style map
volumesMap map[string]collectors.VolumeData // volume-name -> {volume-extended-style, tags} map
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
Expand All @@ -46,7 +46,7 @@ func (v *Volume) Init() error {
return nil
}

v.volumesMap = make(map[string]string)
v.volumesMap = make(map[string]collectors.VolumeData)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()
Expand All @@ -64,6 +64,7 @@ func (v *Volume) Init() error {
func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
tags := "tags"
opsKeyPrefix := "temp_"
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0
Expand All @@ -73,17 +74,17 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
}

v.currentVal++
return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
return collectors.ProcessFlexGroupData(v.SLogger, data, style, tags, v.includeConstituents, opsKeyPrefix, v.volumesMap)
}

func (v *Volume) fetchVolumes() map[string]string {
func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
var (
result *node.Node
volumes []*node.Node
volumesMap map[string]string
volumesMap map[string]collectors.VolumeData
)

volumesMap = make(map[string]string)
volumesMap = make(map[string]collectors.VolumeData)
query := "volume-get-iter"
tag := "initial"
request := node.NewXMLS(query)
Expand Down Expand Up @@ -119,7 +120,7 @@ func (v *Volume) fetchVolumes() map[string]string {
for _, volume := range volumes {
styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
volumesMap[name] = styleExtended
volumesMap[name] = collectors.VolumeData{Style: styleExtended, Tags: ""}
}
}

Expand Down
2 changes: 2 additions & 0 deletions conf/rest/9.12.0/volume.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ counters:
- ^snaplock_type => snaplock_type
- ^snapshot_policy => snapshot_policy
- ^state => state
- ^tags => tags
- ^type => type
- ^uuid => uuid
- ^volume_style_extended => style
Expand Down Expand Up @@ -139,4 +140,5 @@ export_options:
- snapshot_policy
- state
- svm_root
- tags
- type
1 change: 1 addition & 0 deletions conf/restperf/9.12.0/volume.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,5 @@ export_options:
- node
- style
- svm
- tags
- volume
Loading

0 comments on commit f3dd31f

Please sign in to comment.