Skip to content

Commit

Permalink
feat: handled user/group quota in HistoricalLabels (#3060)
Browse files Browse the repository at this point in the history 
* feat: handled user/group quota in HistoricalLabels for zapi/rest
  • Loading branch information
@Hardikl
Hardikl authored Jul 23, 2024
1 parent 8cccb5f commit c4dc19d
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 232 deletions.
155 changes: 43 additions & 112 deletions cmd/collectors/rest/plugins/qtree/qtree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"github.com/tidwall/gjson"
"strconv"
"strings"
"time"
)
Expand Down Expand Up @@ -106,7 +105,6 @@ func (q *Qtree) Init() error {
}

instanceKeys.NewChildS("", "type")
instanceKeys.NewChildS("", "index")
instanceKeys.NewChildS("", "unit")

q.data.SetExportOptions(exportOptions)
Expand Down Expand Up @@ -175,15 +173,9 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.
}

quotaCount := 0
cluster := data.GetGlobalLabels()["cluster"]

if q.historicalLabels {
// In 22.05, populate metrics with qtree prefix and old labels
err = q.handlingHistoricalMetrics(result, data, cluster, &quotaCount, &numMetrics)
} else {
// Populate metrics with quota prefix and current labels
err = q.handlingQuotaMetrics(result, &quotaCount, &numMetrics)
}
// Populate metrics with quota prefix
err = q.handlingQuotaMetrics(result, data, &quotaCount, &numMetrics)

if err != nil {
return nil, nil, err
Expand All @@ -207,11 +199,11 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.
return []*matrix.Matrix{q.data}, q.client.Metadata, nil
}

func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Matrix, cluster string, quotaIndex *int, numMetrics *int) error {
func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, data *matrix.Matrix, quotaCount *int, numMetrics *int) error {
for _, quota := range result {
var tree string
var quotaInstance *matrix.Instance
var err error
var qtreeInstance *matrix.Instance
var value float64

if !quota.IsObject() {
q.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping")
Expand All @@ -221,115 +213,39 @@ func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Ma
if quota.Get("qtree.name").Exists() {
tree = quota.Get("qtree.name").String()
}
quotaType := quota.Get("type").String()
volume := quota.Get("volume.name").String()
vserver := quota.Get("svm.name").String()
qIndex := quota.Get("index").String()
quotaType := quota.Get("type").String()

// If quota-type is not a tree, then skip
if quotaType != "tree" {
continue
}

// Ex. InstanceKey: vserver1vol1qtree15989279
quotaInstanceKey := vserver + volume + tree + qIndex
uName := quota.Get("users.0.name").String()
uid := quota.Get("users.0.id").String()
group := quota.Get("group.name").String()
*quotaCount++

if quotaInstance = q.data.GetInstance(quotaInstanceKey); quotaInstance == nil {
if quotaInstance, err = q.data.NewInstance(quotaInstanceKey); err != nil {
q.Logger.Error().Stack().Err(err).Str("quotaInstanceKey", quotaInstanceKey).Msg("Failed to create quota instance")
return err
// In 22.05, populate metrics with qtree labels
if q.historicalLabels {
// qtree instancekey would be qtree, svm and volume(sorted keys)
qtreeInstance = data.GetInstance(tree + vserver + volume)
if qtreeInstance == nil {
q.Logger.Warn().
Str("tree", tree).
Str("volume", volume).
Str("vserver", vserver).
Msg("No instance matching tree.vserver.volume")
continue
}
}

// qtree instancekey would be qtree, svm and volume(sorted keys)
qtreeInstance := data.GetInstance(tree + vserver + volume)
if qtreeInstance == nil {
q.Logger.Warn().
Str("tree", tree).
Str("volume", volume).
Str("vserver", vserver).
Msg("No instance matching tree.vserver.volume")
continue
}

if !qtreeInstance.IsExportable() {
continue
}

for _, label := range q.data.GetExportOptions().GetChildS("instance_keys").GetAllChildContentS() {
if value := qtreeInstance.GetLabel(label); value != "" {
quotaInstance.SetLabel(label, value)
if !qtreeInstance.IsExportable() {
continue
}
}

// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
quotaInstance.SetLabel("svm", vserver)
quotaInstance.SetLabel("index", cluster+"_"+strconv.Itoa(*quotaIndex))

// If the Qtree is the volume itself, then qtree label is empty, so copy the volume name to qtree.
if tree == "" {
quotaInstance.SetLabel("qtree", volume)
}

*quotaIndex++
for attribute, m := range q.data.GetMetrics() {
value := 0.0

if attrValue := quota.Get(attribute); attrValue.Exists() {
// space limits are in bytes, converted to kilobytes
if attribute == "space.hard_limit" || attribute == "space.soft_limit" {
value = attrValue.Float() / 1024
quotaInstance.SetLabel("unit", "Kbyte")
if attribute == "space.soft_limit" {
t := q.data.GetMetric("threshold")
if err = t.SetValueFloat64(quotaInstance, value); err != nil {
q.Logger.Error().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value")
} else {
*numMetrics++
}
}
} else {
value = attrValue.Float()
}
}

// populate numeric data
if err = m.SetValueFloat64(quotaInstance, value); err != nil {
q.Logger.Error().Stack().Err(err).Str("attribute", attribute).Float64("value", value).Msg("Failed to parse value")
// In 22.05, populate metrics value with 0
if q.historicalLabels {
value = 0.0
} else {
*numMetrics++
// set -1 for unlimited
value = -1.0
}
}
}
return nil
}

func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, numMetrics *int) error {
for _, quota := range result {
var tree string

if !quota.IsObject() {
q.Logger.Error().Str("type", quota.Type.String()).Msg("Quota is not an object, skipping")
return errs.New(errs.ErrNoInstance, "quota is not an object")
}

if quota.Get("qtree.name").Exists() {
tree = quota.Get("qtree.name").String()
}
quotaType := quota.Get("type").String()
volume := quota.Get("volume.name").String()
vserver := quota.Get("svm.name").String()
uName := quota.Get("users.0.name").String()
uid := quota.Get("users.0.id").String()
group := quota.Get("group.name").String()
*quotaCount++

for attribute, m := range q.data.GetMetrics() {
// set -1 for unlimited
value := -1.0

quotaInstanceKey := vserver + "." + volume + "." + tree + "." + group + "." + uName + "." + attribute

Expand All @@ -338,12 +254,26 @@ func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, num
q.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err)
return err
}

// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
quotaInstance.SetLabel("svm", vserver)

// In 22.05, populate metrics with qtree labels
if q.historicalLabels {
for _, label := range q.data.GetExportOptions().GetChildS("instance_keys").GetAllChildContentS() {
if val := qtreeInstance.GetLabel(label); val != "" {
quotaInstance.SetLabel(label, val)
}
}
// If the Qtree is the volume itself, then qtree label is empty, so copy the volume name to qtree.
if tree == "" {
quotaInstance.SetLabel("qtree", volume)
}
}

if quotaType == "user" {
if uName != "" {
quotaInstance.SetLabel("user", uName)
Expand All @@ -357,6 +287,7 @@ func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, quotaCount *int, num
quotaInstance.SetLabel("group", uid)
}
}

if attrValue := quota.Get(attribute); attrValue.Exists() {
// space limits are in bytes, converted to kilobytes to match ZAPI
if attribute == "space.hard_limit" || attribute == "space.soft_limit" || attribute == "space.used.total" {
Expand Down
46 changes: 38 additions & 8 deletions cmd/collectors/rest/plugins/qtree/qtree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/tidwall/gjson"
"os"
"testing"
Expand All @@ -19,27 +20,56 @@ func NewQtree() *Qtree {
}

func TestHandlingQuotaMetrics(t *testing.T) {
q := NewQtree()

jsonResponse, err := os.ReadFile("testdata/quota.json")
if err != nil {
t.Fatalf("Failed to read JSON response from file: %v", err)
}

result := gjson.Get(string(jsonResponse), "records").Array()

// Case 1: with historicalLabels = false
q1 := NewQtree()
q1.historicalLabels = false
testLabels(t, q1, result, nil, "astra_300.trident_qtree_pool_trident_TIXRBILLKA.trident_pvc_19913841_a29f_4a54_8bc0_a3c1c4155826...space.hard_limit", 3, 6, 5)

// Case 2: with historicalLabels = true
q2 := NewQtree()
data := matrix.New(q2.Parent+".Qtree", "qtree", "qtree")
qtreeInstance, _ := data.NewInstance("" + "abcde" + "abcd_root")
qtreeInstance.SetLabel("export_policy", "default")
qtreeInstance.SetLabel("oplocks", "enabled")
qtreeInstance.SetLabel("security_style", "unix")
qtreeInstance.SetLabel("status", "normal")

exportOptions := node.NewS("export_options")
instanceKeys := exportOptions.NewChildS("instance_keys", "")
// apply all instance keys, instance labels from qtree.yaml to all quota metrics
keys := []string{"export_policy", "oplocks", "security_style", "status"}
for _, key := range keys {
instanceKeys.NewChildS("", key)
}
q2.data.SetExportOptions(exportOptions)
q2.historicalLabels = true
testLabels(t, q2, result, data, "abcde.abcd_root...root.space.used.total", 3, 4, 10)
}

func testLabels(t *testing.T, q *Qtree, quotas []gjson.Result, data *matrix.Matrix, quotaInstanceKey string, expectedQuotaCount int, expectedQuotaMetricCount int, expectedQuotaLabels int) {
quotaCount := 0
numMetrics := 0

err = q.handlingQuotaMetrics(result, &quotaCount, &numMetrics)
err := q.handlingQuotaMetrics(quotas, data, &quotaCount, &numMetrics)
if err != nil {
t.Errorf("handlingQuotaMetrics returned an error: %v", err)
}

if quotaCount != 3 {
t.Errorf("quotaCount = %d; want 3", quotaCount)
if quotaCount != expectedQuotaCount {
t.Errorf("quotaCount = %d; want %d", quotaCount, expectedQuotaCount)
}
if numMetrics != 6 {
t.Errorf("numMetrics = %d; want 6", numMetrics)
if numMetrics != expectedQuotaMetricCount {
t.Errorf("numMetrics = %d; want %d", numMetrics, expectedQuotaMetricCount)
}

quotaInstance := q.data.GetInstance(quotaInstanceKey)
if len(quotaInstance.GetLabels()) != expectedQuotaLabels {
t.Errorf("labels = %d; want %d", len(quotaInstance.GetLabels()), expectedQuotaLabels)
}
}
Loading

0 comments on commit c4dc19d

Please sign in to comment.