Skip to content

Commit

Permalink
fix: handled non exported qtrees in template (#3105)
Browse files Browse the repository at this point in the history 
* fix: handled non exported qtrees in template

* fix: use struct

* fix: handled non exported qtrees in template (#3107)

* fix: handled non exported qtrees in template

---------

Co-authored-by: Chris Grindstaff <[email protected]>
  • Loading branch information
@Hardikl @cgrinds
Hardikl and cgrinds authored Aug 14, 2024
1 parent cae9771 commit f95fef1
Show file tree
Hide file tree
Showing 8 changed files with 258 additions and 166 deletions.
24 changes: 12 additions & 12 deletions cmd/collectors/zapi/plugins/qtree/qtree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ type Qtree struct {
client *zapi.Client
query string
quotaType []string
historicalLabels bool // supports labels, metrics for 22.05
qtreeMetrics bool // supports quota metrics with qtree prefix
historicalLabels bool // supports labels, metrics for 22.05
qtreeMetrics bool // supports quota metrics with qtree prefix
testFilePath string // Used only from unit test
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
Expand All @@ -43,6 +44,10 @@ func (q *Qtree) Init() error {
return err
}

if q.Options.IsTest {
return nil
}

if q.client, err = zapi.New(conf.ZapiPoller(q.ParentParams), q.Auth); err != nil {
q.Logger.Error().Stack().Err(err).Msg("connecting")
return err
Expand Down Expand Up @@ -141,7 +146,6 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.
request, response *node.Node
quotas []*node.Node
ad, pd time.Duration // Request/API time, Parse time, Fetch time
err error
numMetrics int
)

Expand Down Expand Up @@ -171,19 +175,15 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.
tag := "initial"
quotaIndex := 0

// In 22.05, all qtrees were exported
if q.historicalLabels {
for _, qtreeInstance := range data.GetInstances() {
qtreeInstance.SetExportable(true)
}
}

for {
response, tag, ad, pd, err = q.client.InvokeBatchWithTimers(request, tag)

responseData, err := q.client.InvokeBatchRequest(request, tag, q.testFilePath)
if err != nil {
return nil, nil, err
}
response = responseData.Result
tag = responseData.Tag
ad = responseData.Rd
pd = responseData.Pd

if response == nil {
break
Expand Down
191 changes: 133 additions & 58 deletions cmd/collectors/zapi/plugins/qtree/qtree_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,77 +1,151 @@
package qtree

import (
"github.com/netapp/harvest/v2/cmd/poller/options"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
client "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree"
"github.com/netapp/harvest/v2/pkg/tree/node"
"os"
"testing"
)

func NewQtree() *Qtree {
q := &Qtree{AbstractPlugin: plugin.New("qtree", nil, nil, nil, "qtree", nil)}
func NewQtree(historicalLabels bool) plugin.Plugin {
params := node.NewS("Qtree")
pp := node.NewS("QtreeParent")
pp.NewChildS("poller_name", "test")
pp.NewChildS("addr", "1.2.3.4")
o := options.Options{IsTest: true}
q := &Qtree{AbstractPlugin: plugin.New("qtree", &o, params, pp, "qtree", nil)}
q.Logger = logging.Get()
q.historicalLabels = historicalLabels
q.data = matrix.New(q.Parent+".Qtree", "quota", "quota")
q.client = client.NewTestClient()
q.testFilePath = "testdata/quotas.xml"
exportOptions := node.NewS("export_options")
instanceKeys := exportOptions.NewChildS("instance_keys", "")
// apply all instance keys, instance labels from qtree.yaml to all quota metrics
keys := []string{"export_policy", "oplocks", "security_style", "status"}
for _, key := range keys {
instanceKeys.NewChildS("", key)
}
q.data.SetExportOptions(exportOptions)
_, _ = q.data.NewMetricFloat64("disk-limit", "disk_limit")
_, _ = q.data.NewMetricFloat64("disk-used", "disk_used")
return q
}

func TestHandlingQuotaMetrics(t *testing.T) {
xmlResponse, err := os.ReadFile("testdata/quotas.xml")
if err != nil {
t.Fatalf("Failed to read XML response from file: %v", err)
func TestRunForAllImplementations(t *testing.T) {
testCases := []struct {
name string
createQtree func(historicalLabels bool) plugin.Plugin
historicalLabels bool
expectedQuotaCount int
expectedQtreeCount int
quotaInstanceKey string
expectedQuotaLabels int
withNonExportedQtree bool
}{
{
// Case 1: with historicalLabels = false, all qtrees were exported, total 4 quotas: 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// 1 empty tree quota will be skipped and 5 labels[qtree, svm, type, unit, volume] would be exported for tree quota.
name: "historicalLabels=false,withNonExportedQtree=false",
createQtree: NewQtree,
historicalLabels: false,
withNonExportedQtree: false,
expectedQuotaCount: 6, // Only 3 quotas each with 2 metrics
expectedQtreeCount: 3, // All 3 qtrees
quotaInstanceKey: "astra_300.trident_qtree_pool_trident_TIXRBILLKA.trident_pvc_2a6d71d9_1c78_4e9a_84a2_59d316adfae9..disk-limit.tree",
expectedQuotaLabels: 5,
},
{
// Case 2: with historicalLabels = false, 2 qtrees were not exported, total 4 quotas: 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// 1 empty tree quota will be skipped, 2 qtree will be skipped and 6 labels[qtree, svm, type, unit, user, volume] would be exported for user/group quota.
name: "historicalLabels=false,withNonExportedQtree=true",
createQtree: NewQtree,
historicalLabels: false,
withNonExportedQtree: true,
expectedQuotaCount: 6, // Only 3 quotas each with 2 metrics
expectedQtreeCount: 1, // Only 1 qtrees, because 2 qtree is not exported
quotaInstanceKey: "abcde.abcd_root..root.disk-used.user",
expectedQuotaLabels: 6,
},
{
// Case 3: with historicalLabels = true, all qtrees were exported, total 4 quotas: 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// all quotas with 9 labels [export_policy, oplocks, qtree, security_style, status, svm, type, unit, volume] would be exported.
name: "historicalLabels=true,withNonExportedQtree=false",
createQtree: NewQtree,
historicalLabels: true,
withNonExportedQtree: false,
expectedQuotaCount: 8, // All 4 quotas each with 2 metrics
expectedQtreeCount: 3, // All 3 qtrees
quotaInstanceKey: "svm1.volume1...disk-used.tree",
expectedQuotaLabels: 9,
},
{
// Case 4: with historicalLabels = true, 2 qtrees were not exported, total 4 quotas: 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// 2 qtree will be skipped, 3 quotas will be skipped and specified quota instancekey hasn't been exported because that qtree is not being exported.
name: "historicalLabels=true,withNonExportedQtree=true",
createQtree: NewQtree,
historicalLabels: true,
withNonExportedQtree: true,
expectedQuotaCount: 2, // Only 1 quotas each with 2 metrics
expectedQtreeCount: 1, // Only 1 qtrees, because 2 qtree is not exported
quotaInstanceKey: "svm1.volume1...disk-used.tree",
expectedQuotaLabels: 0,
},
}

root, err := tree.LoadXML(xmlResponse)
if err != nil {
t.Fatalf("Failed to parse XML response: %v", err)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
runQtreeTest(t, tc.createQtree, tc.historicalLabels, tc.expectedQuotaCount, tc.expectedQtreeCount, tc.quotaInstanceKey, tc.expectedQuotaLabels, tc.withNonExportedQtree)
})
}
}

response := root.GetChildS("results")
if response == nil {
t.Fatalf("empty response: %v", err)
}
// Common test logic for Qtree plugin
func runQtreeTest(t *testing.T, createQtree func(historicalLabels bool) plugin.Plugin, historicalLabels bool, expectedQuotaCount int, expectedQtreeCount int, quotaInstanceKey string, expectedQuotaLabels int, withNonExportedQtree bool) {
q := createQtree(historicalLabels)

var quotas []*node.Node
if x := response.GetChildS("attributes-list"); x != nil {
quotas = x.GetChildren()
// Initialize the plugin
if err := q.Init(); err != nil {
t.Fatalf("failed to initialize plugin: %v", err)
}

exportOptions := node.NewS("export_options")
instanceKeys := exportOptions.NewChildS("instance_keys", "")
// apply all instance keys, instance labels from qtree.yaml to all quota metrics
keys := []string{"export_policy", "oplocks", "security_style", "status"}
for _, key := range keys {
instanceKeys.NewChildS("", key)
// Create test data
qtreeData := matrix.New("qtree", "qtree", "qtree")
if withNonExportedQtree {
qInstance1, _ := qtreeData.NewInstance("" + "." + "volume1" + "." + "svm1")
addLabels(qInstance1)
qInstance1.SetExportable(false)
qInstance2, _ := qtreeData.NewInstance("trident_pvc_2a6d71d9_1c78_4e9a_84a2_59d316adfae9" + "." + "trident_qtree_pool_trident_TIXRBILLKA" + "." + "astra_300")
addLabels(qInstance2)
qInstance3, _ := qtreeData.NewInstance("" + "." + "abcd_root" + "." + "abcde")
addLabels(qInstance3)
qInstance3.SetExportable(false)
} else {
qInstance1, _ := qtreeData.NewInstance("" + "." + "volume1" + "." + "svm1")
addLabels(qInstance1)
qInstance2, _ := qtreeData.NewInstance("trident_pvc_2a6d71d9_1c78_4e9a_84a2_59d316adfae9" + "." + "trident_qtree_pool_trident_TIXRBILLKA" + "." + "astra_300")
addLabels(qInstance2)
qInstance3, _ := qtreeData.NewInstance("" + "." + "abcd_root" + "." + "abcde")
addLabels(qInstance3)
}

// Case 1: with historicalLabels = false, total 4 quotas, 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// 1 empty tree quota will be skipped and 5 labels[qtree, svm, type, unit, volume] would be exported for tree quota.
q1 := NewQtree()
testLabels(t, q1, false, quotas, nil, "astra_300.trident_qtree_pool_trident_TIXRBILLKA.trident_pvc_2a6d71d9_1c78_4e9a_84a2_59d316adfae9..disk-limit.tree", 3, 6, 5)
dataMap := map[string]*matrix.Matrix{
"qtree": qtreeData,
}

// Case 2: with historicalLabels = true, total 4 quotas, 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// 1 empty tree quota will be skipped and 6 labels[qtree, svm, type, unit, user, volume] would be exported for user/group quota.
q2 := NewQtree()
testLabels(t, q2, false, quotas, nil, "abcde.abcd_root..root.disk-used.user", 3, 6, 6)
// Run the plugin
output, _, err := q.Run(dataMap)
if err != nil {
t.Fatalf("Run method failed: %v", err)
}

// Case 3: with historicalLabels = true, total 4 quotas, 2 user/group quota, 1 empty qtree tree quota and 1 non-empty tree quota,
// all quotas with 9 labels [export_policy, oplocks, qtree, security_style, status, svm, type, unit, volume] would be exported.
q3 := NewQtree()
data := matrix.New(q3.Parent+".Qtree", "qtree", "qtree")
q3.data.SetExportOptions(exportOptions)
qtreeInstance1, _ := data.NewInstance("" + "." + "volume1" + "." + "svm1")
addLabels(qtreeInstance1)
qtreeInstance2, _ := data.NewInstance("trident_pvc_2a6d71d9_1c78_4e9a_84a2_59d316adfae9" + "." + "trident_qtree_pool_trident_TIXRBILLKA" + "." + "astra_300")
addLabels(qtreeInstance2)
qtreeInstance3, _ := data.NewInstance("" + "." + "abcd_root" + "." + "abcde")
addLabels(qtreeInstance3)
testLabels(t, q3, true, quotas, data, "svm1.volume1...disk-used.tree", 4, 8, 9)
quotaOutput := output[0]
verifyInstanceCount(t, quotaOutput, expectedQuotaCount)
verifyInstanceCount(t, qtreeData, expectedQtreeCount)
verifyLabelCount(t, quotaOutput, quotaInstanceKey, expectedQuotaLabels)
}

func addLabels(qtreeInstance *matrix.Instance) {
Expand All @@ -81,26 +155,27 @@ func addLabels(qtreeInstance *matrix.Instance) {
qtreeInstance.SetLabel("status", "normal")
}

func testLabels(t *testing.T, q *Qtree, historicalLabels bool, quotas []*node.Node, data *matrix.Matrix, quotaInstanceKey string, expectedQuotaCount int, expectedQuotaMetricCount int, expectedQuotaLabels int) {
quotaCount := 0
numMetrics := 0
quotaLabels := 0
func verifyInstanceCount(t *testing.T, output *matrix.Matrix, expectedCount int) {
// count exportable instances
currentCount := 0
for _, i := range output.GetInstances() {
if i.IsExportable() {
currentCount++
}
}

q.historicalLabels = historicalLabels
if err := q.handlingQuotaMetrics(quotas, data, &quotaCount, &numMetrics); err != nil {
t.Errorf("handlingQuotaMetrics returned an error: %v", err)
// Verify the number of instances
if currentCount != expectedCount {
t.Errorf("expected %d instances, got %d", expectedCount, currentCount)
}
}

if quotaInstance := q.data.GetInstance(quotaInstanceKey); quotaInstance != nil {
func verifyLabelCount(t *testing.T, quotaOutput *matrix.Matrix, quotaInstanceKey string, expectedQuotaLabels int) {
quotaLabels := 0
if quotaInstance := quotaOutput.GetInstance(quotaInstanceKey); quotaInstance != nil {
quotaLabels = len(quotaInstance.GetLabels())
}

if quotaCount != expectedQuotaCount {
t.Errorf("quotaCount = %d; want %d", quotaCount, expectedQuotaCount)
}
if numMetrics != expectedQuotaMetricCount {
t.Errorf("numMetrics = %d; want %d", numMetrics, expectedQuotaMetricCount)
}
if quotaLabels != expectedQuotaLabels {
t.Errorf("labels = %d; want %d", quotaLabels, expectedQuotaLabels)
}
Expand Down
Loading

0 comments on commit f95fef1

Please sign in to comment.