Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: remove extra dict wrapper #2396

Merged
merged 2 commits into from
Oct 3, 2023
Merged

perf: remove extra dict wrapper #2396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c04054a
perf: remove extra dict wrapper
cgrinds Oct 2, 2023
ccd4e45
perf: remove extra dict wrapper
cgrinds Oct 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions cmd/collectors/power.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/dict"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/util"
Expand Down Expand Up @@ -384,7 +383,7 @@ type Sensor struct {
data *matrix.Matrix
client *rest.Client
instanceKeys map[string]string
instanceLabels map[string]*dict.Dict
instanceLabels map[string]map[string]string
}

func (my *Sensor) Init() error {
Expand All @@ -406,7 +405,7 @@ func (my *Sensor) Init() error {

my.data = matrix.New(my.Parent+".Sensor", "environment_sensor", "environment_sensor")
my.instanceKeys = make(map[string]string)
my.instanceLabels = make(map[string]*dict.Dict)
my.instanceLabels = make(map[string]map[string]string)

// init environment metrics in plugin matrix
// create environment metric if not exists
Expand Down
2 changes: 1 addition & 1 deletion cmd/collectors/rest/plugins/netroute/netroute.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err

count := 0
for key, instance := range data.GetInstances() {
cluster := data.GetGlobalLabels().Get("cluster")
cluster := data.GetGlobalLabels()["cluster"]
routeID := instance.GetLabel("uuid")
interfaceName := instance.GetLabel("interface_name")
interfaceAddress := instance.GetLabel("interface_address")
Expand Down
17 changes: 8 additions & 9 deletions cmd/collectors/rest/plugins/qtree/qtree.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/dict"
"github.com/netapp/harvest/v2/pkg/errs"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
Expand All @@ -24,7 +23,7 @@ type Qtree struct {
*plugin.AbstractPlugin
data *matrix.Matrix
instanceKeys map[string]string
instanceLabels map[string]*dict.Dict
instanceLabels map[string]map[string]string
client *rest.Client
query string
quotaType []string
Expand All @@ -44,13 +43,13 @@ func (q *Qtree) Init() error {
"space.used.hard_limit_percent => disk_used_pct_disk_limit",
"space.used.soft_limit_percent => disk_used_pct_soft_disk_limit",
"space.soft_limit => soft_disk_limit",
//"disk-used-pct-threshold" # deprecated and workaround to use same as disk_used_pct_soft_disk_limit
// "disk-used-pct-threshold" # deprecated and workaround to use same as disk_used_pct_soft_disk_limit
"files.hard_limit => file_limit",
"files.used.total => files_used",
"files.used.hard_limit_percent => files_used_pct_file_limit",
"files.used.soft_limit_percent => files_used_pct_soft_file_limit",
"files.soft_limit => soft_file_limit",
//"threshold", # deprecated
// "threshold", # deprecated
}

if err = q.InitAbc(); err != nil {
Expand Down Expand Up @@ -78,7 +77,7 @@ func (q *Qtree) Init() error {

q.data = matrix.New(q.Parent+".Qtree", "quota", "quota")
q.instanceKeys = make(map[string]string)
q.instanceLabels = make(map[string]*dict.Dict)
q.instanceLabels = make(map[string]map[string]string)
q.historicalLabels = false

if q.Params.HasChildS("historicalLabels") {
Expand All @@ -87,7 +86,7 @@ func (q *Qtree) Init() error {

// apply all instance keys, instance labels from parent (qtree.yaml) to all quota metrics
if exportOption := q.ParentParams.GetChildS("export_options"); exportOption != nil {
//parent instancekeys would be added in plugin metrics
// parent instancekeys would be added in plugin metrics
if parentKeys := exportOption.GetChildS("instance_keys"); parentKeys != nil {
for _, parentKey := range parentKeys.GetAllChildContentS() {
instanceKeys.NewChildS("", parentKey)
Expand Down Expand Up @@ -167,7 +166,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error)
}

quotaCount := 0
cluster, _ := data.GetGlobalLabels().GetHas("cluster")
cluster := data.GetGlobalLabels()["cluster"]

if q.historicalLabels {
// In 22.05, populate metrics with qtree prefix and old labels
Expand Down Expand Up @@ -251,7 +250,7 @@ func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Ma
}
}

//set labels
// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
Expand Down Expand Up @@ -320,7 +319,7 @@ func (q *Qtree) handlingQuotaMetrics(result []gjson.Result, cluster string, quot
q.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err)
return err
}
//set labels
// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
Expand Down
2 changes: 1 addition & 1 deletion cmd/collectors/rest/plugins/securityaccount/securityaccount.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr
return nil, err
}

for k, v := range securityAccountInstance.GetLabels().Map() {
for k, v := range securityAccountInstance.GetLabels() {
securityAccountNewInstance.SetLabel(k, v)
}
securityAccountNewInstance.SetLabel("applications", application)
Expand Down
6 changes: 3 additions & 3 deletions cmd/collectors/rest/plugins/snapmirror/snapmirror.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix,
if my.currentVal >= PluginInvocationRate {
my.currentVal = 0

if cluster, ok := data.GetGlobalLabels().GetHas("cluster"); ok {
if cluster, ok := data.GetGlobalLabels()["cluster"]; ok {
if err := my.getSVMPeerData(cluster); err != nil {
return nil, err
}
Expand Down Expand Up @@ -135,7 +135,7 @@ func (my *SnapMirror) getSVMPeerData(cluster string) error {

func (my *SnapMirror) updateSMLabels(data *matrix.Matrix) {
var keys []string
cluster, _ := data.GetGlobalLabels().GetHas("cluster")
cluster := data.GetGlobalLabels()["cluster"]

lastTransferSizeMetric := data.GetMetric("last_transfer_size")
lagTimeMetric := data.GetMetric("lag_time")
Expand Down Expand Up @@ -204,7 +204,7 @@ func (my *SnapMirror) handleCGRelationships(data *matrix.Matrix, keys []string)
continue
}

for k, v := range cgInstance.GetLabels().Map() {
for k, v := range cgInstance.GetLabels() {
cgVolumeInstance.SetLabel(k, v)
}
cgVolumeInstance.SetLabel("relationship_id", cgVolumeInstanceKey)
Expand Down
2 changes: 1 addition & 1 deletion cmd/collectors/rest/plugins/volume/volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func (my *Volume) handleARWProtection(data *matrix.Matrix) {
}
}

arwInstanceKey := data.GetGlobalLabels().Get("cluster") + data.GetGlobalLabels().Get("datacenter")
arwInstanceKey := data.GetGlobalLabels()["cluster"] + data.GetGlobalLabels()["datacenter"]
if arwInstance, err = my.arw.NewInstance(arwInstanceKey); err != nil {
my.Logger.Error().Err(err).Str("arwInstanceKey", arwInstanceKey).Msg("Failed to create arw instance")
return
Expand Down
4 changes: 2 additions & 2 deletions cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func (v *VolumeAnalytics) initMatrix() error {

func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
data := dataMap[v.Object]
cluster, _ := data.GetGlobalLabels().GetHas("cluster")
cluster := data.GetGlobalLabels()["cluster"]
clusterVersion := v.client.Cluster().GetVersion()
ontapVersion, err := goversion.NewVersion(clusterVersion)
if err != nil {
Expand Down Expand Up @@ -152,7 +152,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr
instance.SetLabel("dir_name", name)
instance.SetLabel("index", cluster+"_"+strconv.Itoa(index))
// copy all labels
for k1, v1 := range dataInstance.GetLabels().Map() {
for k1, v1 := range dataInstance.GetLabels() {
instance.SetLabel(k1, v1)
}
if bytesUsed != "" {
Expand Down
15 changes: 7 additions & 8 deletions cmd/collectors/restperf/plugins/disk/disk.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/dict"
"github.com/netapp/harvest/v2/pkg/errs"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
Expand Down Expand Up @@ -44,7 +43,7 @@ type Disk struct {
shelfData map[string]*matrix.Matrix
powerData map[string]*matrix.Matrix
instanceKeys map[string]string
instanceLabels map[string]*dict.Dict
instanceLabels map[string]map[string]string
client *rest.Client
query string
aggrMap map[string]*aggregate
Expand Down Expand Up @@ -169,7 +168,7 @@ func (d *Disk) Init() error {
d.powerData = make(map[string]*matrix.Matrix)

d.instanceKeys = make(map[string]string)
d.instanceLabels = make(map[string]*dict.Dict)
d.instanceLabels = make(map[string]map[string]string)

for attribute, childObj := range shelfMetric {

Expand All @@ -180,7 +179,7 @@ func (d *Disk) Init() error {
objectName = strings.TrimSpace(x[1])
}

d.instanceLabels[attribute] = dict.New()
d.instanceLabels[attribute] = make(map[string]string)

d.shelfData[attribute] = matrix.New(d.Parent+".Shelf", "shelf_"+objectName, "shelf_"+objectName)
d.shelfData[attribute].SetGlobalLabel("datacenter", d.ParentParams.GetChildContentS("datacenter"))
Expand All @@ -201,11 +200,11 @@ func (d *Disk) Init() error {
switch kind {
case "key":
d.instanceKeys[attribute] = metricName
d.instanceLabels[attribute].Set(metricName, display)
d.instanceLabels[attribute][metricName] = display
instanceKeys.NewChildS("", display)
d.Logger.Debug().Msgf("added instance key: (%s) [%s]", attribute, display)
case "label":
d.instanceLabels[attribute].Set(metricName, display)
d.instanceLabels[attribute][metricName] = display
instanceLabels.NewChildS("", display)
d.Logger.Debug().Msgf("added instance label: (%s) [%s]", attribute, display)
case "float":
Expand Down Expand Up @@ -305,7 +304,7 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error)
}
d.Logger.Debug().Msgf("add (%s) instance: %s.%s.%s", attribute, shelfSerialNumber, attribute, key)

for label, labelDisplay := range d.instanceLabels[attribute].Map() {
for label, labelDisplay := range d.instanceLabels[attribute] {
if value := obj.Get(label); value.Exists() {
if value.IsArray() {
var labelArray []string
Expand All @@ -331,7 +330,7 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error)
shelfChildInstance.SetLabel("shelf", shelfName)

// Each child would have different possible values which is an ugly way to write all of them,
// so normal value would be mapped to 1 and rest all are mapped to 0.
// so normal value would be mapped to 1, and the rest all are mapped to 0.
if shelfChildInstance.GetLabel("status") == "normal" {
_ = statusMetric.SetValueInt64(shelfChildInstance, 1)
} else {
Expand Down
9 changes: 5 additions & 4 deletions cmd/collectors/restperf/plugins/volume/volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/set"
"maps"
"regexp"
"sort"
"strings"
Expand Down Expand Up @@ -67,7 +68,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
key := i.GetLabel("svm") + "." + match[1]
if cache.GetInstance(key) == nil {
fg, _ := cache.NewInstance(key)
fg.SetLabels(i.GetLabels().Copy())
fg.SetLabels(maps.Clone(i.GetLabels()))
fg.SetLabel("volume", match[1])
// Flexgroup don't show any aggregate, node
fg.SetLabel("aggr", "")
Expand All @@ -77,7 +78,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error

if volumeAggrmetric.GetInstance(key) == nil {
flexgroupInstance, _ := volumeAggrmetric.NewInstance(key)
flexgroupInstance.SetLabels(i.GetLabels().Copy())
flexgroupInstance.SetLabels(maps.Clone(i.GetLabels()))
flexgroupInstance.SetLabel("volume", match[1])
// Flexgroup don't show any node
flexgroupInstance.SetLabel("node", "")
Expand All @@ -98,7 +99,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
v.Logger.Error().Err(err).Str("key", key).Msg("Failed to create new instance")
continue
}
flexvolInstance.SetLabels(i.GetLabels().Copy())
flexvolInstance.SetLabels(maps.Clone(i.GetLabels()))
flexvolInstance.SetLabel(style, "flexvol")
if err := metric.SetValueFloat64(flexvolInstance, 1); err != nil {
v.Logger.Error().Err(err).Str("metric", metricName).Msg("Unable to set value on metric")
Expand All @@ -108,7 +109,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error

v.Logger.Debug().Int("flexgroup volume count", len(cache.GetInstances())).Msg("")

//cache.Reset()
// cache.Reset()

// create summary
for _, i := range data.GetInstances() {
Expand Down
10 changes: 9 additions & 1 deletion cmd/collectors/restperf/restperf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/cmd/tools/rest"
"github.com/netapp/harvest/v2/pkg/color"
"github.com/netapp/harvest/v2/pkg/dict"
"github.com/netapp/harvest/v2/pkg/errs"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/set"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"github.com/rs/zerolog"
"github.com/tidwall/gjson"
"path"
"strconv"
Expand Down Expand Up @@ -1450,7 +1452,13 @@ func (r *RestPerf) updateQosLabels(qos gjson.Result, instance *matrix.Instance,
r.Logger.Trace().Str("label", label).Str("key", key).Msg("Missing label")
}
}
r.Logger.Debug().Str("query", r.Prop.Query).Str("key", key).Str("qos labels", instance.GetLabels().String()).Send()
if r.Logger.GetLevel() == zerolog.DebugLevel {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you clarify the need for this code change? Should we be using if-else like this for debug logging?

r.Logger.Debug().
Str("query", r.Prop.Query).
Str("key", key).
Str("qos labels", dict.String(instance.GetLabels())).
Send()
}
}
}

Expand Down
3 changes: 1 addition & 2 deletions cmd/collectors/sensor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package collectors
import (
"fmt"
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/dict"
"github.com/netapp/harvest/v2/pkg/logging"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree"
Expand Down Expand Up @@ -94,7 +93,7 @@ func loadTestdata() {

sensor.data = matrix.New("Sensor", "environment_sensor", "environment_sensor")
sensor.instanceKeys = make(map[string]string)
sensor.instanceLabels = make(map[string]*dict.Dict)
sensor.instanceLabels = make(map[string]map[string]string)
sensor.AbstractPlugin.Logger = logging.Get()

for _, k := range eMetrics {
Expand Down
13 changes: 6 additions & 7 deletions cmd/collectors/zapi/plugins/qtree/qtree.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/dict"
"github.com/netapp/harvest/v2/pkg/errs"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/tree/node"
Expand All @@ -23,7 +22,7 @@ type Qtree struct {
*plugin.AbstractPlugin
data *matrix.Matrix
instanceKeys map[string]string
instanceLabels map[string]*dict.Dict
instanceLabels map[string]map[string]string
batchSize string
client *zapi.Client
query string
Expand Down Expand Up @@ -61,7 +60,7 @@ func (q *Qtree) Init() error {

q.data = matrix.New(q.Parent+".Qtree", "quota", "quota")
q.instanceKeys = make(map[string]string)
q.instanceLabels = make(map[string]*dict.Dict)
q.instanceLabels = make(map[string]map[string]string)
q.historicalLabels = false

if q.Params.HasChildS("historicalLabels") {
Expand All @@ -70,7 +69,7 @@ func (q *Qtree) Init() error {

// apply all instance keys, instance labels from parent (qtree.yaml) to all quota metrics
if exportOption := q.ParentParams.GetChildS("export_options"); exportOption != nil {
//parent instancekeys would be added in plugin metrics
// parent instancekeys would be added in plugin metrics
if parentKeys := exportOption.GetChildS("instance_keys"); parentKeys != nil {
for _, parentKey := range parentKeys.GetAllChildContentS() {
instanceKeys.NewChildS("", parentKey)
Expand Down Expand Up @@ -173,7 +172,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error)
}
}

cluster, _ := data.GetGlobalLabels().GetHas("cluster")
cluster := data.GetGlobalLabels()["cluster"]

for {
response, tag, ad, pd, err = q.client.InvokeBatchWithTimers(request, tag)
Expand Down Expand Up @@ -291,7 +290,7 @@ func (q *Qtree) handlingHistoricalMetrics(quotas []*node.Node, data *matrix.Matr
}
}

//set labels
// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
Expand Down Expand Up @@ -385,7 +384,7 @@ func (q *Qtree) handlingQuotaMetrics(quotas []*node.Node, cluster string, quotaI
q.Logger.Debug().Msgf("add (%s) instance: %v", attribute, err)
return err
}
//set labels
// set labels
quotaInstance.SetLabel("type", quotaType)
quotaInstance.SetLabel("qtree", tree)
quotaInstance.SetLabel("volume", volume)
Expand Down
Loading
Loading