diff --git a/cmd/collectors/keyperf/keyperf.go b/cmd/collectors/keyperf/keyperf.go index 191dd9f77..6ec2d310a 100644 --- a/cmd/collectors/keyperf/keyperf.go +++ b/cmd/collectors/keyperf/keyperf.go @@ -140,7 +140,14 @@ func (kp *KeyPerf) loadParamInt(name string, defaultValue int) int { } func (kp *KeyPerf) buildCounters() { - for k := range kp.Prop.Metrics { + staticCounterDef, err := loadStaticCounterDefinitions(kp.Prop.Object, "cmd/collectors/keyperf/static_counter_definitions.yaml", kp.Logger) + if err != nil { + // It's acceptable to continue even if there are errors, as the remaining counters will still be processed. + // Any counters that require counter metadata will be skipped. + kp.Logger.Error("Failed to load static counter definitions", slogx.Err(err)) + } + + for k, v := range kp.Prop.Metrics { if _, exists := kp.perfProp.counterInfo[k]; !exists { var ctr *counter @@ -170,6 +177,36 @@ func (kp *KeyPerf) buildCounters() { counterType: "delta", unit: "sec", } + default: + // look up metric in staticCounterDef + if counterDef, exists := staticCounterDef.CounterDefinitions[v.Name]; exists { + ctr = &counter{ + name: k, + counterType: counterDef.Type, + denominator: counterDef.BaseCounter, + } + if counterDef.BaseCounter != "" { + // Ensure denominator exists in counterInfo + if _, denomExists := kp.perfProp.counterInfo[counterDef.BaseCounter]; !denomExists { + var baseCounterType string + if baseCounterDef, baseCounterExists := staticCounterDef.CounterDefinitions[counterDef.BaseCounter]; baseCounterExists { + baseCounterType = baseCounterDef.Type + } + if baseCounterType != "" { + kp.perfProp.counterInfo[counterDef.BaseCounter] = &counter{ + name: counterDef.BaseCounter, + counterType: staticCounterDef.CounterDefinitions[counterDef.BaseCounter].Type, + } + if _, dExists := kp.Prop.Metrics[counterDef.BaseCounter]; !dExists { + m := &rest.Metric{Label: "", Name: counterDef.BaseCounter, MetricType: "", Exportable: false} + kp.Prop.Metrics[counterDef.BaseCounter] = m + } + } + } + } + } else { + slog.Warn("Skipping metric due to unknown metricType", slog.String("name", k), slog.String("metricType", v.MetricType)) + } } if ctr != nil { @@ -304,7 +341,8 @@ func (kp *KeyPerf) pollData( orderedDenominatorKeys = append(orderedDenominatorKeys, key) } } else { - kp.Logger.Warn("Counter is missing or unable to parse", slog.String("counter", metric.GetName())) + kp.Logger.Error("Counter is missing or unable to parse", slog.String("counter", metric.GetName())) + metric.SetExportable(false) } } @@ -375,6 +413,8 @@ func (kp *KeyPerf) pollData( slog.String("denominator", counter.denominator), slog.Int("instIndex", instIndex), ) + skips = curMat.Skip(key) + totalSkips += skips continue } diff --git a/cmd/collectors/keyperf/static_counter_definitions.yaml b/cmd/collectors/keyperf/static_counter_definitions.yaml new file mode 100644 index 000000000..a4887c9c5 --- /dev/null +++ b/cmd/collectors/keyperf/static_counter_definitions.yaml @@ -0,0 +1,8 @@ +objects: + node: + counter_definitions: + - name: statistics.processor_utilization_raw + type: percent + base_counter: statistics.processor_utilization_base + - name: statistics.processor_utilization_base + type: delta \ No newline at end of file diff --git a/cmd/collectors/keyperf/templating.go b/cmd/collectors/keyperf/templating.go new file mode 100644 index 000000000..a13d64fea --- /dev/null +++ b/cmd/collectors/keyperf/templating.go @@ -0,0 +1,65 @@ +package keyperf + +import ( + "gopkg.in/yaml.v3" + "log/slog" + "os" +) + +type CounterDefinition struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + BaseCounter string `yaml:"base_counter,omitempty"` +} + +type ObjectCounters struct { + CounterDefinitions map[string]CounterDefinition `yaml:"-"` +} + +type Object struct { + CounterDefinitions []CounterDefinition `yaml:"counter_definitions"` +} + +type StaticCounterDefinitions struct { + Objects map[string]Object `yaml:"objects"` +} + +func loadStaticCounterDefinitions(object string, filePath string, logger *slog.Logger) (ObjectCounters, error) { + var staticDefinitions StaticCounterDefinitions + var objectCounters ObjectCounters + + data, err := os.ReadFile(filePath) + if err != nil { + return objectCounters, err + } + + err = yaml.Unmarshal(data, &staticDefinitions) + if err != nil { + return objectCounters, err + } + + if obj, exists := staticDefinitions.Objects[object]; exists { + allCounterDefs := make(map[string]CounterDefinition) + for _, def := range obj.CounterDefinitions { + allCounterDefs[def.Name] = def + } + + objectCounters.CounterDefinitions = make(map[string]CounterDefinition) + for _, def := range obj.CounterDefinitions { + if def.Type == "" { + logger.Error("Missing type in counter definition", slog.String("filePath", filePath), slog.String("counterName", def.Name)) + continue + } + if def.BaseCounter != "" { + if _, baseCounterExists := allCounterDefs[def.BaseCounter]; !baseCounterExists { + logger.Error("Base counter definition not found", slog.String("filePath", filePath), slog.String("counterName", def.Name), slog.String("baseCounter", def.BaseCounter)) + continue + } + } + objectCounters.CounterDefinitions[def.Name] = def + } + return objectCounters, nil + } + + return objectCounters, nil +} diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go index 239d10f62..95ee382a4 100644 --- a/cmd/collectors/rest/rest.go +++ b/cmd/collectors/rest/rest.go @@ -630,6 +630,8 @@ func (r *Rest) HandleResults(mat *matrix.Matrix, result []gjson.Result, prop *pr slogx.Err(err), slog.String("name", metric.Name), ) + } else { + metr.SetExportable(metric.Exportable) } } f := instanceData.Get(metric.Name) diff --git a/cmd/tools/template/template_test.go b/cmd/tools/template/template_test.go index 46bba578c..abc861dff 100644 --- a/cmd/tools/template/template_test.go +++ b/cmd/tools/template/template_test.go @@ -19,7 +19,7 @@ import ( const toConf = "../../../conf" -var allTemplatesButEms = []string{"rest", "restperf", "storagegrid", "zapi", "zapiperf"} +var allTemplatesButEms = []string{"rest", "restperf", "storagegrid", "zapi", "zapiperf", "keyperf"} // validates each template file name: // - ends with yaml diff --git a/conf/keyperf/9.15.0/aggr.yaml b/conf/keyperf/9.15.0/aggr.yaml new file mode 100644 index 000000000..5009c97fb --- /dev/null +++ b/conf/keyperf/9.15.0/aggr.yaml @@ -0,0 +1,30 @@ +name: Aggregate +query: api/storage/aggregates +object: aggr + +counters: + - ^^uuid => uuid + - ^name => aggr + - ^node.name => node + - ^statistics.status => status + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency + - statistics.throughput_raw.other => other_data + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + + +export_options: + instance_keys: + - aggr + - node \ No newline at end of file diff --git a/conf/keyperf/9.15.0/cifs_vserver.yaml b/conf/keyperf/9.15.0/cifs_vserver.yaml new file mode 100644 index 000000000..30172bdeb --- /dev/null +++ b/conf/keyperf/9.15.0/cifs_vserver.yaml @@ -0,0 +1,27 @@ +name: CIFSvserver +query: api/protocols/cifs/services +object: svm_cifs + +counters: + - ^^svm.uuid => uuid + - ^statistics.status => status + - ^svm.name => svm + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency + - statistics.throughput_raw.other => other_data + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + + +export_options: + instance_keys: + - svm diff --git a/conf/keyperf/9.15.0/cluster.yaml b/conf/keyperf/9.15.0/cluster.yaml new file mode 100644 index 000000000..b60076bcc --- /dev/null +++ b/conf/keyperf/9.15.0/cluster.yaml @@ -0,0 +1,27 @@ +name: Cluster +query: api/cluster +object: cluster + +counters: + - ^^uuid => uuid + - ^statistics.status => status + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency + - statistics.throughput_raw.other => other_data + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + + +export_options: + instance_keys: + - uuid diff --git a/conf/keyperf/9.15.0/iscsi_svm.yaml b/conf/keyperf/9.15.0/iscsi_svm.yaml new file mode 100644 index 000000000..1bbc20597 --- /dev/null +++ b/conf/keyperf/9.15.0/iscsi_svm.yaml @@ -0,0 +1,27 @@ +name: ISCSISvm +query: api/protocols/san/iscsi/services +object: iscsi_svm + +counters: + - ^^svm.name => svm + - ^statistics.status => status + - ^target.name => target + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + +export_options: + instance_keys: + - svm + - target diff --git a/conf/keyperf/9.15.0/lif.yaml b/conf/keyperf/9.15.0/lif.yaml new file mode 100644 index 000000000..b0d8aca82 --- /dev/null +++ b/conf/keyperf/9.15.0/lif.yaml @@ -0,0 +1,25 @@ +name: LIF +query: api/network/ip/interfaces +object: lif + +counters: + - ^^uuid + - ^location.node.name => node + - ^location.port.name => port + - ^name => lif + - ^statistics.status => status + - ^svm.name => svm + - statistics.throughput_raw.read => sent_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => recv_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + + +export_options: + instance_keys: + - lif + - node + - port + - svm diff --git a/conf/keyperf/9.15.0/lun.yaml b/conf/keyperf/9.15.0/lun.yaml new file mode 100644 index 000000000..f90c3d38a --- /dev/null +++ b/conf/keyperf/9.15.0/lun.yaml @@ -0,0 +1,38 @@ +name: Lun +query: api/storage/luns +object: lun + +counters: + - ^^uuid => uuid + - ^location.volume.name => volume + - ^name => path + - ^statistics.status => status + - ^svm.name => svm + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => avg_read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => avg_write_latency + - statistics.throughput_raw.other => other_data + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + + +plugins: + LabelAgent: + split_regex: + - path `^/[^/]+/([^/]+)(?:/.*?|)/([^/]+)$` volume,lun + - path `^([^/]+)$` lun + +export_options: + instance_keys: + - lun + - svm + - volume diff --git a/conf/keyperf/9.15.0/namespace.yaml b/conf/keyperf/9.15.0/namespace.yaml new file mode 100644 index 000000000..30d314dde --- /dev/null +++ b/conf/keyperf/9.15.0/namespace.yaml @@ -0,0 +1,36 @@ +name: Namespace +query: api/storage/namespaces +object: namespace + +counters: + - ^^uuid => uuid + - ^name => path + - ^statistics.status => status + - ^svm.name => svm + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => avg_other_latency + - statistics.latency_raw.read => avg_read_latency + - statistics.latency_raw.total => avg_total_latency + - statistics.latency_raw.write => avg_write_latency + - statistics.throughput_raw.other => other_data + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + +plugins: + LabelAgent: + split: + - path `/` ,,volume,namespace + +export_options: + instance_keys: + - namespace + - path + - svm + - volume diff --git a/conf/keyperf/9.15.0/node.yaml b/conf/keyperf/9.15.0/node.yaml new file mode 100644 index 000000000..b552fd0dd --- /dev/null +++ b/conf/keyperf/9.15.0/node.yaml @@ -0,0 +1,16 @@ +name: Node +query: api/cluster/nodes +object: node + +counters: + - ^^uuid => uuid + - ^name => node + - ^statistics.status => status + - statistics.processor_utilization_raw => avg_processor_busy + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + +export_options: + instance_keys: + - node \ No newline at end of file diff --git a/conf/keyperf/9.15.0/nvme_services.yaml b/conf/keyperf/9.15.0/nvme_services.yaml new file mode 100644 index 000000000..20588b9b4 --- /dev/null +++ b/conf/keyperf/9.15.0/nvme_services.yaml @@ -0,0 +1,27 @@ +name: NvmeServices +query: api/protocols/nvme/services +object: nvme_services + +counters: + - ^^svm.uuid => uuid + - ^statistics.status => status + - ^svm.name => svm + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + + +export_options: + instance_keys: + - svm diff --git a/conf/keyperf/9.15.0/ontap_s3_svm.yaml b/conf/keyperf/9.15.0/ontap_s3_svm.yaml new file mode 100644 index 000000000..48320c22f --- /dev/null +++ b/conf/keyperf/9.15.0/ontap_s3_svm.yaml @@ -0,0 +1,27 @@ +name: OntapS3SVM +query: api/protocols/s3/services +object: ontaps3_svm + +counters: + - ^^svm.name => svm + - ^name => name + - ^statistics.status => status + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - hidden_fields: + - statistics + +export_options: + instance_keys: + - name + - svm \ No newline at end of file diff --git a/conf/keyperf/9.15.0/qtree.yaml b/conf/keyperf/9.15.0/qtree.yaml new file mode 100644 index 000000000..0e13c072f --- /dev/null +++ b/conf/keyperf/9.15.0/qtree.yaml @@ -0,0 +1,30 @@ +name: Qtree +query: api/storage/qtrees +object: qtree + +client_timeout: 2m + +counters: + - ^^name => qtree + - ^^svm.name => svm + - ^^volume.name => volume + - ^statistics.status => status + - statistics.iops_raw.other => other_ops + - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops + - statistics.iops_raw.write => write_ops + - statistics.throughput_raw.other => other_data + - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data + - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp + - filter: + - name=!"" + - hidden_fields: + - statistics + +export_options: + instance_keys: + - qtree + - svm + - volume diff --git a/conf/keyperf/9.15.0/volume.yaml b/conf/keyperf/9.15.0/volume.yaml index d1eeee28c..ee0d4ccff 100644 --- a/conf/keyperf/9.15.0/volume.yaml +++ b/conf/keyperf/9.15.0/volume.yaml @@ -5,31 +5,31 @@ object: volume counters: - ^^name => volume - ^^svm.name => svm - - ^style => style - ^statistics.status => status - - statistics.timestamp(timestamp) => timestamp - - statistics.latency_raw.other => other_latency - - statistics.latency_raw.total => total_latency - - statistics.latency_raw.read => read_latency - - statistics.latency_raw.write => write_latency + - ^style => style - statistics.iops_raw.other => other_ops - - statistics.iops_raw.total => total_ops - statistics.iops_raw.read => read_ops + - statistics.iops_raw.total => total_ops - statistics.iops_raw.write => write_ops + - statistics.latency_raw.other => other_latency + - statistics.latency_raw.read => read_latency + - statistics.latency_raw.total => total_latency + - statistics.latency_raw.write => write_latency - statistics.throughput_raw.other => other_data - - statistics.throughput_raw.total => total_data - statistics.throughput_raw.read => read_data + - statistics.throughput_raw.total => total_data - statistics.throughput_raw.write => write_data + - statistics.timestamp(timestamp) => timestamp - hidden_fields: - statistics endpoints: - query: api/private/cli/volume counters: - - ^^volume => volume - - ^^vserver => svm - - ^aggr_list => aggr - - ^nodes => node + - ^^volume => volume + - ^^vserver => svm + - ^aggr_list => aggr + - ^nodes => node plugins: - Aggregator: diff --git a/conf/keyperf/default.yaml b/conf/keyperf/default.yaml index 8138417ad..ad72b5e8a 100644 --- a/conf/keyperf/default.yaml +++ b/conf/keyperf/default.yaml @@ -6,4 +6,15 @@ schedule: - data: 1m objects: - Volume: volume.yaml \ No newline at end of file + Aggregate: aggr.yaml + CIFSvserver: cifs_vserver.yaml + Cluster: cluster.yaml + ISCSISvm: iscsi_svm.yaml + LIF: lif.yaml + Lun: lun.yaml + Namespace: namespace.yaml + Node: node.yaml + NvmeServices: nvme_services.yaml + Volume: volume.yaml + Qtree: qtree.yaml + OntapS3SVM: ontap_s3_svm.yaml \ No newline at end of file diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go index 5ce5d4db1..4517ba46e 100644 --- a/pkg/matrix/matrix.go +++ b/pkg/matrix/matrix.go @@ -528,3 +528,15 @@ func (m *Matrix) MultiplyByScalar(metricKey string, s uint) (int, error) { } return skips, nil } + +func (m *Matrix) Skip(metricKey string) int { + var skips int + metric := m.GetMetric(metricKey) + if metric != nil { + for i := range len(metric.values) { + metric.record[i] = false + skips++ + } + } + return skips +}