Merge pull request #2310 from NetApp/release/23.08.0

feat: merge 23.08.0 to main

README.md

@@ -26,6 +26,11 @@ and GitHub [discussions](https://github.com/NetApp/harvest/discussions). Come jo
 
 :closed_book: https://netapp.github.io/harvest/
 
+# Videos
+
+- [Why Harvest](https://youtu.be/04-66_9egJc)
+- [Harvest Quick Start: Docker Compose](https://youtu.be/4cbDKzwjGHI)
+
 ---
 
 <p align="center">

cmd/collectors/restperf/testdata/conf/restperf/9.12.1/workload_volume.yaml

@@ -9,9 +9,9 @@ object:                   qos_volume
 # recommended to use large interval, since workload objects are expensive
 client_timeout:           1m30s
 schedule:
-  - counter: 1200s
-  - instance: 600s
-  - data: 180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - ^^uuid

cmd/poller/collector/asup.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"cmp"
 	"context"
 	"crypto/sha1" //nolint:gosec // used for sha1sum not for security
 	"encoding/hex"
@@ -19,7 +20,7 @@ import (
 	"os/exec"
 	"path"
 	"regexp"
-	"sort"
+	"slices"
 	"time"
 )
 
@@ -347,11 +348,17 @@ func attachMemory(msg *Payload) {
 		msg.Platform.Processes = append(msg.Platform.Processes, pp)
 	}
 
+	// keep at most 100, ordered by size descending
+	if len(msg.Platform.Processes) > 100 {
+		slices.SortStableFunc(msg.Platform.Processes, func(a, b Process) int {
+			return cmp.Compare(b.RssBytes, a.RssBytes)
+		})
+		msg.Platform.Processes = msg.Platform.Processes[:100]
+	}
+
 	// sort processes by pid
-	sort.Slice(msg.Platform.Processes, func(i, j int) bool {
-		a := msg.Platform.Processes[i]
-		b := msg.Platform.Processes[j]
-		return a.Pid < b.Pid
+	slices.SortStableFunc(msg.Platform.Processes, func(a, b Process) int {
+		return cmp.Compare(a.Pid, b.Pid)
 	})
 }
 

cmd/poller/plugin/labelagent/label_agent.go

@@ -20,7 +20,6 @@ type LabelAgent struct {
 	splitRegexRules      []splitRegexRule
 	splitPairsRules      []splitPairsRule
 	joinSimpleRules      []joinSimpleRule
-	renameRules          []renameRule
 	replaceSimpleRules   []replaceSimpleRule
 	replaceRegexRules    []replaceRegexRule
 	excludeEqualsRules   []excludeEqualsRule
@@ -138,24 +137,6 @@ func (a *LabelAgent) joinSimple(matrix *matrix.Matrix) error {
 	return nil
 }
 
-// rename source label, if present, to target label
-// if target label already exists overwrite it
-func (a *LabelAgent) rename(matrix *matrix.Matrix) error {
-	for _, instance := range matrix.GetInstances() {
-		for _, r := range a.renameRules {
-			if old := instance.GetLabel(r.source); old != "" {
-				instance.SetLabel(r.target, old)
-				instance.DeleteLabel(r.source)
-				a.Logger.Trace().
-					Str("source", r.source).
-					Str("target", r.target).
-					Msg("rename")
-			}
-		}
-	}
-	return nil
-}
-
 // replace in source label, if present, and add as new label
 func (a *LabelAgent) replaceSimple(matrix *matrix.Matrix) error {
 	for _, instance := range matrix.GetInstances() {

cmd/poller/plugin/labelagent/label_agent_test.go

@@ -54,7 +54,7 @@ func newLabelAgent() *LabelAgent {
 	// create metric "result", if label "state" matches regex then map to 1 else use default value "4"
 	params.NewChildS("value_to_num_regex", "").NewChildS("", "result value ^test\\d+ ^error `4`")
 
-	// These both are mutually exclusive, and should honor the above one's filtered result.
+	// These both are mutually exclusive, and should honour the above one's filtered result.
 	// exclude instance if label "volstate" has value "offline"
 	params.NewChildS("exclude_equals", "").NewChildS("", "volstate `offline`")
 	// include instance if label "voltype" has value "rw"
@@ -64,9 +64,6 @@ func newLabelAgent() *LabelAgent {
 	// exclude instance if label "volstatus" has value which starts with "stopped_"
 	params.NewChildS("exclude_contains", "").NewChildS("", "volstatus `stop`")
 
-	// rename label named style to type
-	params.NewChildS("rename", "").NewChildS("", "style type")
-
 	abc := plugin.New("Test", nil, params, nil, "", nil)
 	p := &LabelAgent{AbstractPlugin: abc}
 
@@ -148,23 +145,6 @@ func TestJoinSimpleRule(t *testing.T) {
 	}
 }
 
-func TestRenameRule(t *testing.T) {
-	m := matrix.New("TestLabelAgent", "test", "test")
-	p := newLabelAgent()
-
-	instance, _ := m.NewInstance("0")
-	instance.SetLabel("style", "aaa_X")
-
-	_ = p.rename(m)
-
-	if instance.GetLabel("type") != "aaa_X" {
-		t.Errorf("rename failed, label type got=[%s] want=[%s]", instance.GetLabel("type"), "aaa_X")
-	}
-	if instance.GetLabel("style") != "" {
-		t.Errorf("rename failed, style lable should not exist got=[%s] ", instance.GetLabel("style"))
-	}
-}
-
 func TestReplaceSimpleRule(t *testing.T) {
 	m := matrix.New("TestLabelAgent", "test", "test")
 	p := newLabelAgent()

cmd/poller/plugin/labelagent/parse_rules.go

@@ -46,8 +46,6 @@ func (a *LabelAgent) parseRules() int {
 				a.parseSplitPairsRule(rule)
 			case "join":
 				a.parseJoinSimpleRule(rule)
-			case "rename":
-				a.parseRenameRule(rule)
 			case "replace":
 				a.parseReplaceSimpleRule(rule)
 			case "replace_regex":
@@ -102,11 +100,6 @@ func (a *LabelAgent) parseRules() int {
 				a.actions = append(a.actions, a.joinSimple)
 				count += len(a.joinSimpleRules)
 			}
-		case "rename":
-			if len(a.renameRules) != 0 {
-				a.actions = append(a.actions, a.rename)
-				count += len(a.renameRules)
-			}
 		case "replace":
 			if len(a.replaceSimpleRules) != 0 {
 				a.actions = append(a.actions, a.replaceSimple)
@@ -285,28 +278,6 @@ func (a *LabelAgent) parseJoinSimpleRule(rule string) {
 	a.Logger.Warn().Msgf("(join) rule has invalid format [%s]", rule)
 }
 
-type renameRule struct {
-	source string
-	target string
-}
-
-// example rule:
-// style type
-// if the label named `style` exists, rename that label to `type`
-// metric_one{style="flex",vol="vol1"} becomes metric_one{type="flex",vol="vol1"}
-
-func (a *LabelAgent) parseRenameRule(rule string) {
-	if fields := strings.SplitN(rule, " ", 2); len(fields) == 2 {
-		r := renameRule{source: strings.TrimSpace(fields[0]), target: strings.TrimSpace(fields[1])}
-		a.Logger.Debug().Msgf("fields := %v", fields)
-		a.renameRules = append(a.renameRules, r)
-		a.addNewLabels([]string{r.target})
-		a.Logger.Debug().Msgf("(rename) parsed rule [%v]", r)
-		return
-	}
-	a.Logger.Warn().Str("rule", rule).Msg("rename rule has invalid format")
-}
-
 type replaceSimpleRule struct {
 	source string
 	target string

cmd/tools/grafana/dashboard_test.go

@@ -164,7 +164,15 @@ func checkDashboardForDatasource(t *testing.T, path string, data []byte) {
 
 	// Check that the variable DS_PROMETHEUS exist
 	doesDsPromExist := false
+	// This is a list of names that are exempt from the check for a 'true' selected status.
+	excludedNames := map[string]bool{
+		"TopResources": true,
+		"Interval":     true,
+		"IncludeRoot":  true,
+	}
+
 	gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool {
+		name := value.Get("name").String()
 		if value.Get("name").String() == "DS_PROMETHEUS" {
 			doesDsPromExist = true
 			query := value.Get("query").String()
@@ -176,6 +184,19 @@ func checkDashboardForDatasource(t *testing.T, path string, data []byte) {
 				t.Errorf("dashboard=%s var=DS_PROMETHEUS type want=datasource got=%s", path, theType)
 			}
 		}
+
+		if !excludedNames[name] {
+			if value.Get("current.selected").String() == "true" {
+				t.Errorf(
+					"dashboard=%s var=current.selected query want=false got=%s text=%s value=%s name= %s",
+					path,
+					"true",
+					value.Get("current.text"),
+					value.Get("current.value"),
+					name,
+				)
+			}
+		}
 		return true
 	})
 	if !doesDsPromExist {

conf/rest/9.10.0/volume.yaml

@@ -13,6 +13,7 @@ counters:
   - ^anti_ransomware.state                        => antiRansomwareState
   - ^encryption.enabled                           => isEncrypted
   - ^is_svm_root                                  => svm_root
+  - ^nas.path                                     => junction_path
   - ^snaplock.type                                => snaplock_type
   - ^snapshot_policy.name                         => snapshot_policy
   - ^space.snapshot.autodelete_enabled            => snapshot_autodelete
@@ -50,6 +51,7 @@ counters:
       - autosize
       - encryption.enabled
       - is_svm_root
+      - nas.path
       - snaplock.type
       - space
 
@@ -138,6 +140,7 @@ export_options:
     - isEncrypted
     - isHardwareEncrypted
     - is_sis_volume
+    - junction_path
     - root_volume
     - snaplock_type
     - snapshot_autodelete

conf/rest/9.12.0/security_certificate.yaml

@@ -19,7 +19,7 @@ counters:
 plugins:
   - Certificate:
       schedule:
-        - data: 180s  # should be multiple of data poll duration
+        - data: 3m  # should be multiple of data poll duration
 
 export_options:
   instance_keys:

conf/restperf/9.12.0/workload.yaml

@@ -8,9 +8,9 @@ object:                   qos
 client_timeout:           1m30s
 
 schedule:
-  - counter:  1200s
-  - instance: 600s
-  - data:     180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - ^^uuid

conf/restperf/9.12.0/workload_detail.yaml

@@ -9,9 +9,9 @@ object:                   qos_detail
 client_timeout:           1m30s
 
 schedule:
-  - counter:  1200s
-  - instance: 600s
-  - data:     180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - ^^id

conf/restperf/9.12.0/workload_detail_volume.yaml

@@ -8,9 +8,9 @@ object:                   qos_detail
 client_timeout:           1m30s
 
 schedule:
-  - counter: 1200s
-  - instance: 600s
-  - data: 180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - ^^id

conf/restperf/9.12.0/workload_volume.yaml

@@ -10,9 +10,9 @@ object:                   qos
 client_timeout:           1m30s
 
 schedule:
-  - counter: 1200s
-  - instance: 600s
-  - data: 180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - ^^uuid

conf/zapi/cdot/9.8.0/security.yaml

@@ -12,7 +12,7 @@ no_max_records: true
 plugins:
   - Security:
       schedule:
-        - data: 180s  # should be multiple of data poll duration
+        - data: 3m  # should be multiple of data poll duration
 
 export_options:
   instance_keys:

conf/zapi/cdot/9.8.0/security_certificate.yaml

@@ -18,7 +18,7 @@ plugins:
         - type `server`
   - Certificate:
       schedule:
-        - data: 180s  # should be multiple of data poll duration
+        - data: 3m  # should be multiple of data poll duration
 
 export_options:
   instance_keys:

conf/zapi/cdot/9.8.0/volume.yaml

@@ -14,6 +14,7 @@ counters:
     - volume-id-attributes:
         - ^^instance-uuid                       => instance_uuid
         - ^containing-aggregate-uuid            => aggrUuid
+        - ^junction-path                        => junction_path
         - ^name                                 => volume
         - ^owning-vserver-name                  => svm
         - ^style-extended                       => style
@@ -117,6 +118,7 @@ export_options:
     - isEncrypted
     - isHardwareEncrypted
     - is_sis_volume
+    - junction_path
     - node_root
     - root_volume
     - snapshot_autodelete

conf/zapi/default.yaml

@@ -3,7 +3,7 @@ collector:          Zapi
 
 # Order here matters!
 schedule:
-  - data: 180s
+  - data: 3m
 
 objects:
   Aggregate:                   aggr.yaml

conf/zapiperf/cdot/9.8.0/workload.yaml

@@ -10,9 +10,9 @@ instance_key:             uuid
 # recommended to use large interval, since workload objects are expensive
 client_timeout:           1m30s
 schedule:
-  - counter: 1200s
-  - instance: 600s
-  - data: 180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - concurrency

conf/zapiperf/cdot/9.8.0/workload_detail.yaml

@@ -11,9 +11,9 @@ instance_key:             name
 client_timeout:           1m30s
 
 schedule:
-  - counter: 1200s
-  - instance: 600s
-  - data: 180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - instance_name

conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml

@@ -10,9 +10,9 @@ instance_key:             name
 client_timeout:           1m30s
 
 schedule:
-  - counter: 1200s
-  - instance: 600s
-  - data: 180s
+  - counter:  20m
+  - instance: 10m
+  - data:      3m
 
 counters:
   - instance_name