Merge branch 'main' into hl_alert_test

.golangci.yml

@@ -33,6 +33,7 @@ linters:
     - prealloc
     - reassign
     - revive
+    - sloglint
     - staticcheck
     - stylecheck
     - tenv

cmd/admin/admin.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"github.com/netapp/harvest/v2/pkg/conf"
-	"github.com/netapp/harvest/v2/pkg/logging"
 	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/rs/zerolog"
 	"github.com/spf13/cobra"
@@ -111,7 +110,6 @@ func (a *Admin) APISD(w http.ResponseWriter, r *http.Request) {
 
 func (a *Admin) setupLogger() {
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	zerolog.ErrorStackMarshaler = logging.MarshalStack //nolint:reassign
 
 	a.logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).
 		With().Caller().Timestamp().Logger()

cmd/collectors/ems/ems.go

@@ -155,12 +155,12 @@ func (e *Ems) InitCache() error {
 			e.maxURLSize = s
 		}
 	}
-	e.Logger.Debug().Int("max_url_size", e.maxURLSize).Msgf("")
+	e.Logger.Debug().Int("max_url_size", e.maxURLSize).Send()
 
 	if s := e.Params.GetChildContentS("severity"); s != "" {
 		e.severityFilter = severityFilterPrefix + s
 	}
-	e.Logger.Debug().Str("severityFilter", e.severityFilter).Msgf("")
+	e.Logger.Debug().Str("severityFilter", e.severityFilter).Send()
 
 	if export := e.Params.GetChildS("export_options"); export != nil {
 		e.Matrix[e.Object].SetExportOptions(export)

cmd/collectors/rest/rest.go

@@ -239,7 +239,7 @@ func (r *Rest) getClient(a *collector.AbstractCollector, c *auth.Credentials) (*
 
 	opt := a.GetOptions()
 	if poller, err = conf.PollerNamed(opt.Poller); err != nil {
-		r.Logger.Error().Err(err).Str("poller", opt.Poller).Msgf("")
+		r.Logger.Error().Err(err).Str("poller", opt.Poller).Send()
 		return nil, err
 	}
 	if poller.Addr == "" {

cmd/collectors/restperf/plugins/nic/nic.go

@@ -74,7 +74,7 @@ func (n *Nic) Init() error {
 
 	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
 	if n.client, err = rest.New(conf.ZapiPoller(n.ParentParams), timeout, n.Auth); err != nil {
-		n.Logger.Error().Stack().Err(err).Msg("connecting")
+		n.Logger.Error().Err(err).Msg("connecting")
 		return err
 	}
 

cmd/poller/collector/asup.go

@@ -216,7 +216,7 @@ func BuildAndWriteAutoSupport(collectors []Collector, status *matrix.Matrix, pol
 		Platform: &platformInfo{
 			Arch: arch,
 			CPUs: cpus,
-			OS:   getOSName(),
+			OS:   GetOSName(),
 		},
 		Target: &TargetInfo{
 			Ping: status.LazyValueFloat64("ping", "host"),
@@ -261,12 +261,12 @@ func BuildAndWriteAutoSupport(collectors []Collector, status *matrix.Matrix, pol
 	// add harvest release info
 	msg.Harvest = &harvestInfo{
 		// harvest uuid creation from sha1 of cluster uuid
-		UUID:         sha1Sum(msg.Target.ClusterUUID),
+		UUID:         Sha1Sum(msg.Target.ClusterUUID),
 		Version:      version.VERSION,
 		Release:      version.Release,
 		Commit:       version.Commit,
 		BuildDate:    version.BuildDate,
-		HostHash:     sha1Sum(hostname),
+		HostHash:     Sha1Sum(hostname),
 		NumClusters:  1,
 		NumPollers:   uint64(len(conf.Config.Pollers)),
 		NumExporters: uint64(len(conf.Config.Exporters)),
@@ -421,7 +421,7 @@ func getCPUInfo() (string, uint8) {
 	return arch, uint8(cpuCount) // #nosec G115
 }
 
-func getOSName() string {
+func GetOSName() string {
 	info, err := host.Info()
 	if err != nil {
 		return ""
@@ -455,7 +455,7 @@ func getPayloadPath(asupDir string, pollerName string) (string, error) {
 	return path.Join(payloadDir, fmt.Sprintf("%s_%s", pollerName, "payload.json")), nil
 }
 
-func sha1Sum(s string) string {
+func Sha1Sum(s string) string {
 	hash := sha1.New() //nolint:gosec // using sha1 for a hash, not a security risk
 	hash.Write([]byte(s))
 	return hex.EncodeToString(hash.Sum(nil))

cmd/poller/collector/collector.go

@@ -18,6 +18,7 @@ package collector
 
 import (
 	"errors"
+	"fmt"
 	"github.com/netapp/harvest/v2/pkg/auth"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/logging"
@@ -26,6 +27,7 @@ import (
 	"math"
 	"math/rand"
 	"reflect"
+	"runtime/debug"
 	"strconv"
 	"strings"
 	"sync"
@@ -304,7 +306,8 @@ func (c *AbstractCollector) Start(wg *sync.WaitGroup) {
 	defer wg.Done()
 	defer func() {
 		if r := recover(); r != nil {
-			c.Logger.Error().Stack().Err(errs.New(errs.ErrPanic, "")).Any("err", r).Msg("Collector panicked")
+			err := fmt.Sprintf("%+v\n", r)
+			c.Logger.Error().Str("err", err).Bytes("stack", debug.Stack()).Msg("Collector panicked")
 		}
 	}()
 

cmd/poller/poller.go

@@ -46,6 +46,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/options"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/poller/schedule"
+	"github.com/netapp/harvest/v2/cmd/tools/rest"
 	"github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
 	"github.com/netapp/harvest/v2/pkg/auth"
 	"github.com/netapp/harvest/v2/pkg/conf"
@@ -55,6 +56,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/requests"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
+	goversion "github.com/netapp/harvest/v2/third_party/go-version"
 	"github.com/shirou/gopsutil/v4/mem"
 	"github.com/shirou/gopsutil/v4/process"
 	"github.com/spf13/cobra"
@@ -1353,6 +1355,11 @@ func (p *Poller) addMemoryMetadata() {
 }
 
 func (p *Poller) logPollerMetadata() (map[string]*matrix.Matrix, error) {
+	err := p.sendHarvestVersion()
+	if err != nil {
+		logger.Error().Err(err).Msg("Failed to send Harvest version")
+	}
+
 	rss, _ := p.status.LazyGetValueFloat64("memory.rss", "host")
 	logger.Info().
 		Float64("rssKB", rss).
@@ -1363,6 +1370,55 @@ func (p *Poller) logPollerMetadata() (map[string]*matrix.Matrix, error) {
 	return nil, nil
 }
 
+func (p *Poller) sendHarvestVersion() error {
+	var (
+		poller     *conf.Poller
+		connection *rest.Client
+		err        error
+	)
+
+	// connect to the cluster and retrieve the system version
+	if poller, err = conf.PollerNamed(opts.Poller); err != nil {
+		return err
+	}
+	timeout, _ := time.ParseDuration(rest.DefaultTimeout)
+	if connection, err = rest.New(poller, timeout, p.auth); err != nil {
+		return err
+	}
+	err = connection.Init(2)
+	if err != nil {
+		return err
+	}
+
+	// Check if the cluster is running ONTAP 9.11.1 or later
+	// If it is, send a harvestTag to the cluster to indicate that Harvest is running
+	// Otherwise, do nothing
+
+	ontapVersion, err := goversion.NewVersion(connection.Cluster().GetVersion())
+	if err != nil {
+		return err
+	}
+
+	if ontapVersion.LessThan(goversion.Must(goversion.NewVersion("9.11.1"))) {
+		return err
+	}
+
+	// Send the harvestTag to the ONTAP cluster including the OS name, sha1(hostname), Harvest version, and max RSS in MB
+	osName := collector.GetOSName()
+	hostname, _ := os.Hostname()
+	sha1Hostname := collector.Sha1Sum(hostname)
+	rssMB := p.maxRssBytes / 1024 / 1024
+	fields := []string{osName, sha1Hostname, version.VERSION, strconv.FormatUint(rssMB, 10)}
+
+	href := `api/cluster?ignore_unknown_fields=true&fields=harvestTag,` + strings.Join(fields, ",")
+	_, err = connection.GetPlainRest(href, false)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func startPoller(_ *cobra.Command, _ []string) {
 	poller := &Poller{}
 	poller.options = opts
@@ -1404,14 +1460,5 @@ func init() {
 
 // start poller, if fails try to write to syslog
 func main() {
-	// don't recover if a goroutine has panicked, instead
-	// log as much as possible
-	defer func() {
-		if r := recover(); r != nil {
-			logger.Error().Stack().Any("err", r).Msg("Poller panicked")
-			logger.Fatal().Msg(`(main) terminating abnormally, tip: run in foreground mode (with "--loglevel 0") to debug`)
-		}
-	}()
-
 	cobra.CheckErr(pollerCmd.Execute())
 }

cmd/tools/generate/counter.go

@@ -73,6 +73,48 @@ var (
 		"svm_nfs":  true,
 		"node_nfs": true,
 	}
+
+	knownDescriptionGaps = map[string]struct{}{
+		"ontaps3_object_count":                      {},
+		"security_certificate_expiry_time":          {},
+		"volume_capacity_tier_footprint":            {},
+		"volume_capacity_tier_footprint_percent":    {},
+		"volume_num_compress_attempts":              {},
+		"volume_num_compress_fail":                  {},
+		"volume_performance_tier_footprint":         {},
+		"volume_performance_tier_footprint_percent": {},
+	}
+
+	knownMappingGaps = map[string]struct{}{
+		"aggr_snapshot_inode_used_percent":                      {},
+		"aggr_space_reserved":                                   {},
+		"flexcache_blocks_requested_from_client":                {},
+		"flexcache_blocks_retrieved_from_origin":                {},
+		"flexcache_evict_rw_cache_skipped_reason_disconnected":  {},
+		"flexcache_evict_skipped_reason_config_noent":           {},
+		"flexcache_evict_skipped_reason_disconnected":           {},
+		"flexcache_evict_skipped_reason_offline":                {},
+		"flexcache_invalidate_skipped_reason_config_noent":      {},
+		"flexcache_invalidate_skipped_reason_disconnected":      {},
+		"flexcache_invalidate_skipped_reason_offline":           {},
+		"flexcache_miss_percent":                                {},
+		"flexcache_nix_retry_skipped_reason_initiator_retrieve": {},
+		"flexcache_nix_skipped_reason_config_noent":             {},
+		"flexcache_nix_skipped_reason_disconnected":             {},
+		"flexcache_nix_skipped_reason_in_progress":              {},
+		"flexcache_nix_skipped_reason_offline":                  {},
+		"flexcache_reconciled_data_entries":                     {},
+		"flexcache_reconciled_lock_entries":                     {},
+		"quota_disk_used_pct_threshold":                         {},
+		"rw_ctx_cifs_giveups":                                   {},
+		"rw_ctx_cifs_rewinds":                                   {},
+		"rw_ctx_nfs_giveups":                                    {},
+		"rw_ctx_nfs_rewinds":                                    {},
+		"rw_ctx_qos_flowcontrol":                                {},
+		"rw_ctx_qos_rewinds":                                    {},
+		"security_audit_destination_port":                       {},
+		"wafl_reads_from_pmem":                                  {},
+	}
 )
 
 type Counters struct {
@@ -688,7 +730,9 @@ func generateCounterTemplate(counters map[string]Counter, version [3]int) {
 
 		if counter.Description == "" {
 			for _, def := range counter.APIs {
-				appendRow(table, "Description", counter, def)
+				if _, ok := knownDescriptionGaps[counter.Name]; !ok {
+					appendRow(table, "Description", counter, def)
+				}
 			}
 		}
 		values = append(values, counter)
@@ -713,7 +757,9 @@ func generateCounterTemplate(counters map[string]Counter, version [3]int) {
 				// missing Rest Mapping
 				if isPrint {
 					for _, def := range counter.APIs {
-						appendRow(table, "REST", counter, def)
+						if _, ok := knownMappingGaps[counter.Name]; !ok {
+							appendRow(table, "REST", counter, def)
+						}
 					}
 				}
 			}
@@ -722,7 +768,9 @@ func generateCounterTemplate(counters map[string]Counter, version [3]int) {
 		for _, def := range counter.APIs {
 			if def.ONTAPCounter == "" {
 				for _, def := range counter.APIs {
-					appendRow(table, "Mapping", counter, def)
+					if _, ok := knownMappingGaps[counter.Name]; !ok {
+						appendRow(table, "Mapping", counter, def)
+					}
 				}
 			}
 		}
@@ -742,6 +790,10 @@ func generateCounterTemplate(counters map[string]Counter, version [3]int) {
 		panic(err)
 	}
 	fmt.Printf("Harvest metric documentation generated at %s \n", targetPath)
+
+	if table.NumLines() > 0 {
+		log.Fatalf("Issues found: refer table above")
+	}
 }
 
 func appendRow(table *tw.Table, missing string, counter Counter, def MetricDef) {

cmd/tools/grafana/dashboard_test.go

@@ -1139,7 +1139,8 @@ func checkConnectNullValues(t *testing.T, path string, data []byte) {
 			return
 		}
 		if !spanNulls.Bool() {
-			t.Errorf(`dashboard=%s panel="%s got=[%s] want=true`, dashPath, value.Get("title").String(), spanNulls.String())
+			t.Errorf(`dashboard=%s panel="%s fieldConfig.defaults.custom.spanNulls got=[%s] want=true`,
+				dashPath, value.Get("title").String(), spanNulls.String())
 		}
 	})
 }

cmd/tools/rest/client.go

@@ -114,16 +114,19 @@ func (c *Client) printRequestAndResponse(req string, response []byte) {
 	}
 }
 
-// GetRest makes a REST request to the cluster and returns a json response as a []byte
-func (c *Client) GetRest(request string) ([]byte, error) {
+// GetPlainRest makes a REST request to the cluster and returns a json response as a []byte
+func (c *Client) GetPlainRest(request string, encodeURL bool) ([]byte, error) {
 	var err error
 	if strings.Index(request, "/") == 0 {
 		request = request[1:]
 	}
-	request, err = util.EncodeURL(request)
-	if err != nil {
-		return nil, err
+	if encodeURL {
+		request, err = util.EncodeURL(request)
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	u := c.baseURL + request
 	c.request, err = requests.New("GET", u, nil)
 	if err != nil {
@@ -154,6 +157,11 @@ func (c *Client) GetRest(request string) ([]byte, error) {
 	return result, err
 }
 
+// GetRest makes a REST request to the cluster and returns a json response as a []byte
+func (c *Client) GetRest(request string) ([]byte, error) {
+	return c.GetPlainRest(request, true)
+}
+
 func (c *Client) invokeWithAuthRetry() ([]byte, error) {
 	var (
 		body []byte