feat: Harvest should support recording and replaying HTTP requests (#…

…3235) * feat: Harvest should support recording and replaying HTTP requests * feat: Harvest should support recording and replaying HTTP requests * feat: Harvest should support recording and replaying HTTP requests
NetApp · Nov 4, 2024 · 1fc13b0 · 1fc13b0
1 parent 83a8527
commit 1fc13b0
Show file tree

Hide file tree

Showing 17 changed files with 531 additions and 108 deletions.
diff --git a/cmd/collectors/keyperf/keyperf.go b/cmd/collectors/keyperf/keyperf.go
@@ -5,6 +5,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/rest"
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/slogx"
@@ -20,8 +21,10 @@ const (
 )
 
 type KeyPerf struct {
-	*rest.Rest // provides: AbstractCollector, Client, Object, Query, TemplateFn, TemplateType
-	perfProp   *perfProp
+	*rest.Rest    // provides: AbstractCollector, Client, Object, Query, TemplateFn, TemplateType
+	perfProp      *perfProp
+	pollDataCalls int
+	recordsToSave int // Number of records to save when using the recorder
 }
 
 type counter struct {
@@ -91,6 +94,8 @@ func (kp *KeyPerf) Init(a *collector.AbstractCollector) error {
 
 	kp.buildCounters()
 
+	kp.recordsToSave = collector.RecordKeepLast(kp.Params, kp.Logger)
+
 	kp.Logger.Debug(
 		"initialized cache",
 		slog.Int("numMetrics", len(kp.Prop.Metrics)),
@@ -234,7 +239,25 @@ func (kp *KeyPerf) PollData() (map[string]*matrix.Matrix, error) {
 		return nil, errs.New(errs.ErrConfig, "empty url")
 	}
 
-	perfRecords, err = kp.GetRestData(href)
+	kp.pollDataCalls++
+	if kp.pollDataCalls >= kp.recordsToSave {
+		kp.pollDataCalls = 0
+	}
+
+	var headers map[string]string
+
+	poller, err := conf.PollerNamed(kp.Options.Poller)
+	if err != nil {
+		slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", kp.Options.Poller))
+	}
+
+	if poller.IsRecording() {
+		headers = map[string]string{
+			"From": strconv.Itoa(kp.pollDataCalls),
+		}
+	}
+
+	perfRecords, err = kp.GetRestData(href, headers)
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch href=%s %w", href, err)
 	}

diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go
@@ -679,13 +679,13 @@ func (r *Rest) HandleResults(mat *matrix.Matrix, result []gjson.Result, prop *pr
 	return count, numPartials
 }
 
-func (r *Rest) GetRestData(href string) ([]gjson.Result, error) {
+func (r *Rest) GetRestData(href string, headers ...map[string]string) ([]gjson.Result, error) {
 	r.Logger.Debug("Fetching data", slog.String("href", href))
 	if href == "" {
 		return nil, errs.New(errs.ErrConfig, "empty url")
 	}
 
-	result, err := rest.FetchAll(r.Client, href)
+	result, err := rest.FetchAll(r.Client, href, headers...)
 	if err != nil {
 		return r.handleError(err)
 	}

diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go
@@ -18,6 +18,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/tools/rest"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
@@ -46,17 +47,15 @@ const (
 )
 
 var (
-	constituentRegex = regexp.MustCompile(`^(.*)__(\d{4})$`)
+	constituentRegex      = regexp.MustCompile(`^(.*)__(\d{4})$`)
+	qosQuery              = "api/cluster/counter/tables/qos"
+	qosVolumeQuery        = "api/cluster/counter/tables/qos_volume"
+	qosDetailQuery        = "api/cluster/counter/tables/qos_detail"
+	qosDetailVolumeQuery  = "api/cluster/counter/tables/qos_detail_volume"
+	qosWorkloadQuery      = "api/storage/qos/workloads"
+	workloadDetailMetrics = []string{"resource_latency"}
 )
 
-var qosQuery = "api/cluster/counter/tables/qos"
-var qosVolumeQuery = "api/cluster/counter/tables/qos_volume"
-var qosDetailQuery = "api/cluster/counter/tables/qos_detail"
-var qosDetailVolumeQuery = "api/cluster/counter/tables/qos_detail_volume"
-var qosWorkloadQuery = "api/storage/qos/workloads"
-
-var workloadDetailMetrics = []string{"resource_latency"}
-
 var qosQueries = map[string]string{
 	qosQuery:       qosQuery,
 	qosVolumeQuery: qosVolumeQuery,
@@ -71,6 +70,9 @@ type RestPerf struct {
 	perfProp            *perfProp
 	archivedMetrics     map[string]*rest2.Metric // Keeps metric definitions that are not found in the counter schema. These metrics may be available in future ONTAP versions.
 	hasInstanceSchedule bool
+	pollInstanceCalls   int
+	pollDataCalls       int
+	recordsToSave       int // Number of records to save when using the recorder
 }
 
 type counter struct {
@@ -147,6 +149,8 @@ func (r *RestPerf) Init(a *collector.AbstractCollector) error {
 
 	r.InitSchedule()
 
+	r.recordsToSave = collector.RecordKeepLast(r.Params, r.Logger)
+
 	r.Logger.Debug(
 		"initialized cache",
 		slog.Int("numMetrics", len(r.Prop.Metrics)),
@@ -721,7 +725,26 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 		return nil, errs.New(errs.ErrConfig, "empty url")
 	}
 
-	err = rest.FetchRestPerfData(r.Client, href, &perfRecords)
+	r.pollDataCalls++
+	if r.pollDataCalls >= r.recordsToSave {
+		r.pollDataCalls = 0
+	}
+
+	var headers map[string]string
+
+	poller, err := conf.PollerNamed(r.Options.Poller)
+	if err != nil {
+		slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", r.Options.Poller))
+	}
+
+	if poller.IsRecording() {
+		headers = map[string]string{
+			"From": strconv.Itoa(r.pollDataCalls),
+		}
+	}
+
+	err = rest.FetchRestPerfData(r.Client, href, &perfRecords, headers)
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch href=%s %w", href, err)
 	}
@@ -1476,6 +1499,24 @@ func (r *RestPerf) PollInstance() (map[string]*matrix.Matrix, error) {
 		}
 	}
 
+	r.pollInstanceCalls++
+	if r.pollInstanceCalls > r.recordsToSave/3 {
+		r.pollInstanceCalls = 0
+	}
+
+	var headers map[string]string
+
+	poller, err := conf.PollerNamed(r.Options.Poller)
+	if err != nil {
+		slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", r.Options.Poller))
+	}
+
+	if poller.IsRecording() {
+		headers = map[string]string{
+			"From": strconv.Itoa(r.pollInstanceCalls),
+		}
+	}
+
 	href := rest.NewHrefBuilder().
 		APIPath(dataQuery).
 		Fields([]string{fields}).
@@ -1491,7 +1532,7 @@ func (r *RestPerf) PollInstance() (map[string]*matrix.Matrix, error) {
 
 	apiT := time.Now()
 	r.Client.Metadata.Reset()
-	records, err = rest.FetchAll(r.Client, href)
+	records, err = rest.FetchAll(r.Client, href, headers)
 	if err != nil {
 		return r.handleError(err, href)
 	}

diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go
@@ -75,7 +75,7 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie
 	var (
 		client     Client
 		httpclient *http.Client
-		transport  *http.Transport
+		transport  http.RoundTripper
 		addr       string
 		href       string
 		err        error
@@ -96,7 +96,7 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie
 	client.baseURL = href
 	client.Timeout = timeout
 
-	transport, err = c.Transport(nil)
+	transport, err = c.Transport(nil, poller)
 	if err != nil {
 		return nil, err
 	}

diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go
@@ -19,24 +19,22 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/systemnode"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/volume"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/workload"
+	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	client "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
 	"github.com/netapp/harvest/v2/pkg/conf"
+	"github.com/netapp/harvest/v2/pkg/errs"
+	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
 	"github.com/netapp/harvest/v2/pkg/slogx"
+	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
 	"log/slog"
 	"slices"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/netapp/harvest/v2/cmd/poller/collector"
-	"github.com/netapp/harvest/v2/pkg/errs"
-	"github.com/netapp/harvest/v2/pkg/matrix"
-	"github.com/netapp/harvest/v2/pkg/tree/node"
-
-	client "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
 )
 
 const BatchSize = "500"