feat: Harvest should support multiple poller files to allow refactori…

…ng large `harvest.yml` files (#2388)

cmd/tools/doctor/doctor.go

@@ -96,39 +96,39 @@ func doDoctorCmd(cmd *cobra.Command, _ []string) {
 }
 
 func doDoctor(path string, confPath string) {
-	contents, err := os.ReadFile(path)
-	if err != nil {
-		fmt.Printf("error reading config file. err=%+v\n", err)
-		return
-	}
 	if opts.ShouldPrintConfig {
+		contents, err := os.ReadFile(path)
+		if err != nil {
+			fmt.Printf("error reading config file. err=%+v\n", err)
+			return
+		}
 		printRedactedConfig(path, contents)
 	}
-	checkAll(path, contents, confPath)
+	checkAll(path, confPath)
 }
 
 // checkAll runs all doctor checks
 // If all checks succeed, print nothing and exit with a return code of 0
 // Otherwise, print what failed and exit with a return code of 1
-func checkAll(path string, contents []byte, confPath string) {
+func checkAll(path string, confPath string) {
 	// See https://github.com/NetApp/harvest/issues/16 for more checks to add
 	color.DetectConsole(opts.Color)
-	// Validate that the config file can be parsed
-	harvestConfig := &conf.HarvestConfig{}
-	err := yaml.Unmarshal(contents, harvestConfig)
+
+	_, err := conf.LoadHarvestConfig(path)
 	if err != nil {
 		fmt.Printf("error reading config file=[%s] %+v\n", path, err)
 		os.Exit(1)
 		return
 	}
 
+	cfg := conf.Config
 	confPaths := filepath.SplitList(confPath)
 	anyFailed := false
-	anyFailed = !checkUniquePromPorts(*harvestConfig).isValid || anyFailed
-	anyFailed = !checkPollersExportToUniquePromPorts(*harvestConfig).isValid || anyFailed
-	anyFailed = !checkExporterTypes(*harvestConfig).isValid || anyFailed
+	anyFailed = !checkUniquePromPorts(cfg).isValid || anyFailed
+	anyFailed = !checkPollersExportToUniquePromPorts(cfg).isValid || anyFailed
+	anyFailed = !checkExporterTypes(cfg).isValid || anyFailed
 	anyFailed = !checkConfTemplates(confPaths).isValid || anyFailed
-	anyFailed = !checkCollectorName(*harvestConfig).isValid || anyFailed
+	anyFailed = !checkCollectorName(cfg).isValid || anyFailed
 
 	if anyFailed {
 		os.Exit(1)

docs/configure-harvest-basic.md

@@ -63,6 +63,66 @@ Tools:
   #grafana_api_token: 'aaa-bbb-ccc-ddd'
 ```
 
+## Poller_files
+
+Harvest supports loading pollers from multiple files specified in the `Poller_files` section of your `harvest.yml` file.
+For example, the following snippet tells harvest to load pollers from all the `*.yml` files under the `configs` directory, 
+and from the `path/to/single.yml` file.
+
+Paths may be relative or absolute.
+
+```yaml
+Poller_files:
+    - configs/*.yml
+    - path/to/single.yml
+
+Pollers:
+    u2:
+        datacenter: dc-1
+```
+
+Each referenced file can contain one or more unique pollers.
+Ensure that you include the top-level `Pollers` section in these files.
+All other top-level sections will be ignored.
+For example:
+
+```yaml
+# contents of configs/00-rtp.yml
+Pollers:
+  ntap3:
+    datacenter: rtp
+
+  ntap4:
+    datacenter: rtp
+---
+# contents of configs/01-rtp.yml
+Pollers:
+  ntap5:
+    datacenter: blr
+---
+# contents of path/to/single.yml
+Pollers:
+  ntap1:
+    datacenter: dc-1
+
+  ntap2:
+    datacenter: dc-1
+```
+
+At runtime, all files will be read and combined into a single configuration.
+The example above would result in the following set of pollers, in this order.
+```yaml
+- u2
+- ntap3
+- ntap4
+- ntap5
+- ntap1
+- ntap2
+```
+
+When using glob patterns, the list of matching paths will be sorted before they are read.
+Errors will be logged for all duplicate pollers and Harvest will refuse to start.
+
 ## Configuring collectors
 
 Collectors are configured by their own configuration files ([templates](configure-templates.md)), which are stored in subdirectories

pkg/conf/conf.go

@@ -6,6 +6,7 @@ package conf
 
 import (
 	"dario.cat/mergo"
+	"errors"
 	"fmt"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
@@ -15,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 )
 
@@ -31,7 +33,7 @@ const (
 	HomeEnvVar        = "HARVEST_CONF"
 )
 
-// TestLoadHarvestConfig is used by testing code to reload a new config
+// TestLoadHarvestConfig loads a new config - used by testing code
 func TestLoadHarvestConfig(configPath string) {
 	configRead = false
 	Config = HarvestConfig{}
@@ -59,11 +61,17 @@ func ConfigPath(path string) string {
 }
 
 func LoadHarvestConfig(configPath string) (string, error) {
+	var (
+		contents   []byte
+		duplicates []error
+		err        error
+	)
+
 	configPath = ConfigPath(configPath)
 	if configRead {
 		return configPath, nil
 	}
-	contents, err := os.ReadFile(configPath)
+	contents, err = os.ReadFile(configPath)
 
 	if err != nil {
 		return "", fmt.Errorf("error reading %s err=%w", configPath, err)
@@ -73,27 +81,86 @@ func LoadHarvestConfig(configPath string) (string, error) {
 		fmt.Printf("error unmarshalling config file=[%s] %+v\n", configPath, err)
 		return "", err
 	}
+
+	for _, pat := range Config.PollerFiles {
+		fs, err := filepath.Glob(pat)
+		if err != nil {
+			return "", fmt.Errorf("error retrieving poller_files path=%s err=%w", pat, err)
+		}
+
+		sort.Strings(fs)
+
+		if len(fs) == 0 {
+			fmt.Printf("add 0 poller(s) from poller_file=%s because no matching paths\n", pat)
+			continue
+		}
+
+		for _, filename := range fs {
+			fsContents, err := os.ReadFile(filename)
+			if err != nil {
+				return "", fmt.Errorf("error reading poller_file=%s err=%w", filename, err)
+			}
+			cfg, err := unmarshalConfig(fsContents)
+			if err != nil {
+				return "", fmt.Errorf("error unmarshalling poller_file=%s err=%w", filename, err)
+			}
+			for _, pName := range cfg.PollersOrdered {
+				_, ok := Config.Pollers[pName]
+				if ok {
+					duplicates = append(duplicates, fmt.Errorf("poller name=%s from poller_file=%s is not unique", pName, filename))
+					continue
+				}
+				Config.Pollers[pName] = cfg.Pollers[pName]
+				Config.PollersOrdered = append(Config.PollersOrdered, pName)
+			}
+			fmt.Printf("add %d poller(s) from poller_file=%s\n", len(cfg.PollersOrdered), filename)
+		}
+	}
+
+	if len(duplicates) > 0 {
+		return "", errors.Join(duplicates...)
+	}
+
+	// Fix promIndex for combined pollers
+	for i, name := range Config.PollersOrdered {
+		Config.Pollers[name].promIndex = i
+	}
 	return configPath, nil
 }
 
-func DecodeConfig(contents []byte) error {
-	err := yaml.Unmarshal(contents, &Config)
-	configRead = true
+func unmarshalConfig(contents []byte) (*HarvestConfig, error) {
+	var (
+		cfg           HarvestConfig
+		orderedConfig OrderedConfig
+		err           error
+	)
+
+	err = yaml.Unmarshal(contents, &cfg)
 	if err != nil {
-		return fmt.Errorf("error unmarshalling config err: %w", err)
+		return nil, fmt.Errorf("error unmarshalling config: %w", err)
 	}
-	// Until https://github.com/go-yaml/yaml/issues/717 is fixed
-	// read the yaml again to determine poller order
-	orderedConfig := OrderedConfig{}
+
+	// Read the yaml again to determine poller order
 	err = yaml.Unmarshal(contents, &orderedConfig)
 	if err != nil {
-		return err
+		return nil, fmt.Errorf("error unmarshalling ordered config: %w", err)
 	}
-	Config.PollersOrdered = orderedConfig.Pollers.namesInOrder
+	cfg.PollersOrdered = orderedConfig.Pollers.namesInOrder
 	for i, name := range Config.PollersOrdered {
 		Config.Pollers[name].promIndex = i
 	}
 
+	return &cfg, nil
+}
+
+func DecodeConfig(contents []byte) error {
+	cfg, err := unmarshalConfig(contents)
+	configRead = true
+	if err != nil {
+		return fmt.Errorf("error unmarshalling config err: %w", err)
+	}
+	Config = *cfg
+
 	// Merge pollers and defaults
 	pollers := Config.Pollers
 	defaults := Config.Defaults
@@ -293,8 +360,8 @@ func (i *IntRange) UnmarshalYAML(node *yaml.Node) error {
 	return nil
 }
 
-// GetUniqueExporters returns the unique set of exporter types from the list of export names
-// For example: If 2 prometheus exporters are configured for a poller, the last one is returned
+// GetUniqueExporters returns the unique set of exporter types from the list of export names.
+// For example, if two prometheus exporters are configured for a poller, the last one is returned
 func GetUniqueExporters(exporterNames []string) []string {
 	var resultExporters []string
 	definedExporters := Config.Exporters
@@ -572,6 +639,7 @@ type HarvestConfig struct {
 	Tools          *Tools              `yaml:"Tools,omitempty"`
 	Exporters      map[string]Exporter `yaml:"Exporters,omitempty"`
 	Pollers        map[string]*Poller  `yaml:"Pollers,omitempty"`
+	PollerFiles    []string            `yaml:"Poller_files,omitempty"`
 	Defaults       *Poller             `yaml:"Defaults,omitempty"`
 	Admin          Admin               `yaml:"Admin,omitempty"`
 	PollersOrdered []string            // poller names in same order as yaml config

pkg/conf/conf_test.go

@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"sort"
 	"strconv"
+	"strings"
 	"testing"
 )
 
@@ -284,8 +285,7 @@ func TestNodeToPoller(t *testing.T) {
 
 func TestReadHarvestConfigFromEnv(t *testing.T) {
 	t.Helper()
-	configRead = false
-	Config = HarvestConfig{}
+	resetConfig()
 	t.Setenv(HomeEnvVar, "testdata")
 	cp, err := LoadHarvestConfig(HarvestYML)
 	if err != nil {
@@ -301,3 +301,58 @@ func TestReadHarvestConfigFromEnv(t *testing.T) {
 		t.Errorf("check if star poller exists. got=nil want=poller")
 	}
 }
+
+func resetConfig() {
+	configRead = false
+	Config = HarvestConfig{}
+}
+
+func TestMultiplePollerFiles(t *testing.T) {
+	t.Helper()
+	resetConfig()
+	_, err := LoadHarvestConfig("testdata/pollerFiles/harvest.yml")
+
+	wantNumErrs := 2
+	numErrs := strings.Count(err.Error(), "\n") + 1
+	if numErrs != wantNumErrs {
+		t.Errorf("got %d errors, want %d", numErrs, wantNumErrs)
+	}
+
+	wantNumPollers := 10
+	if len(Config.Pollers) != wantNumPollers {
+		t.Errorf("got %d pollers, want %d", len(Config.Pollers), wantNumPollers)
+	}
+
+	if len(Config.PollersOrdered) != wantNumPollers {
+		t.Errorf("got %d ordered pollers, want %d", len(Config.PollersOrdered), wantNumPollers)
+	}
+
+	wantToken := "token"
+	if Config.Tools.GrafanaAPIToken != wantToken {
+		t.Errorf("got token=%s, want token=%s", Config.Tools.GrafanaAPIToken, wantToken)
+	}
+
+	orderWanted := []string{
+		"star",
+		"netapp1",
+		"netapp2",
+		"netapp3",
+		"netapp4",
+		"netapp5",
+		"netapp6",
+		"netapp7",
+		"netapp8",
+		"moon",
+	}
+
+	for i, n := range orderWanted {
+		named, err := PollerNamed(n)
+		if err != nil {
+			t.Errorf("got no poller, want poller named=%s", n)
+			continue
+		}
+		if named.promIndex != i {
+			t.Errorf("got promIndex=%d, want promIndex=%d", named.promIndex, i)
+		}
+	}
+}

pkg/conf/testdata/pollerFiles/dup.yml

@@ -0,0 +1,4 @@
+
+Pollers:
+  star:
+    addr: localhost

pkg/conf/testdata/pollerFiles/harvest.yml

@@ -0,0 +1,16 @@
+Tools:
+  grafana_api_token: token
+
+Poller_files:
+  - testdata/pollerFiles/many/*.yml
+  - testdata/pollerFiles/single.yml
+  - testdata/pollerFiles/missing1.yml
+  - testdata/pollerFiles/missing2.yml
+  - testdata/pollerFiles/single.yml    # will cause duplicate because it is listed twice
+  - testdata/pollerFiles/dup.yml       # will cause duplicate because it contains star again
+
+Pollers:
+  star:
+    addr: localhost
+    collectors:
+      - Simple

pkg/conf/testdata/pollerFiles/many/00.yml

@@ -0,0 +1,16 @@
+Pollers:
+  netapp1:
+    datacenter: rtp
+    addr: 1.1.1.1
+  netapp2:
+    datacenter: rtp
+    addr: 1.1.1.2
+  netapp3:
+    datacenter: rtp
+    addr: 1.1.1.3
+  netapp4:
+    datacenter: rtp
+    addr: 1.1.1.4
+
+Tools:
+  grafana_api_token: ignore