From 109e222903e76fe58eedcd39139cc3718ec08604 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Thu, 28 Sep 2023 13:48:58 -0400 Subject: [PATCH] feat: Harvest should support multiple poller files to allow refactoring large `harvest.yml` files --- cmd/tools/doctor/doctor.go | 28 +++--- docs/configure-harvest-basic.md | 60 ++++++++++++ pkg/conf/conf.go | 94 ++++++++++++++++--- pkg/conf/conf_test.go | 59 +++++++++++- pkg/conf/testdata/pollerFiles/dup.yml | 4 + pkg/conf/testdata/pollerFiles/harvest.yml | 16 ++++ pkg/conf/testdata/pollerFiles/many/00.yml | 16 ++++ pkg/conf/testdata/pollerFiles/many/b.yml | 13 +++ .../testdata/pollerFiles/many/nomatch.yaml | 16 ++++ pkg/conf/testdata/pollerFiles/single.yml | 14 +++ 10 files changed, 291 insertions(+), 29 deletions(-) create mode 100644 pkg/conf/testdata/pollerFiles/dup.yml create mode 100644 pkg/conf/testdata/pollerFiles/harvest.yml create mode 100644 pkg/conf/testdata/pollerFiles/many/00.yml create mode 100644 pkg/conf/testdata/pollerFiles/many/b.yml create mode 100644 pkg/conf/testdata/pollerFiles/many/nomatch.yaml create mode 100644 pkg/conf/testdata/pollerFiles/single.yml diff --git a/cmd/tools/doctor/doctor.go b/cmd/tools/doctor/doctor.go index ec347add7..33c753ddd 100644 --- a/cmd/tools/doctor/doctor.go +++ b/cmd/tools/doctor/doctor.go @@ -96,39 +96,39 @@ func doDoctorCmd(cmd *cobra.Command, _ []string) { } func doDoctor(path string, confPath string) { - contents, err := os.ReadFile(path) - if err != nil { - fmt.Printf("error reading config file. err=%+v\n", err) - return - } if opts.ShouldPrintConfig { + contents, err := os.ReadFile(path) + if err != nil { + fmt.Printf("error reading config file. err=%+v\n", err) + return + } printRedactedConfig(path, contents) } - checkAll(path, contents, confPath) + checkAll(path, confPath) } // checkAll runs all doctor checks // If all checks succeed, print nothing and exit with a return code of 0 // Otherwise, print what failed and exit with a return code of 1 -func checkAll(path string, contents []byte, confPath string) { +func checkAll(path string, confPath string) { // See https://github.com/NetApp/harvest/issues/16 for more checks to add color.DetectConsole(opts.Color) - // Validate that the config file can be parsed - harvestConfig := &conf.HarvestConfig{} - err := yaml.Unmarshal(contents, harvestConfig) + + _, err := conf.LoadHarvestConfig(path) if err != nil { fmt.Printf("error reading config file=[%s] %+v\n", path, err) os.Exit(1) return } + cfg := conf.Config confPaths := filepath.SplitList(confPath) anyFailed := false - anyFailed = !checkUniquePromPorts(*harvestConfig).isValid || anyFailed - anyFailed = !checkPollersExportToUniquePromPorts(*harvestConfig).isValid || anyFailed - anyFailed = !checkExporterTypes(*harvestConfig).isValid || anyFailed + anyFailed = !checkUniquePromPorts(cfg).isValid || anyFailed + anyFailed = !checkPollersExportToUniquePromPorts(cfg).isValid || anyFailed + anyFailed = !checkExporterTypes(cfg).isValid || anyFailed anyFailed = !checkConfTemplates(confPaths).isValid || anyFailed - anyFailed = !checkCollectorName(*harvestConfig).isValid || anyFailed + anyFailed = !checkCollectorName(cfg).isValid || anyFailed if anyFailed { os.Exit(1) diff --git a/docs/configure-harvest-basic.md b/docs/configure-harvest-basic.md index e135a4e89..d926daec4 100644 --- a/docs/configure-harvest-basic.md +++ b/docs/configure-harvest-basic.md @@ -63,6 +63,66 @@ Tools: #grafana_api_token: 'aaa-bbb-ccc-ddd' ``` +## Poller_files + +Harvest supports loading pollers from multiple files specified in the `Poller_files` section of your `harvest.yml` file. +For example, the following snippet tells harvest to load pollers from all the `*.yml` files under the `configs` directory, +and from the `path/to/single.yml` file. + +Paths may be relative or absolute. + +```yaml +Poller_files: + - configs/*.yml + - path/to/single.yml + +Pollers: + u2: + datacenter: dc-1 +``` + +Each referenced file can contain one or more unique pollers. +Ensure that you include the top-level `Pollers` section in these files. +All other top-level sections will be ignored. +For example: + +```yaml +# contents of configs/00-rtp.yml +Pollers: + ntap3: + datacenter: rtp + + ntap4: + datacenter: rtp +--- +# contents of configs/01-rtp.yml +Pollers: + ntap5: + datacenter: blr +--- +# contents of path/to/single.yml +Pollers: + ntap1: + datacenter: dc-1 + + ntap2: + datacenter: dc-1 +``` + +At runtime, all files will be read and combined into a single configuration. +The example above would result in the following set of pollers, in this order. +```yaml +- u2 +- ntap3 +- ntap4 +- ntap5 +- ntap1 +- ntap2 +``` + +When using glob patterns, the list of matching paths will be sorted before they are read. +Errors will be logged for all duplicate pollers and Harvest will refuse to start. + ## Configuring collectors Collectors are configured by their own configuration files ([templates](configure-templates.md)), which are stored in subdirectories diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index ae14956f0..64c52cbdc 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -6,6 +6,7 @@ package conf import ( "dario.cat/mergo" + "errors" "fmt" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/tree/node" @@ -15,6 +16,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strconv" ) @@ -31,7 +33,7 @@ const ( HomeEnvVar = "HARVEST_CONF" ) -// TestLoadHarvestConfig is used by testing code to reload a new config +// TestLoadHarvestConfig loads a new config - used by testing code func TestLoadHarvestConfig(configPath string) { configRead = false Config = HarvestConfig{} @@ -59,11 +61,17 @@ func ConfigPath(path string) string { } func LoadHarvestConfig(configPath string) (string, error) { + var ( + contents []byte + duplicates []error + err error + ) + configPath = ConfigPath(configPath) if configRead { return configPath, nil } - contents, err := os.ReadFile(configPath) + contents, err = os.ReadFile(configPath) if err != nil { return "", fmt.Errorf("error reading %s err=%w", configPath, err) @@ -73,27 +81,86 @@ func LoadHarvestConfig(configPath string) (string, error) { fmt.Printf("error unmarshalling config file=[%s] %+v\n", configPath, err) return "", err } + + for _, pat := range Config.PollerFiles { + fs, err := filepath.Glob(pat) + if err != nil { + return "", fmt.Errorf("error retrieving poller_files path=%s err=%w", pat, err) + } + + sort.Strings(fs) + + if len(fs) == 0 { + fmt.Printf("add 0 poller(s) from poller_file=%s because no matching paths\n", pat) + continue + } + + for _, filename := range fs { + fsContents, err := os.ReadFile(filename) + if err != nil { + return "", fmt.Errorf("error reading poller_file=%s err=%w", filename, err) + } + cfg, err := unmarshalConfig(fsContents) + if err != nil { + return "", fmt.Errorf("error unmarshalling poller_file=%s err=%w", filename, err) + } + for _, pName := range cfg.PollersOrdered { + _, ok := Config.Pollers[pName] + if ok { + duplicates = append(duplicates, fmt.Errorf("poller name=%s from poller_file=%s is not unique", pName, filename)) + continue + } + Config.Pollers[pName] = cfg.Pollers[pName] + Config.PollersOrdered = append(Config.PollersOrdered, pName) + } + fmt.Printf("add %d poller(s) from poller_file=%s\n", len(cfg.PollersOrdered), filename) + } + } + + if len(duplicates) > 0 { + return "", errors.Join(duplicates...) + } + + // Fix promIndex for combined pollers + for i, name := range Config.PollersOrdered { + Config.Pollers[name].promIndex = i + } return configPath, nil } -func DecodeConfig(contents []byte) error { - err := yaml.Unmarshal(contents, &Config) - configRead = true +func unmarshalConfig(contents []byte) (*HarvestConfig, error) { + var ( + cfg HarvestConfig + orderedConfig OrderedConfig + err error + ) + + err = yaml.Unmarshal(contents, &cfg) if err != nil { - return fmt.Errorf("error unmarshalling config err: %w", err) + return nil, fmt.Errorf("error unmarshalling config: %w", err) } - // Until https://github.com/go-yaml/yaml/issues/717 is fixed - // read the yaml again to determine poller order - orderedConfig := OrderedConfig{} + + // Read the yaml again to determine poller order err = yaml.Unmarshal(contents, &orderedConfig) if err != nil { - return err + return nil, fmt.Errorf("error unmarshalling ordered config: %w", err) } - Config.PollersOrdered = orderedConfig.Pollers.namesInOrder + cfg.PollersOrdered = orderedConfig.Pollers.namesInOrder for i, name := range Config.PollersOrdered { Config.Pollers[name].promIndex = i } + return &cfg, nil +} + +func DecodeConfig(contents []byte) error { + cfg, err := unmarshalConfig(contents) + configRead = true + if err != nil { + return fmt.Errorf("error unmarshalling config err: %w", err) + } + Config = *cfg + // Merge pollers and defaults pollers := Config.Pollers defaults := Config.Defaults @@ -293,8 +360,8 @@ func (i *IntRange) UnmarshalYAML(node *yaml.Node) error { return nil } -// GetUniqueExporters returns the unique set of exporter types from the list of export names -// For example: If 2 prometheus exporters are configured for a poller, the last one is returned +// GetUniqueExporters returns the unique set of exporter types from the list of export names. +// For example, if two prometheus exporters are configured for a poller, the last one is returned func GetUniqueExporters(exporterNames []string) []string { var resultExporters []string definedExporters := Config.Exporters @@ -572,6 +639,7 @@ type HarvestConfig struct { Tools *Tools `yaml:"Tools,omitempty"` Exporters map[string]Exporter `yaml:"Exporters,omitempty"` Pollers map[string]*Poller `yaml:"Pollers,omitempty"` + PollerFiles []string `yaml:"Poller_files,omitempty"` Defaults *Poller `yaml:"Defaults,omitempty"` Admin Admin `yaml:"Admin,omitempty"` PollersOrdered []string // poller names in same order as yaml config diff --git a/pkg/conf/conf_test.go b/pkg/conf/conf_test.go index 5e5a49da2..51d30b234 100644 --- a/pkg/conf/conf_test.go +++ b/pkg/conf/conf_test.go @@ -5,6 +5,7 @@ import ( "reflect" "sort" "strconv" + "strings" "testing" ) @@ -284,8 +285,7 @@ func TestNodeToPoller(t *testing.T) { func TestReadHarvestConfigFromEnv(t *testing.T) { t.Helper() - configRead = false - Config = HarvestConfig{} + resetConfig() t.Setenv(HomeEnvVar, "testdata") cp, err := LoadHarvestConfig(HarvestYML) if err != nil { @@ -301,3 +301,58 @@ func TestReadHarvestConfigFromEnv(t *testing.T) { t.Errorf("check if star poller exists. got=nil want=poller") } } + +func resetConfig() { + configRead = false + Config = HarvestConfig{} +} + +func TestMultiplePollerFiles(t *testing.T) { + t.Helper() + resetConfig() + _, err := LoadHarvestConfig("testdata/pollerFiles/harvest.yml") + + wantNumErrs := 2 + numErrs := strings.Count(err.Error(), "\n") + 1 + if numErrs != wantNumErrs { + t.Errorf("got %d errors, want %d", numErrs, wantNumErrs) + } + + wantNumPollers := 10 + if len(Config.Pollers) != wantNumPollers { + t.Errorf("got %d pollers, want %d", len(Config.Pollers), wantNumPollers) + } + + if len(Config.PollersOrdered) != wantNumPollers { + t.Errorf("got %d ordered pollers, want %d", len(Config.PollersOrdered), wantNumPollers) + } + + wantToken := "token" + if Config.Tools.GrafanaAPIToken != wantToken { + t.Errorf("got token=%s, want token=%s", Config.Tools.GrafanaAPIToken, wantToken) + } + + orderWanted := []string{ + "star", + "netapp1", + "netapp2", + "netapp3", + "netapp4", + "netapp5", + "netapp6", + "netapp7", + "netapp8", + "moon", + } + + for i, n := range orderWanted { + named, err := PollerNamed(n) + if err != nil { + t.Errorf("got no poller, want poller named=%s", n) + continue + } + if named.promIndex != i { + t.Errorf("got promIndex=%d, want promIndex=%d", named.promIndex, i) + } + } +} diff --git a/pkg/conf/testdata/pollerFiles/dup.yml b/pkg/conf/testdata/pollerFiles/dup.yml new file mode 100644 index 000000000..7c1c5be71 --- /dev/null +++ b/pkg/conf/testdata/pollerFiles/dup.yml @@ -0,0 +1,4 @@ + +Pollers: + star: + addr: localhost diff --git a/pkg/conf/testdata/pollerFiles/harvest.yml b/pkg/conf/testdata/pollerFiles/harvest.yml new file mode 100644 index 000000000..8e744cc64 --- /dev/null +++ b/pkg/conf/testdata/pollerFiles/harvest.yml @@ -0,0 +1,16 @@ +Tools: + grafana_api_token: token + +Poller_files: + - testdata/pollerFiles/many/*.yml + - testdata/pollerFiles/single.yml + - testdata/pollerFiles/missing1.yml + - testdata/pollerFiles/missing2.yml + - testdata/pollerFiles/single.yml # will cause duplicate because it is listed twice + - testdata/pollerFiles/dup.yml # will cause duplicate because it contains star again + +Pollers: + star: + addr: localhost + collectors: + - Simple diff --git a/pkg/conf/testdata/pollerFiles/many/00.yml b/pkg/conf/testdata/pollerFiles/many/00.yml new file mode 100644 index 000000000..06761669f --- /dev/null +++ b/pkg/conf/testdata/pollerFiles/many/00.yml @@ -0,0 +1,16 @@ +Pollers: + netapp1: + datacenter: rtp + addr: 1.1.1.1 + netapp2: + datacenter: rtp + addr: 1.1.1.2 + netapp3: + datacenter: rtp + addr: 1.1.1.3 + netapp4: + datacenter: rtp + addr: 1.1.1.4 + +Tools: + grafana_api_token: ignore diff --git a/pkg/conf/testdata/pollerFiles/many/b.yml b/pkg/conf/testdata/pollerFiles/many/b.yml new file mode 100644 index 000000000..08a8bae2d --- /dev/null +++ b/pkg/conf/testdata/pollerFiles/many/b.yml @@ -0,0 +1,13 @@ +Pollers: + netapp5: + datacenter: blr + addr: 1.1.1.5 + netapp6: + datacenter: blr + addr: 1.1.1.6 + netapp7: + datacenter: blr + addr: 1.1.1.7 + netapp8: + datacenter: blr + addr: 1.1.1.8 diff --git a/pkg/conf/testdata/pollerFiles/many/nomatch.yaml b/pkg/conf/testdata/pollerFiles/many/nomatch.yaml new file mode 100644 index 000000000..06761669f --- /dev/null +++ b/pkg/conf/testdata/pollerFiles/many/nomatch.yaml @@ -0,0 +1,16 @@ +Pollers: + netapp1: + datacenter: rtp + addr: 1.1.1.1 + netapp2: + datacenter: rtp + addr: 1.1.1.2 + netapp3: + datacenter: rtp + addr: 1.1.1.3 + netapp4: + datacenter: rtp + addr: 1.1.1.4 + +Tools: + grafana_api_token: ignore diff --git a/pkg/conf/testdata/pollerFiles/single.yml b/pkg/conf/testdata/pollerFiles/single.yml new file mode 100644 index 000000000..fa45e5fab --- /dev/null +++ b/pkg/conf/testdata/pollerFiles/single.yml @@ -0,0 +1,14 @@ + +Poller_files: # these will be ignored since they are in single.yml + - testdata/pollerFiles/many/*.yml + - testdata/pollerFiles/single.yml + - testdata/pollerFiles/missing1.yml + - testdata/pollerFiles/missing2.yml + - testdata/pollerFiles/missing3.yml + - testdata/pollerFiles/single.yml + +Pollers: + moon: + addr: localhost + collectors: + - Simple