From 4f96e22eb9f142779f332c146446f430a705a632 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 11 Nov 2024 15:16:46 -0500 Subject: [PATCH] feat: Harvest should support per-poller prom_ports feat: Harvest should support disabled pollers doc: Explain how prom_port and port range options for the Prometheus exporter work --- cmd/harvest/harvest.go | 22 +- cmd/tools/doctor/doctor.go | 69 ++++++- cmd/tools/doctor/doctor_test.go | 12 ++ .../testdata/promPortNoPromExporters.yml | 19 ++ cmd/tools/generate/generate.go | 34 +-- docs/prometheus-exporter.md | 194 ++++++++++++++---- harvest.cue | 2 + pkg/conf/conf.go | 51 ++++- pkg/conf/conf_test.go | 62 +++++- pkg/conf/testdata/prom_ports.yml | 32 +++ pkg/util/util.go | 1 + 11 files changed, 421 insertions(+), 77 deletions(-) create mode 100644 cmd/tools/doctor/testdata/promPortNoPromExporters.yml create mode 100644 pkg/conf/testdata/prom_ports.yml diff --git a/cmd/harvest/harvest.go b/cmd/harvest/harvest.go index 02a3cea44..361b15550 100644 --- a/cmd/harvest/harvest.go +++ b/cmd/harvest/harvest.go @@ -172,10 +172,10 @@ func doManageCmd(cmd *cobra.Command, args []string) { case "start": startAllPollers(pollersFiltered, statusesByName) } - printTable(pollersFiltered) + printTable(pollersFiltered, statusesByName) } -func printTable(filteredPollers []string) { +func printTable(filteredPollers []string, statusesByName map[string][]*util.PollerStatus) { table := tw.NewWriter(os.Stdout) table.SetBorder(false) table.SetAutoFormatHeaders(false) @@ -186,7 +186,8 @@ func printTable(filteredPollers []string) { } table.SetColumnAlignment([]int{tw.ALIGN_LEFT, tw.ALIGN_LEFT, tw.ALIGN_RIGHT, tw.ALIGN_RIGHT, tw.ALIGN_RIGHT}) notRunning := &util.PollerStatus{Status: util.StatusNotRunning} - statusesByName := getPollersStatus() + disabled := &util.PollerStatus{Status: util.StatusDisabled} + for _, name := range filteredPollers { var ( poller *conf.Poller @@ -196,6 +197,7 @@ func printTable(filteredPollers []string) { // should never happen, ignore since this was handled earlier continue } + if statuses, ok := statusesByName[name]; ok { // print each status, annotate extra rows with a + for i, status := range statuses { @@ -207,7 +209,11 @@ func printTable(filteredPollers []string) { } } else { // poller not running - printStatus(table, opts.longStatus, poller.Datacenter, name, notRunning) + if poller.IsDisabled { + printStatus(table, opts.longStatus, poller.Datacenter, name, disabled) + } else { + printStatus(table, opts.longStatus, poller.Datacenter, name, notRunning) + } } } table.Render() @@ -230,7 +236,11 @@ func startAllPollers(pollersFiltered []string, statusesByName map[string][]*util startPoller(name, promPort, opts) } } else { - // poller not already running or just stopped + // poller not already running, just stopped, or disabled + poller, _ := conf.PollerNamed(name) + if poller == nil || poller.IsDisabled { + continue + } promPort := getPollerPrometheusPort(name, opts) startPoller(name, promPort, opts) } @@ -256,7 +266,7 @@ func getPollersStatus() map[string][]*util.PollerStatus { fmt.Printf("Unable to GetPollerStatuses err: %+v\n", err) return nil } - // create map of status names + // create a map of status names for _, status := range statuses { statusesByName[status.Name] = append(statusesByName[status.Name], &status) // #nosec G601 } diff --git a/cmd/tools/doctor/doctor.go b/cmd/tools/doctor/doctor.go index ff0fdc1ef..77876ad6c 100644 --- a/cmd/tools/doctor/doctor.go +++ b/cmd/tools/doctor/doctor.go @@ -101,7 +101,9 @@ func doDoctorCmd(cmd *cobra.Command, _ []string) { pathI := conf.ConfigPath(config.Value.String()) confPath := confPaths.Value.String() out := doDoctor(pathI) - fmt.Println(out) + if opts.ShouldPrintConfig { + fmt.Println(out) + } checkAll(pathI, confPath) } @@ -233,6 +235,7 @@ func checkAll(aPath string, confPath string) { anyFailed = !checkExporterTypes(cfg).isValid || anyFailed anyFailed = !checkConfTemplates(confPaths).isValid || anyFailed anyFailed = !checkCollectorName(cfg).isValid || anyFailed + anyFailed = !checkPollerPromPorts(cfg).isValid || anyFailed if anyFailed { os.Exit(1) @@ -275,6 +278,7 @@ func checkCollectorName(config conf.HarvestConfig) validation { // if no collector is configured in default and poller if !isDefaultCollectorExist && !isPollerCollectorExist { + fmt.Printf("%s: No collectors are defined. Nothing will be collected.\n", color.Colorize("Error", color.Red)) valid.isValid = false } @@ -600,6 +604,69 @@ func printRedactedConfig(aPath string, contents []byte) (*yaml.Node, error) { return root, nil } +// checkPollerPromPorts checks that +// - pollers that define a prom_port do so uniquely. +// - when a prom_port is defined, but there are no Prometheus exporters +func checkPollerPromPorts(config conf.HarvestConfig) validation { + seen := make(map[int][]string) + + for _, pName := range config.PollersOrdered { + poller := config.Pollers[pName] + if poller.PromPort == 0 { + continue + } + previous := seen[poller.PromPort] + previous = append(previous, pName) + seen[poller.PromPort] = previous + } + + valid := validation{isValid: true} + for _, pollerNames := range seen { + if len(pollerNames) == 1 { + continue + } + valid.isValid = false + break + } + + if !valid.isValid { + fmt.Printf("%s: Multiple pollers use the same prom_port.\n", color.Colorize("Error", color.Red)) + fmt.Println(" Each poller's prom_port should be unique. Change the following pollers to use unique prom_ports:") + + for port, pollerNames := range seen { + if len(pollerNames) == 1 { + continue + } + names := strings.Join(pollerNames, ", ") + fmt.Printf(" pollers [%s] specify the same prom_port: [%s]\n", color.Colorize(names, color.Yellow), color.Colorize(port, color.Red)) + valid.invalid = append(valid.invalid, names) + } + fmt.Println() + } + + // Check if there are any pollers that define a prom_port but there are no Prometheus exporters + if config.Exporters == nil { + fmt.Printf("%s: No Exporters section defined. At least one Prometheus exporter is needed for prom_port to export.\n", color.Colorize("Error", color.Red)) + valid.invalid = append(valid.invalid, "No Prometheus exporters defined") + valid.isValid = false + } else { + hasPromExporter := false + for _, exporter := range config.Exporters { + if exporter.Type == "Prometheus" { + hasPromExporter = true + break + } + } + if !hasPromExporter { + fmt.Printf("%s: No Prometheus exporters defined. At least one Prometheus exporter is needed for prom_port to export.\n", color.Colorize("Error", color.Red)) + valid.invalid = append(valid.invalid, "No Prometheus exporters defined") + valid.isValid = false + } + } + + return valid +} + func sanitize(nodes []*yaml.Node) { // Update this list when there are additional tokens to sanitize sanitizeWords := []string{"username", "password", "grafana_api_token", "token", diff --git a/cmd/tools/doctor/doctor_test.go b/cmd/tools/doctor/doctor_test.go index b10033628..0a3d76261 100644 --- a/cmd/tools/doctor/doctor_test.go +++ b/cmd/tools/doctor/doctor_test.go @@ -228,3 +228,15 @@ func TestExportersExist(t *testing.T) { t.Errorf(`got isValid=true, want isValid=false since there is no exporters section`) } } + +func TestPollerPromPorts(t *testing.T) { + conf.TestLoadHarvestConfig("testdata/promPortNoPromExporters.yml") + valid := checkPollerPromPorts(conf.Config) + if valid.isValid { + t.Errorf(`got isValid=true, want isValid=false since there are non unique prom ports`) + } + + if len(valid.invalid) != 2 { + t.Errorf(`got %d invalid, want 2`, len(valid.invalid)) + } +} diff --git a/cmd/tools/doctor/testdata/promPortNoPromExporters.yml b/cmd/tools/doctor/testdata/promPortNoPromExporters.yml new file mode 100644 index 000000000..7c585e76b --- /dev/null +++ b/cmd/tools/doctor/testdata/promPortNoPromExporters.yml @@ -0,0 +1,19 @@ +Exporters: + influx3: + exporter: InfluxDB + url: http://localhost:123809/api/v2/write?org=harvest&bucket=harvest&precision=s + token: my-token + +Defaults: + datacenter: rtp + collectors: + - Rest + +Pollers: + sar: + addr: 10.1.1.1 + prom_port: 3000 + + abc: + addr: 10.1.1.1 + prom_port: 3000 diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go index daa9209b4..6384a1e7f 100644 --- a/cmd/tools/generate/generate.go +++ b/cmd/tools/generate/generate.go @@ -188,20 +188,24 @@ func generateDocker(kind int) { certDirPath = asComposePath(opts.certDir) filesd := make([]string, 0, len(conf.Config.PollersOrdered)) - for _, v := range conf.Config.PollersOrdered { - port, _ := conf.GetLastPromPort(v, true) + for _, pollerName := range conf.Config.PollersOrdered { + poller, ok := conf.Config.Pollers[pollerName] + if !ok || poller == nil || poller.IsDisabled { + continue + } + port, _ := conf.GetLastPromPort(pollerName, true) pollerInfo := PollerInfo{ - ServiceName: normalizeContainerNames(v), - PollerName: v, + ServiceName: normalizeContainerNames(pollerName), + PollerName: pollerName, ConfigFile: configFilePath, Port: port, LogLevel: opts.loglevel, Image: opts.image, - ContainerName: normalizeContainerNames("poller_" + v), + ContainerName: normalizeContainerNames("poller_" + pollerName), ShowPorts: opts.showPorts, IsFull: kind == full, CertDir: certDirPath, - Mounts: makeMounts(v), + Mounts: makeMounts(pollerName), } pollerTemplate.Pollers = append(pollerTemplate.Pollers, pollerInfo) filesd = append(filesd, fmt.Sprintf("- targets: ['%s:%d']", pollerInfo.ServiceName, pollerInfo.Port)) @@ -500,26 +504,28 @@ func generateSystemd() { } println("and then run " + color.Colorize("systemctl daemon-reload", color.Green)) writeAdminSystemd(opts.configPath) - // reorder list of pollers so that unix collectors are last, see https://github.com/NetApp/harvest/issues/643 pollers := make([]string, 0) unixPollers := make([]string, 0) - pollers = append(pollers, conf.Config.PollersOrdered...) - // iterate over the pollers backwards, so we don't skip any when removing - for i := len(pollers) - 1; i >= 0; i-- { - pollerName := pollers[i] + + for _, pollerName := range conf.Config.PollersOrdered { poller, ok := conf.Config.Pollers[pollerName] - if !ok || poller == nil { + if !ok || poller == nil || poller.IsDisabled { continue } + // reorder list of pollers so that unix collectors are last, see https://github.com/NetApp/harvest/issues/643 // if unix is in the poller's list of collectors, remove it from the list of pollers + skipPoller := false for _, c := range poller.Collectors { if c.Name == "Unix" { - pollers = append(pollers[:i], pollers[i+1:]...) unixPollers = append(unixPollers, pollerName) - break + skipPoller = true } } + if !skipPoller { + pollers = append(pollers, pollerName) + } } + pollers = append(pollers, unixPollers...) err = t.Execute(os.Stdout, struct { Admin string diff --git a/docs/prometheus-exporter.md b/docs/prometheus-exporter.md index 6d118dfc3..683419bc5 100644 --- a/docs/prometheus-exporter.md +++ b/docs/prometheus-exporter.md @@ -18,14 +18,33 @@ A web end-point is required because Prometheus scrapes Harvest by polling that e In addition to the `/metrics` end-point, the Prometheus exporter also serves an overview of all metrics and collectors available on its root address `scheme://:/`. -Because Prometheus polls Harvest, don't forget +Because Prometheus polls Harvest, remember to [update your Prometheus configuration](#configure-prometheus-to-scrape-harvest-pollers) and tell Prometheus how to scrape each poller. -There are two ways to configure the Prometheus exporter: using a `port range` or individual `port`s. +## How should I configure the Prometheus exporter? -The `port range` is more flexible and should be used when you want multiple pollers all exporting to the same instance -of Prometheus. Both options are explained below. +There are several ways to configure the Prometheus exporter with various trade-offs outlined below: + +- a per-poller prom_port in the `Pollers` section +- a port range in the `Exporters` section +- a single port in the `Exporters` section +- embedded exporters in the `Pollers` section + +We recommend the first two options, using a per-poller `prom_port` or `port_range`, since they work for the majority of +use cases and are the easiest to manage. + +Use `prom_port` when you want the most control over the Prometheus port with the least amount of management. +Use `port_range` when you want Harvest to manage the port numbers for you, +but be aware that the port numbers depend on the order of pollers in your `harvest.yml`. +You need to keep that order consistent otherwise it will appear that you have lost data. + +| Name | Pros | Cons | Notes | +|--------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [prom_port](#per-poller-prom_port) | Precisely control each Poller's Prometheus exporter port. The port is defined in one place, beside each poller. | You have to manage which port each poller should use. | Start with this until you outgrow it. Many folks never do. | +| [port_range](#port-range) | Less to manage since Harvest assigns the port numbers for you based on the order of the pollers in your `harvest.yml` | You need to be mindful of the order of pollers in your `harvest.yml` file and be careful about changing that order when adding/removing pollers. Reordering will cause the Prometheus port to change. Since Prometheus includes the port in the `instance` label, changing the port causes Prometheus to treat metrics with different ports as different instances. That means it will appear that you have lost data because the metrics with the new port are distinct from the metrics with the older port. See [#2782](https://github.com/NetApp/harvest/issues/2782) for details. | Less to manage, but makes sure you understand how to [control the order of your pollers](#port-range). | +| [port in Exporters](#single-port-exporter) | Precisely control each Poller's Prometheus exporter port.
Can define multiple Prometheus exporters, each with custom configuration. | Similar to `prom_port` but with an unnecessary level of indirection that makes you repeat yourself. | Exporter that Harvest always shipped with. Most folks should use `prom_port` unless they need to configure many instances of the Prometheus exporter, which is rare. | +| [embedded exporter](#embedded-exporter) | All the pros of `port in Exporters` but without the unnecessary indirection | Removes the level of indirection and allows you to define the exporter in one place, but more verbose than per-poller `prom_port`. | Most folks should use `prom_port` unless they need to configure many instances of the Prometheus exporter, which is rare. | ## Parameters @@ -33,66 +52,153 @@ All parameters of the exporter are defined in the `Exporters` section of `harves An overview of all parameters: -| parameter | type | description | default | -|-----------------------------|------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| -| `port_range` | int-int (range), overrides `port` if specified | lower port to upper port (inclusive) of the HTTP end-point to create when a poller specifies this exporter. Starting at lower port, each free port will be tried sequentially up to the upper port. | | -| `port` | int, required if port_range is not specified | port of the HTTP end-point | | -| `local_http_addr` | string, optional | address of the HTTP server Harvest starts for Prometheus to scrape:
use `localhost` to serve only on the local machine
use `0.0.0.0` (default) if Prometheus is scrapping from another machine | `0.0.0.0` | -| `global_prefix` | string, optional | add a prefix to all metrics (e.g. `netapp_`) | | -| `allow_addrs` | list of strings, optional | allow access only if host matches any of the provided addresses | | -| `allow_addrs_regex` | list of strings, optional | allow access only if host address matches at least one of the regular expressions | | -| `cache_max_keep` | string (Go duration format), optional | maximum amount of time metrics are cached (in case Prometheus does not timely collect the metrics) | `5m` | -| `add_meta_tags` | bool, optional | add `HELP` and `TYPE` [metatags](https://prometheus.io/docs/instrumenting/exposition_formats/#comments-help-text-and-type-information) to metrics (currently no useful information, but required by some tools) | `false` | -| `sort_labels` | bool, optional | sort metric labels before exporting. Some [open-metrics scrapers report](https://github.com/NetApp/harvest/issues/756) stale metrics when labels are not sorted. | `false` | -| `tls` | `tls` | optional | If present, enables TLS transport. If running in a container, see [note](https://github.com/NetApp/harvest/issues/672#issuecomment-1036338589) | -| tls `cert_file`, `key_file` | **required** child of `tls` | Relative or absolute path to TLS certificate and key file. TLS 1.3 certificates required.
FIPS complaint P-256 TLS 1.3 certificates can be created with `bin/harvest admin tls create server`, `openssl`, `mkcert`, etc. | | +| parameter | type | description | default | +|-------------------------------------------|------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| `add_meta_tags` | bool, optional | add `HELP` and `TYPE` [metatags](https://prometheus.io/docs/instrumenting/exposition_formats/#comments-help-text-and-type-information) to metrics (currently no useful information, but required by some tools) | `false` | +| [`allow_addrs`](#allow_addrs) | list of strings, optional | allow access only if host matches any of the provided addresses | | +| [`allow_addrs_regex`](#allow_addrs_regex) | list of strings, optional | allow access only if host address matches at least one of the regular expressions | | +| `cache_max_keep` | string (Go duration format), optional | maximum amount of time metrics are cached (in case Prometheus does not timely collect the metrics) | `5m` | +| `global_prefix` | string, optional | add a prefix to all metrics (e.g. `netapp_`) | | +| `local_http_addr` | string, optional | address of the HTTP server Harvest starts for Prometheus to scrape:
use `localhost` to serve only on the local machine
use `0.0.0.0` (default) if Prometheus is scrapping from another machine | `0.0.0.0` | +| `port_range` | int-int (range), overrides `port` if specified | lower port to upper port (inclusive) of the HTTP end-point to create when a poller specifies this exporter. Starting at lower port, each free port will be tried sequentially up to the upper port. | | +| `port` | int, required if port_range is not specified | port of the HTTP end-point | | +| `sort_labels` | bool, optional | sort metric labels before exporting. [VictoriaMetrics](https://github.com/NetApp/harvest/issues/756) requires this otherwise stale metrics are reported. | `false` | +| `tls` | `tls` | optional | If present, enables TLS transport. If running in a container, see [note](https://github.com/NetApp/harvest/issues/672#issuecomment-1036338589) | +| tls `cert_file`, `key_file` | **required** child of `tls` | Relative or absolute path to TLS certificate and key file. TLS 1.3 certificates required.
FIPS complaint P-256 TLS 1.3 certificates can be created with `bin/harvest admin tls create server`, `openssl`, `mkcert`, etc. | | + +### Per-poller prom_port + +Define a Prometheus exporter in the `Exporters` section of your `harvest.yml` file, use that exporter in `Defaults`, +and then each poller lists its `prom_port` in the `Pollers` section. -A few examples: +```yaml +Exporters: + my_prom: + exporter: Prometheus + add_meta_tags: true + sort_labels: true -#### port_range +Defaults: + auth_style: basic_auth + username: harvest + password: pass + exporters: + - my_prom +``` + +Then update your pollers in the `Pollers` section of your `harvest.yml` file. + +```yaml +Pollers: + cluster-01: + addr: 10.0.1.1 + prom_port: 12990 + cluster-02: + addr: 10.0.1.2 + prom_port: 12991 +``` + + +### Port Range + +Port range works by defining a range of ports in the `Exporters` section of your `harvest.yml` file. +Harvest will assign the first available port in the range to each poller that uses the exporter. +That means you need to be careful about the order of your pollers in the `harvest.yml` file. + +If you add or remove pollers, the order of the pollers may change, and the port assigned to each poller may change. +To mitigate this: + +- when you add new pollers to the `harvest.yml` file, add them to the end of the `Pollers` section of your `harvest.yml` file. +- when you want to remove pollers, instead of deleting them, add the `disabled: true` parameter to the poller. +The poller will not be started, but the port will be reserved. +That way, the order of later pollers won't change. ```yaml Exporters: prom-prod: exporter: Prometheus port_range: 2000-2030 +Defaults: + exporters: + - prom-prod Pollers: cluster-01: - exporters: - - prom-prod + addr: 10.0.1.1 + disabled: true # This poller will not be used cluster-02: - exporters: - - prom-prod + addr: 10.0.1.2 cluster-03: - exporters: - - prom-prod + addr: 10.0.1.3 # ... more cluster-16: - exporters: - - prom-prod + addr: 10.0.1.16 ``` -Sixteen pollers will collect metrics from 16 clusters and make those metrics available to a single instance of -Prometheus named `prom-prod`. Sixteen web end-points will be created on the first 16 available free ports between 2000 -and 2030 (inclusive). +In the example above, fifteen pollers will collect metrics from 15 clusters +and make those metrics available to a single instance of Prometheus named `prom-prod`. +Fifteen web end-points will be created on the available free ports between 2000 and 2030 (inclusive). +Port 2000 will be assigned to `cluster-01`, port 2001 to `cluster-02`, and so on. -After staring the pollers in the example above, running `bin/harvest status` shows the following. Note that ports 2000 -and 2003 were not available so the next free port in the range was selected. If no free port can be found an error will +After starting the pollers in the example above, running `bin/harvest status` shows the following. +Since `cluster-01` is disabled, it won't be started and its port will be skipped. +If no free port can be found, an error will be logged. ``` -Datacenter Poller PID PromPort Status -++++++++++++ ++++++++++++ +++++++ +++++++++ ++++++++++++++++++++ -DC-01 cluster-01 2339 2001 running -DC-01 cluster-02 2343 2002 running -DC-01 cluster-03 2351 2004 running + Datacenter | Poller | PID | PromPort | Status +--------------+--------------+------+----------+---------- + dc-01 | cluster-01 | 2339 | | disabled + dc-01 | cluster-02 | 2343 | 2001 | running + dc-01 | cluster-03 | 2351 | 2002 | running ... -DC-01 cluster-14 2405 2015 running -DC-01 cluster-15 2502 2016 running -DC-01 cluster-16 2514 2017 running + dc-01 | cluster-14 | 2405 | 2013 | running + dc-01 | cluster-15 | 2502 | 2014 | running + dc-01 | cluster-16 | 2514 | 2015 | running +``` + +### Single Port Exporter + +Define a Prometheus exporter in the `Exporters` section of your `harvest.yml` file. +Give that exporter a `port` and update a single poller to use this exporter. +Each poller requires a different Prometheus exporter. + +```yaml +Exporters: + my_prom: + exporter: Prometheus + add_meta_tags: true + sort_labels: true + port: 12990 +``` + +Then update a single poller in the `Pollers` section of your `harvest.yml` file to reference the Prometheus exporter. + +```yaml +Pollers: + cluster-01: + addr: 10.0.1.1 + exporters: + - my_prom +``` + +### Embedded Exporter + +This example is similar to the [single port exporter](#single-port-exporter) example, +but the exporter is defined in the `Pollers` section. +No need to define the exporter in the `Exporters` section. + +```yaml +Pollers: + cluster-01: + addr: 10.0.1.1 + exporters: + - exporter: Prometheus + add_meta_tags: true + sort_labels: true + port: 12990 ``` -#### allow_addrs +### allow_addrs ```yaml Exporters: @@ -102,9 +208,9 @@ Exporters: - 192.168.0.103 ``` -will only allow access from exactly these two addresses. +Access will only be allowed from these two addresses. -#### allow_addrs_regex +### allow_addrs_regex ```yaml Exporters: @@ -113,7 +219,7 @@ Exporters: - `^192.168.0.\d+$` ``` -will only allow access from the IP4 range `192.168.0.0`-`192.168.0.255`. +Access will only be allowed from the IP4 range `192.168.0.0`-`192.168.0.255`. ## Configure Prometheus to scrape Harvest pollers diff --git a/harvest.cue b/harvest.cue index 304b7ce19..e5325c0bd 100644 --- a/harvest.cue +++ b/harvest.cue @@ -85,6 +85,7 @@ Pollers: [Name=_]: #Poller credentials_file?: string credentials_script?: #CredentialsScript datacenter?: string + disabled?: bool exporters: [...#ExporterDefs] is_kfs?: bool labels?: [...label] @@ -94,6 +95,7 @@ Pollers: [Name=_]: #Poller password?: string poller_log_schedule?: string prefer_zapi?: bool + prom_port?: int recorder?: #Recorder ssl_cert?: string ssl_key?: string diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index cfa6f86b3..59ae90687 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -327,10 +327,14 @@ func Path(aPath string) string { } // GetLastPromPort returns the Prometheus port for the given poller -// If multiple Prometheus exporters are configured for a poller, the port for the last exporter is returned. +// If a poller has multiple Prometheus exporters in its `exporters` section, +// the port for the last exporter in the list is used. func GetLastPromPort(pollerName string, validatePortInUse bool) (int, error) { - var port int - var isPrometheusExporterConfigured bool + var ( + port int + isPrometheusExporterConfigured bool + preferredPort int + ) if len(promPortRangeMapping) == 0 { loadPrometheusExporterPortRangeMapping(validatePortInUse) @@ -341,34 +345,48 @@ func GetLastPromPort(pollerName string, validatePortInUse bool) (int, error) { } exporters := poller.Exporters +exporter: for i := len(exporters) - 1; i >= 0; i-- { e := exporters[i] exporter := Config.Exporters[e] if exporter.Type == "Prometheus" { isPrometheusExporterConfigured = true - if exporter.PortRange != nil { + switch { + case exporter.PortRange != nil: ports := promPortRangeMapping[e] - preferredPort := exporter.PortRange.Min + poller.promIndex + if poller.PromPort == 0 { + preferredPort = exporter.PortRange.Min + poller.promIndex + } else { + port = poller.PromPort + delete(ports.freePorts, port) + break exporter + } _, isFree := ports.freePorts[preferredPort] if isFree { port = preferredPort delete(ports.freePorts, preferredPort) - break + break exporter } for k := range ports.freePorts { port = k delete(ports.freePorts, k) - break + break exporter } - } else if exporter.Port != nil && *exporter.Port != 0 { + // This case is checked before the next one because PromPort wins over an embedded exporter + case poller.PromPort != 0: + port = poller.PromPort + break exporter + case exporter.Port != nil && *exporter.Port != 0: port = *exporter.Port - break + break exporter } } } + if port == 0 && isPrometheusExporterConfigured { return port, errs.New(errs.ErrConfig, "No free port found for poller "+pollerName) } + return port, nil } @@ -537,6 +555,7 @@ type Poller struct { CredentialsFile string `yaml:"credentials_file,omitempty"` CredentialsScript CredentialsScript `yaml:"credentials_script,omitempty"` Datacenter string `yaml:"datacenter,omitempty"` + IsDisabled bool `yaml:"disabled,omitempty"` ExporterDefs []ExporterDef `yaml:"exporters,omitempty"` Exporters []string `yaml:"-"` IsKfs bool `yaml:"is_kfs,omitempty"` @@ -548,6 +567,7 @@ type Poller struct { PollerLogSchedule string `yaml:"poller_log_schedule,omitempty"` PollerSchedule string `yaml:"poller_schedule,omitempty"` PreferZAPI bool `yaml:"prefer_zapi,omitempty"` + PromPort int `yaml:"prom_port,omitempty"` Recorder Recorder `yaml:"recorder,omitempty"` SslCert string `yaml:"ssl_cert,omitempty"` SslKey string `yaml:"ssl_key,omitempty"` @@ -562,24 +582,35 @@ type Poller struct { // For all keys in default, copy them to the poller if the poller does not already include them func (p *Poller) Union(defaults *Poller) { // this is needed because of how mergo handles boolean zero values + var ( + pUseInsecureTLS bool + ) + isInsecureNil := true - var pUseInsecureTLS bool + pIsKfs := p.IsKfs + pIsDisabled := p.IsDisabled + if p.UseInsecureTLS != nil { isInsecureNil = false pUseInsecureTLS = *p.UseInsecureTLS } + // Don't copy auth related fields from defaults to poller, even when the poller is missing those fields. // Save a copy of the poller's auth fields and restore after merge pPassword := p.Password pAuthStyle := p.AuthStyle pCredentialsFile := p.CredentialsFile pCredentialsScript := p.CredentialsScript.Path + _ = mergo.Merge(p, defaults) + if !isInsecureNil { p.UseInsecureTLS = &pUseInsecureTLS } + p.IsKfs = pIsKfs + p.IsDisabled = pIsDisabled p.Password = pPassword p.AuthStyle = pAuthStyle p.CredentialsFile = pCredentialsFile diff --git a/pkg/conf/conf_test.go b/pkg/conf/conf_test.go index 81ffe4b4b..eb72cb44f 100644 --- a/pkg/conf/conf_test.go +++ b/pkg/conf/conf_test.go @@ -494,8 +494,8 @@ func TestEmbeddedExporter(t *testing.T) { } uniqueExporters := GetUniqueExporters(p.Exporters) - want := []string{"u2-1", "u2-2})"} - if slices.Equal(uniqueExporters, want) { + want := []string{"u2-1", "u2-2"} + if !slices.Equal(uniqueExporters, want) { t.Errorf("got %v, want %v", uniqueExporters, want) } @@ -508,3 +508,61 @@ func TestEmbeddedExporter(t *testing.T) { t.Errorf("got port=%d, want port=32990", port) } } + +// TestPromPort tests the prom_port configuration +// - If there are multiple Prometheus exporters defined for a poller, pick the last one. +// (see GetUniqueExporters) +// - If there is an embedded exporter, prom_port wins +// If the embedded exporter is the last one in the list, it will be picked (per the above rule), but the +// prom_port will be used instead of any port defined in the embedded exporter. +func TestPromPort(t *testing.T) { + t.Helper() + resetConfig() + + configYaml := "testdata/prom_ports.yml" + _, err := LoadHarvestConfig(configYaml) + if err != nil { + t.Fatalf("got error loading config: %s, want no errors", err) + } + + p, err := PollerNamed("sar") + if err != nil { + t.Fatalf("got no poller, want poller named=u2") + } + if len(p.Exporters) != 2 { + t.Errorf("got %d exporters, want 2", len(p.Exporters)) + } + + // Ensure that the last exporter is used + uniqueExporters := GetUniqueExporters(p.Exporters) + want := []string{"prometheus1"} + if !slices.Equal(uniqueExporters, want) { + t.Errorf("got %v, want %v", uniqueExporters, want) + } + + port, err := GetLastPromPort("sar", false) + if err != nil { + t.Fatalf("got error: %v, want no error", err) + } + if port != 3000 { + t.Errorf("got port=%d, want port=3000", port) + } + + // Ensure that the prom_port is used instead of the port defined in the embedded exporter + port, err = GetLastPromPort("u3", false) + if err != nil { + t.Fatalf("got error: %v, want no error", err) + } + if port != 9999 { + t.Errorf("got port=%d, want port=9999", port) + } + + // Ensure that zero is returned if the poller does not have an exporter + port, err = GetLastPromPort("no-exporter", false) + if err != nil { + t.Fatalf("got error: %v, want no error", err) + } + if port != 0 { + t.Errorf("got port=%d, want port=0", port) + } +} diff --git a/pkg/conf/testdata/prom_ports.yml b/pkg/conf/testdata/prom_ports.yml new file mode 100644 index 000000000..ecb748300 --- /dev/null +++ b/pkg/conf/testdata/prom_ports.yml @@ -0,0 +1,32 @@ +Exporters: + prometheus0: + exporter: Prometheus + addr: 0.0.0.0 + port: 4444 + prometheus1: + exporter: Prometheus + addr: 0.0.0.0 + port_range: 2000-2030 + +Defaults: + collectors: + - Rest + +Pollers: + no-exporter: + addr: 10.193.48.11 + prom_port: 3000 + + sar: + addr: 10.193.48.11 + prom_port: 3000 + exporters: + - prometheus0 + - prometheus1 + + u3: + addr: 10.0.1.1 + prom_port: 9999 + exporters: + - exporter: Prometheus + port: 32990 diff --git a/pkg/util/util.go b/pkg/util/util.go index b44ff2631..92d971011 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -238,6 +238,7 @@ const ( StatusNotRunning Status = "not running" StatusKilled Status = "killed" StatusAlreadyExited Status = "already exited" + StatusDisabled Status = "disabled" ) type PollerStatus struct {