feat: Harvest should support per-poller prom_ports

cmd/harvest/harvest.go

@@ -172,10 +172,10 @@ func doManageCmd(cmd *cobra.Command, args []string) {
 	case "start":
 		startAllPollers(pollersFiltered, statusesByName)
 	}
-	printTable(pollersFiltered)
+	printTable(pollersFiltered, statusesByName)
 }
 
-func printTable(filteredPollers []string) {
+func printTable(filteredPollers []string, statusesByName map[string][]*util.PollerStatus) {
 	table := tw.NewWriter(os.Stdout)
 	table.SetBorder(false)
 	table.SetAutoFormatHeaders(false)
@@ -186,7 +186,8 @@ func printTable(filteredPollers []string) {
 	}
 	table.SetColumnAlignment([]int{tw.ALIGN_LEFT, tw.ALIGN_LEFT, tw.ALIGN_RIGHT, tw.ALIGN_RIGHT, tw.ALIGN_RIGHT})
 	notRunning := &util.PollerStatus{Status: util.StatusNotRunning}
-	statusesByName := getPollersStatus()
+	disabled := &util.PollerStatus{Status: util.StatusDisabled}
+
 	for _, name := range filteredPollers {
 		var (
 			poller       *conf.Poller
@@ -196,6 +197,7 @@ func printTable(filteredPollers []string) {
 			// should never happen, ignore since this was handled earlier
 			continue
 		}
+
 		if statuses, ok := statusesByName[name]; ok {
 			// print each status, annotate extra rows with a +
 			for i, status := range statuses {
@@ -207,7 +209,11 @@ func printTable(filteredPollers []string) {
 			}
 		} else {
 			// poller not running
-			printStatus(table, opts.longStatus, poller.Datacenter, name, notRunning)
+			if poller.IsDisabled {
+				printStatus(table, opts.longStatus, poller.Datacenter, name, disabled)
+			} else {
+				printStatus(table, opts.longStatus, poller.Datacenter, name, notRunning)
+			}
 		}
 	}
 	table.Render()
@@ -230,7 +236,11 @@ func startAllPollers(pollersFiltered []string, statusesByName map[string][]*util
 				startPoller(name, promPort, opts)
 			}
 		} else {
-			// poller not already running or just stopped
+			// poller not already running, just stopped, or disabled
+			poller, _ := conf.PollerNamed(name)
+			if poller == nil || poller.IsDisabled {
+				continue
+			}
 			promPort := getPollerPrometheusPort(name, opts)
 			startPoller(name, promPort, opts)
 		}
@@ -256,7 +266,7 @@ func getPollersStatus() map[string][]*util.PollerStatus {
 		fmt.Printf("Unable to GetPollerStatuses err: %+v\n", err)
 		return nil
 	}
-	// create map of status names
+	// create a map of status names
 	for _, status := range statuses {
 		statusesByName[status.Name] = append(statusesByName[status.Name], &status) // #nosec G601
 	}

cmd/tools/doctor/doctor.go

@@ -101,7 +101,9 @@ func doDoctorCmd(cmd *cobra.Command, _ []string) {
 	pathI := conf.ConfigPath(config.Value.String())
 	confPath := confPaths.Value.String()
 	out := doDoctor(pathI)
-	fmt.Println(out)
+	if opts.ShouldPrintConfig {
+		fmt.Println(out)
+	}
 	checkAll(pathI, confPath)
 }
 
@@ -233,6 +235,7 @@ func checkAll(aPath string, confPath string) {
 	anyFailed = !checkExporterTypes(cfg).isValid || anyFailed
 	anyFailed = !checkConfTemplates(confPaths).isValid || anyFailed
 	anyFailed = !checkCollectorName(cfg).isValid || anyFailed
+	anyFailed = !checkPollerPromPorts(cfg).isValid || anyFailed
 
 	if anyFailed {
 		os.Exit(1)
@@ -275,6 +278,7 @@ func checkCollectorName(config conf.HarvestConfig) validation {
 
 	// if no collector is configured in default and poller
 	if !isDefaultCollectorExist && !isPollerCollectorExist {
+		fmt.Printf("%s: No collectors are defined. Nothing will be collected.\n", color.Colorize("Error", color.Red))
 		valid.isValid = false
 	}
 
@@ -600,6 +604,69 @@ func printRedactedConfig(aPath string, contents []byte) (*yaml.Node, error) {
 	return root, nil
 }
 
+// checkPollerPromPorts checks that
+//   - pollers that define a prom_port do so uniquely.
+//   - when a prom_port is defined, but there are no Prometheus exporters
+func checkPollerPromPorts(config conf.HarvestConfig) validation {
+	seen := make(map[int][]string)
+
+	for _, pName := range config.PollersOrdered {
+		poller := config.Pollers[pName]
+		if poller.PromPort == 0 {
+			continue
+		}
+		previous := seen[poller.PromPort]
+		previous = append(previous, pName)
+		seen[poller.PromPort] = previous
+	}
+
+	valid := validation{isValid: true}
+	for _, pollerNames := range seen {
+		if len(pollerNames) == 1 {
+			continue
+		}
+		valid.isValid = false
+		break
+	}
+
+	if !valid.isValid {
+		fmt.Printf("%s: Multiple pollers use the same prom_port.\n", color.Colorize("Error", color.Red))
+		fmt.Println("  Each poller's prom_port should be unique. Change the following pollers to use unique prom_ports:")
+
+		for port, pollerNames := range seen {
+			if len(pollerNames) == 1 {
+				continue
+			}
+			names := strings.Join(pollerNames, ", ")
+			fmt.Printf("  pollers [%s] specify the same prom_port: [%s]\n", color.Colorize(names, color.Yellow), color.Colorize(port, color.Red))
+			valid.invalid = append(valid.invalid, names)
+		}
+		fmt.Println()
+	}
+
+	// Check if there are any pollers that define a prom_port but there are no Prometheus exporters
+	if config.Exporters == nil {
+		fmt.Printf("%s: No Exporters section defined. At least one Prometheus exporter is needed for prom_port to export.\n", color.Colorize("Error", color.Red))
+		valid.invalid = append(valid.invalid, "No Prometheus exporters defined")
+		valid.isValid = false
+	} else {
+		hasPromExporter := false
+		for _, exporter := range config.Exporters {
+			if exporter.Type == "Prometheus" {
+				hasPromExporter = true
+				break
+			}
+		}
+		if !hasPromExporter {
+			fmt.Printf("%s: No Prometheus exporters defined. At least one Prometheus exporter is needed for prom_port to export.\n", color.Colorize("Error", color.Red))
+			valid.invalid = append(valid.invalid, "No Prometheus exporters defined")
+			valid.isValid = false
+		}
+	}
+
+	return valid
+}
+
 func sanitize(nodes []*yaml.Node) {
 	// Update this list when there are additional tokens to sanitize
 	sanitizeWords := []string{"username", "password", "grafana_api_token", "token",

cmd/tools/doctor/doctor_test.go

@@ -228,3 +228,15 @@ func TestExportersExist(t *testing.T) {
 		t.Errorf(`got isValid=true, want isValid=false since there is no exporters section`)
 	}
 }
+
+func TestPollerPromPorts(t *testing.T) {
+	conf.TestLoadHarvestConfig("testdata/promPortNoPromExporters.yml")
+	valid := checkPollerPromPorts(conf.Config)
+	if valid.isValid {
+		t.Errorf(`got isValid=true, want isValid=false since there are non unique prom ports`)
+	}
+
+	if len(valid.invalid) != 2 {
+		t.Errorf(`got %d invalid, want 2`, len(valid.invalid))
+	}
+}

cmd/tools/doctor/testdata/promPortNoPromExporters.yml

@@ -0,0 +1,19 @@
+Exporters:
+  influx3:
+    exporter: InfluxDB
+    url: http://localhost:123809/api/v2/write?org=harvest&bucket=harvest&precision=s
+    token: my-token
+
+Defaults:
+  datacenter: rtp
+  collectors:
+    - Rest
+
+Pollers:
+  sar:
+    addr: 10.1.1.1
+    prom_port: 3000
+
+  abc:
+    addr: 10.1.1.1
+    prom_port: 3000

cmd/tools/generate/generate.go

@@ -188,20 +188,24 @@ func generateDocker(kind int) {
 	certDirPath = asComposePath(opts.certDir)
 	filesd := make([]string, 0, len(conf.Config.PollersOrdered))
 
-	for _, v := range conf.Config.PollersOrdered {
-		port, _ := conf.GetLastPromPort(v, true)
+	for _, pollerName := range conf.Config.PollersOrdered {
+		poller, ok := conf.Config.Pollers[pollerName]
+		if !ok || poller == nil || poller.IsDisabled {
+			continue
+		}
+		port, _ := conf.GetLastPromPort(pollerName, true)
 		pollerInfo := PollerInfo{
-			ServiceName:   normalizeContainerNames(v),
-			PollerName:    v,
+			ServiceName:   normalizeContainerNames(pollerName),
+			PollerName:    pollerName,
 			ConfigFile:    configFilePath,
 			Port:          port,
 			LogLevel:      opts.loglevel,
 			Image:         opts.image,
-			ContainerName: normalizeContainerNames("poller_" + v),
+			ContainerName: normalizeContainerNames("poller_" + pollerName),
 			ShowPorts:     opts.showPorts,
 			IsFull:        kind == full,
 			CertDir:       certDirPath,
-			Mounts:        makeMounts(v),
+			Mounts:        makeMounts(pollerName),
 		}
 		pollerTemplate.Pollers = append(pollerTemplate.Pollers, pollerInfo)
 		filesd = append(filesd, fmt.Sprintf("- targets: ['%s:%d']", pollerInfo.ServiceName, pollerInfo.Port))
@@ -500,26 +504,28 @@ func generateSystemd() {
 	}
 	println("and then run " + color.Colorize("systemctl daemon-reload", color.Green))
 	writeAdminSystemd(opts.configPath)
-	// reorder list of pollers so that unix collectors are last, see https://github.com/NetApp/harvest/issues/643
 	pollers := make([]string, 0)
 	unixPollers := make([]string, 0)
-	pollers = append(pollers, conf.Config.PollersOrdered...)
-	// iterate over the pollers backwards, so we don't skip any when removing
-	for i := len(pollers) - 1; i >= 0; i-- {
-		pollerName := pollers[i]
+
+	for _, pollerName := range conf.Config.PollersOrdered {
 		poller, ok := conf.Config.Pollers[pollerName]
-		if !ok || poller == nil {
+		if !ok || poller == nil || poller.IsDisabled {
 			continue
 		}
+		// reorder list of pollers so that unix collectors are last, see https://github.com/NetApp/harvest/issues/643
 		// if unix is in the poller's list of collectors, remove it from the list of pollers
+		skipPoller := false
 		for _, c := range poller.Collectors {
 			if c.Name == "Unix" {
-				pollers = append(pollers[:i], pollers[i+1:]...)
 				unixPollers = append(unixPollers, pollerName)
-				break
+				skipPoller = true
 			}
 		}
+		if !skipPoller {
+			pollers = append(pollers, pollerName)
+		}
 	}
+
 	pollers = append(pollers, unixPollers...)
 	err = t.Execute(os.Stdout, struct {
 		Admin          string