ci: Harvest should lint metrics with promtool (#3246)

Makefile

@@ -177,4 +177,6 @@ endif
 	VERSION=${VERSION} INSTALL_DOCKER=1 ./integration/test/test.sh
 	VERSION=${VERSION} REGRESSION=1 ./integration/test/test.sh
 	VERSION=${VERSION} ANALYZE_DOCKER_LOGS=1 ./integration/test/test.sh
+	VERSION=${VERSION} CHECK_METRICS=1 ./integration/test/test.sh
+
 

cmd/collectors/zapi/plugins/systemnode/systemnode.go

@@ -68,10 +68,6 @@ func (s *SystemNode) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *
 		inst.SetLabel("bmc_firmware_version", serviceProcessorMap[nodeName])
 	}
 
-	// update node metrics with partner info
-	for _, metric := range data.GetMetrics() {
-		metric.SetLabel("ha_partner", partnerNameMap[metric.GetLabel("node")])
-	}
 	return nil, s.client.Metadata, nil
 }
 

integration/Jenkinsfile

@@ -207,6 +207,7 @@ pipeline {
                                         zip -r docker_logs.zip /var/log/harvest
                                         cd $WORKSPACE/docker
                                         ANALYZE_DOCKER_LOGS=1 bash $WORKSPACE/harvest/integration/test/test.sh
+                                        CHECK_METRICS=1 bash $WORKSPACE/harvest/integration/test/test.sh
                                         docker ps -q | xargs docker stop | xargs docker rm --force
                                     """
                                     archiveArtifacts artifacts: "docker_logs.zip", fingerprint: true
@@ -267,6 +268,7 @@ pipeline {
                                         zip -r docker_fips_logs.zip /var/log/harvest
                                         cd $WORKSPACE/docker
                                         ANALYZE_DOCKER_LOGS=1 bash $WORKSPACE/harvest/integration/test/test.sh
+                                        CHECK_METRICS=1 bash $WORKSPACE/harvest/integration/test/test.sh
                                         docker ps -q | xargs docker stop | xargs docker rm --force
                                     """
                                     archiveArtifacts artifacts: "docker_fips_logs.zip", fingerprint: true

integration/test/promtool_metrics_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"github.com/Netapp/harvest-automation/test/utils"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestPrometheusMetrics(t *testing.T) {
+	utils.SkipIfMissing(t, utils.CheckMetrics)
+
+	ports := []int{12990, 12992, 12993, 12994}
+	for _, port := range ports {
+		checkMetrics(t, port)
+	}
+}
+
+func checkMetrics(t *testing.T, port int) {
+	cli := fmt.Sprintf(`curl -s http://localhost:%d/metrics | tee /tmp/metrics:%d.txt | promtool check metrics`, port, port)
+	command := exec.Command("bash", "-c", cli)
+	output, err := command.CombinedOutput()
+
+	if err != nil {
+		var ee *exec.ExitError
+		if !errors.As(err, &ee) {
+			// An exit code can't be used since we need to ignore metrics that are not valid but can't change
+			t.Errorf("ERR checking metrics cli=%s err=%v output=%s", cli, err, string(output))
+			return
+		}
+	}
+
+	if len(output) == 0 {
+		return
+	}
+
+	// Read the output, line by line, and check for errors, non-errors are ignored
+
+	lines := strings.Split(string(output), "\n")
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		if strings.Contains(line, "error while linting: ") {
+			t.Errorf("promtool: %s", line)
+		}
+	}
+}

integration/test/utils/utils.go

@@ -29,6 +29,7 @@ const (
 	GrafanaTokeKey = "grafana_api_token"
 
 	AnalyzeDockerLogs = "ANALYZE_DOCKER_LOGS"
+	CheckMetrics      = "CHECK_METRICS"
 	CopyDockerLogs    = "COPY_DOCKER_LOGS"
 	InstallDocker     = "INSTALL_DOCKER"
 	InstallNative     = "INSTALL_NATIVE"