Add waiting for prometheus target in custom prometheus and federated …

…openshift monitoring test (#742)

pkg/tests/tasks/observability/custom_prometheus_test.go

@@ -74,19 +74,29 @@ func TestCustomPrometheus(t *testing.T) {
 		enableIstioProxiesMonitoring(t, customPrometheusNs, meshNamespace, ns.Bookinfo)
 		enableAppMtlsMonitoring(t, customPrometheusNs, ns.Bookinfo)
 
+		t.LogStep("Testing if telemetry was enabled")
+		ocWaitJsonpath(t, meshNamespace, "smcp", "basic",
+			"{.status.appliedValues.istio.telemetry.enabled}", "true",
+			"Telemetry was enabled.", "Telemetry failed to enable.")
+
 		t.LogStep("Waiting for installs to complete")
 		bookinfoApp.WaitReady(t)
 
+		t.LogStep("Wait until istio targets appear in the Prometheus")
+		retry.UntilSuccess(t, func(t test.TestHelper) {
+			resp := prometheus.CustomPrometheusTargets(t, customPrometheusNs)
+			if !strings.Contains(resp, "istiod-monitor") ||
+				!strings.Contains(resp, "app-metrics-monitor-mtls") ||
+				!strings.Contains(resp, "istio-proxies-monitor") {
+				t.Error("Istio Prometheus targets istiod-monitor/app-metrics-monitor-mtls/istio-proxies-monitor are not ready")
+			}
+		})
+
 		t.LogStep("Sending request to Bookinfo app")
 		retry.UntilSuccess(t, func(t test.TestHelper) {
 			curl.Request(t, app.BookinfoProductPageURL(t, meshNamespace), nil, assert.ResponseStatus(http.StatusOK))
 		})
 
-		t.LogStep("Testing if telemetry was enabled")
-		ocWaitJsonpath(t, meshNamespace, "smcp", "basic",
-			"{.status.appliedValues.istio.telemetry.enabled}", "true",
-			"Telemetry was enabled.", "Telemetry failed to enable.")
-
 		t.LogStep("Testing if 'istio_requests_total' metric is available through Prometheus API")
 		retry.UntilSuccess(t, func(t test.TestHelper) {
 			resp := prometheus.CustomPrometheusQuery(t, customPrometheusNs,

pkg/tests/tasks/observability/federated_openshift_monitoring_test.go

@@ -17,6 +17,7 @@ package observability
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/maistra/maistra-test-tool/pkg/app"
@@ -62,16 +63,24 @@ func TestFederatedOpenShiftMonitoring(t *testing.T) {
 		oc.WaitPodsExist(t, ns.Foo)
 		oc.WaitAllPodsReady(t, ns.Foo)
 
+		t.LogStep("Apply federated metrics monitor")
+		oc.ApplyString(t, meshNamespace, federatedMonitor)
+
+		t.LogStep("Wait until istio targets appear in the Prometheus")
+		retry.UntilSuccess(t, func(t test.TestHelper) {
+			resp := prometheus.ThanosTargets(t, monitoringNs)
+			if !strings.Contains(resp, "serviceMonitor/istio-system/istio-federation") {
+				t.Error("Istio Prometheus target serviceMonitor/istio-system/istio-federation are not ready")
+			}
+		})
+
 		t.LogStep("Generate some ingress traffic")
 		oc.ApplyFile(t, ns.Foo, "https://raw.githubusercontent.com/maistra/istio/maistra-2.6/samples/httpbin/httpbin-gateway.yaml")
 		httpbinURL := fmt.Sprintf("http://%s/headers", istio.GetIngressGatewayHost(t, meshNamespace))
 		retry.UntilSuccess(t, func(t test.TestHelper) {
 			curl.Request(t, httpbinURL, nil, assert.ResponseStatus(http.StatusOK))
 		})
 
-		t.LogStep("Apply federated metrics monitor")
-		oc.ApplyString(t, meshNamespace, federatedMonitor)
-
 		t.LogStep("Check istiod metrics")
 		retry.UntilSuccess(t, func(t test.TestHelper) {
 			resp := prometheus.ThanosQuery(t, monitoringNs, `pilot_info{mesh_id="unique-mesh-id"}`)

pkg/util/prometheus/prometheus.go

@@ -46,6 +46,7 @@ type Prometheus interface {
 	WithSelector(selector string) Prometheus
 	WithContainerName(containerName string) Prometheus
 	Query(t test.TestHelper, ns string, query string) PrometheusResponse
+	Targets(t test.TestHelper, ns string) string
 }
 
 func Query(t test.TestHelper, ns string, query string) PrometheusResponse {
@@ -56,6 +57,14 @@ func CustomPrometheusQuery(t test.TestHelper, ns string, query string) Prometheu
 	return DefaultCustomPrometheus.Query(t, ns, query)
 }
 
+func CustomPrometheusTargets(t test.TestHelper, ns string) string {
+	return DefaultCustomPrometheus.Targets(t, ns)
+}
+
 func ThanosQuery(t test.TestHelper, ns string, query string) PrometheusResponse {
 	return DefaultThanos.Query(t, ns, query)
 }
+
+func ThanosTargets(t test.TestHelper, ns string) string {
+	return DefaultThanos.Targets(t, ns)
+}

pkg/util/prometheus/prometheus_struct.go

@@ -55,17 +55,15 @@ func (pi *prometheus_struct) WithContainerName(containerName string) Prometheus
 
 func (pi *prometheus_struct) Query(t test.TestHelper, ns string, query string) PrometheusResponse {
 	queryString := url.Values{"query": []string{query}}.Encode()
-	url := fmt.Sprintf(`http://localhost:9090/api/v1/query?%s`, queryString)
-	urlShellEscaped := strings.ReplaceAll(url, `'`, `'\\''`)
-
-	output := oc.Exec(t,
-		pod.MatchingSelectorFirst(pi.selector, ns), pi.containerName,
-		// comunity prometheus image doesn't have `curl`, use wget instead
-		fmt.Sprintf("wget -qO- '%s'", urlShellEscaped))
-
+	output := getPrometheusApi(t, pi, ns, fmt.Sprintf(`query?%s`, queryString))
 	return parsePrometheusResponse(t, output)
 }
 
+func (pi *prometheus_struct) Targets(t test.TestHelper, ns string) string {
+	output := getPrometheusApi(t, pi, ns, `targets?state=active`)
+	return output
+}
+
 func parsePrometheusResponse(t test.TestHelper, response string) PrometheusResponse {
 	result := &PrometheusResponse{}
 	err := json.Unmarshal([]byte(response), result)
@@ -76,3 +74,14 @@ func parsePrometheusResponse(t test.TestHelper, response string) PrometheusRespo
 
 	return *result
 }
+
+func getPrometheusApi(t test.TestHelper, pi *prometheus_struct, ns string, endpoint string) string {
+	url := fmt.Sprintf(`http://localhost:9090/api/v1/%s`, endpoint)
+	urlShellEscaped := strings.ReplaceAll(url, `'`, `'\\''`)
+
+	output := oc.Exec(t,
+		pod.MatchingSelectorFirst(pi.selector, ns), pi.containerName,
+		// comunity prometheus image doesn't have `curl`, use wget instead
+		fmt.Sprintf("wget -qO- '%s'", urlShellEscaped))
+	return output
+}