From baca9eedcbb84407081b127fc0f4491407070377 Mon Sep 17 00:00:00 2001
From: Grace Wehner <grace.wehner@microsoft.com>
Date: Mon, 29 Jan 2024 10:39:46 -0800
Subject: [PATCH] fix bugs, add test for node-exporter

---
 .../containerstatus/container_status_test.go  | 19 ++++++++++++---
 otelcollector/test/utils/constants.go         |  2 +-
 otelcollector/test/utils/utils.go             | 24 +++++++++++++++++--
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/otelcollector/test/containerstatus/container_status_test.go b/otelcollector/test/containerstatus/container_status_test.go
index 4cd3222ae..c9a97a0c4 100644
--- a/otelcollector/test/containerstatus/container_status_test.go
+++ b/otelcollector/test/containerstatus/container_status_test.go
@@ -7,6 +7,18 @@ import (
 	. "github.com/onsi/gomega"
 )
 
+var _ = DescribeTable("The containers should be running",
+  func(namespace string, controllerLabelName string, controllerLabelValue string) {
+		err := utils.CheckIfAllContainersAreRunning(K8sClient, namespace, controllerLabelName, controllerLabelValue)
+		Expect(err).NotTo(HaveOccurred())
+	},
+	Entry("when checking the ama-metrics replica pod(s)", "kube-system", "rsName", "ama-metrics"),
+	Entry("when checking the ama-metrics-node", "kube-system", "dsName", "ama-metrics-node"),
+	Entry("when checking the ama-metrics-ksm pod", "kube-system", "app.kubernetes.io/name", "ama-metrics-ksm"),
+	Entry("when checking the ama-metrics-operator-targets pod", "kube-system", "rsName", "ama-metrics-operator-targets", Label("operator")),
+	Entry("when checking the prometheus-node-exporter pod", "kube-system", "app", "prometheus-node-exporter", Label("arc-extension")),
+)
+
 var _ = DescribeTable("All processes are running",
 	func(namespace, labelName, labelValue, containerName string, processes []string) {
 		err := utils.CheckAllProcessesRunning(K8sClient, Cfg, labelName, labelValue, namespace, containerName, processes)
@@ -24,7 +36,7 @@ var _ = DescribeTable("All processes are running",
 			"crond",
 		},
 	),
-	Entry("when checking the ama-metrics-node daemonset pods", "kube-system", "dsName", "ama-metrics", "prometheus-collector",
+	Entry("when checking the ama-metrics-node daemonset pods", "kube-system", "dsName", "ama-metrics-node", "prometheus-collector",
 	  []string {
 			"fluent-bit",
 			"telegraf",
@@ -45,6 +57,7 @@ var _ = DescribeTable("The container logs should not contain errors",
 	},
 	Entry("when checking the ama-metrics replica pods", "kube-system", "rsName", "ama-metrics"),
 	Entry("when checking the ama-metrics-node", "kube-system", "dsName", "ama-metrics-node"),
-	Entry("when checking the ama-metrics-operator-targets pod", "kube-system", "rsName", "ama-metrics-operator-targets"),
-	Entry("when checking the ama-metrics-ksm pod", "kube-system", "rsName", "ama-metrics-ksm"),
+	Entry("when checking the ama-metrics-ksm pod", "kube-system", "app.kubernetes.io/name", "ama-metrics-ksm"),
+	Entry("when checking the ama-metrics-operator-targets pod", "kube-system", "rsName", "ama-metrics-operator-targets", Label("operator")),
+	Entry("when checking the prometheus-node-exporter pod", "kube-system", "app", "prometheus-node-exporter", Label("arc-extension")),
 )
diff --git a/otelcollector/test/utils/constants.go b/otelcollector/test/utils/constants.go
index d1061f768..76fbf4737 100644
--- a/otelcollector/test/utils/constants.go
+++ b/otelcollector/test/utils/constants.go
@@ -1,9 +1,9 @@
 package utils
 
 var(
-
 	// Slices can't be constants
 	LogLineErrorsToExclude = [...]string{
 		"\"filepath\":\"/MetricsExtensionConsoleDebugLog.log\"",
+		"create or renew cluster identity error"
 	}
 )
\ No newline at end of file
diff --git a/otelcollector/test/utils/utils.go b/otelcollector/test/utils/utils.go
index 6dccd69b8..67e91362b 100644
--- a/otelcollector/test/utils/utils.go
+++ b/otelcollector/test/utils/utils.go
@@ -83,7 +83,6 @@ func CheckContainerLogsForErrors(clientset *kubernetes.Clientset, namespace, lab
 						return fmt.Errorf("Logs for container %s in pod %s contain errors:\n %s", container.Name, pod.Name, line)
 					}
 				}
-				return fmt.Errorf("Logs for container %s in pod %s contain errors", container.Name, pod.Name)
 			}
 		}
 	}
@@ -97,7 +96,7 @@ func GetPodsWithLabel(clientset *kubernetes.Clientset, namespace string, labelKe
 	if err != nil {
 		return nil, err
 	}
-	if podList == nil {
+	if podList == nil || len(podList.Items) == 0{
 		return nil, fmt.Errorf("no pods found with label %s=%s", labelKey, labelValue)
 	}
 
@@ -244,3 +243,24 @@ func CheckLivenessProbeRestartForProcess(K8sClient *kubernetes.Clientset, Cfg *r
 
 	return nil
 }
+
+func CheckIfAllContainersAreRunning(clientset *kubernetes.Clientset, namespace, labelKey string, labelValue string) (error) {
+	pods, err := GetPodsWithLabel(clientset, namespace, labelKey, labelValue)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Error getting pods with the specified labels: %v", err))
+	}
+
+	for _, pod := range pods {
+		if pod.Status.Phase != corev1.PodRunning {
+			return errors.New(fmt.Sprintf("Pod is not runinng. Phase is: %v", pod.Status.Phase))
+		}
+
+		for _, containerStatus := range pod.Status.ContainerStatuses {
+			if containerStatus.State.Running == nil {
+				return errors.New(fmt.Sprintf("Container %s is not running", containerStatus.Name))
+			}
+		}
+	}
+
+	return nil
+}