From 23a58472d37998bee212161b7d6e9455954e49ae Mon Sep 17 00:00:00 2001
From: Sivanantham Chinnaiyan <sivanantham.chinnaiyan@ideas2it.com>
Date: Wed, 6 Mar 2024 18:47:23 +0530
Subject: [PATCH] Only display running pod logs in CI

Signed-off-by: Sivanantham Chinnaiyan <sivanantham.chinnaiyan@ideas2it.com>
---
 .../reconcilers/ingress/ingress_reconciler.go             | 4 +++-
 test/scripts/gh-actions/status-check.sh                   | 8 ++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go
index 88400bf55e3..e67cd75d168 100644
--- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go
+++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go
@@ -483,7 +483,7 @@ func probeIngress(url string) (bool, error) {
 	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil)
 	if err != nil {
-		return isReady, errors.Wrapf(err, "failed to probe ingress %s", target)
+		return isReady, errors.Wrapf(err, "failed to create ingress probe request %s", target)
 	}
 	// ProbeKey is the name of a header that can be added to requests to probe the ingress.
 	// Requests with this header will not be passed to the user container or included in request metrics.
@@ -497,6 +497,8 @@ func probeIngress(url string) (bool, error) {
 	}
 	if resp.StatusCode == http.StatusOK {
 		isReady = true
+	} else {
+		log.V(1).Error(fmt.Errorf("failed to probe ingress"), "Failed to probe ingress", "url", target)
 	}
 	return isReady, nil
 }
diff --git a/test/scripts/gh-actions/status-check.sh b/test/scripts/gh-actions/status-check.sh
index bdd28387b73..ee74918c0c5 100755
--- a/test/scripts/gh-actions/status-check.sh
+++ b/test/scripts/gh-actions/status-check.sh
@@ -31,7 +31,7 @@ kubectl logs -l control-plane=kserve-controller-manager -n kserve -c manager --t
 echo "::endgroup::"
 
 echo "::group::Predictor Pod logs"
-for pod in $(kubectl get pods -l 'component in (predictor)' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
+for pod in $(kubectl get pods -l 'component in (predictor)' --field-selector=status.phase!=Terminating -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
     echo "=====================================  Logs for Predictor Pod: $pod  ========================================="
     kubectl logs "$pod" -c kserve-container -n kserve-ci-e2e-test --tail 500
     echo "================================================================================================================"
@@ -39,7 +39,7 @@ done
 echo "::endgroup::"
 
 echo "::group::Transformer Pod logs"
-for pod in $(kubectl get pods -l 'component in (transformer)' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
+for pod in $(kubectl get pods -l 'component in (transformer)' --field-selector=status.phase!=Terminating -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
     echo "=====================================  Logs for Transformer Pod: $pod  ======================================="
     kubectl logs "$pod" -c kserve-container -n kserve-ci-e2e-test --tail 500
     echo "================================================================================================================"
@@ -47,7 +47,7 @@ done
 echo "::endgroup::"
 
 echo "::group::Explainer Pod logs"
-for pod in $(kubectl get pods -l 'component in (explainer)' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
+for pod in $(kubectl get pods -l 'component in (explainer)' --field-selector=status.phase!=Terminating -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
     echo "=====================================  Logs for Explainer Pod: $pod  ========================================="
     kubectl logs "$pod" -c kserve-container -n kserve-ci-e2e-test --tail 500
     echo "================================================================================================================"
@@ -55,7 +55,7 @@ done
 echo "::endgroup::"
 
 echo "::group::InferenceGraph Pod logs"
-for pod in $(kubectl get pods -l 'serving.kserve.io/inferencegraph=model-chainer' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
+for pod in $(kubectl get pods -l 'serving.kserve.io/inferencegraph=model-chainer' --field-selector=status.phase!=Terminating -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do
     echo "=====================================  Logs for Graph Pod: $pod  ========================================="
     kubectl logs "$pod" -c user-container -n kserve-ci-e2e-test --tail 500
     echo "================================================================================================================"