From 7b8cfc3249f2ed0b00793d79c75ecd96a6d8dd2c Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Wed, 6 Mar 2024 18:47:23 +0530 Subject: [PATCH] Only display running pod logs in CI Signed-off-by: Sivanantham Chinnaiyan --- .github/workflows/scheduled-image-scan.yml | 6 +++--- .../reconcilers/ingress/ingress_reconciler.go | 4 +++- test/scripts/gh-actions/setup-deps.sh | 4 ++++ test/scripts/gh-actions/status-check.sh | 16 ++++++++++++---- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/scheduled-image-scan.yml b/.github/workflows/scheduled-image-scan.yml index 386168a13cb..c18478dfa7e 100644 --- a/.github/workflows/scheduled-image-scan.yml +++ b/.github/workflows/scheduled-image-scan.yml @@ -45,7 +45,7 @@ jobs: - name: Upload sarif file to Github Code Scanning if: always() - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: application/${{ matrix.image.name }}/docker.snyk.sarif @@ -81,7 +81,7 @@ jobs: - name: Upload sarif file to Github Code Scanning if: always() - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: application/${{ matrix.image.name }}/docker.snyk.sarif @@ -115,6 +115,6 @@ jobs: - name: Upload sarif file to Github Code Scanning if: always() - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: application/${{ matrix.image.name }}/docker.snyk.sarif diff --git a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go index 88400bf55e3..e67cd75d168 100644 --- a/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go +++ b/pkg/controller/v1beta1/inferenceservice/reconcilers/ingress/ingress_reconciler.go @@ -483,7 +483,7 @@ func probeIngress(url string) (bool, error) { defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) if err != nil { - return isReady, errors.Wrapf(err, "failed to probe ingress %s", target) + return isReady, errors.Wrapf(err, "failed to create ingress probe request %s", target) } // ProbeKey is the name of a header that can be added to requests to probe the ingress. // Requests with this header will not be passed to the user container or included in request metrics. @@ -497,6 +497,8 @@ func probeIngress(url string) (bool, error) { } if resp.StatusCode == http.StatusOK { isReady = true + } else { + log.V(1).Error(fmt.Errorf("failed to probe ingress"), "Failed to probe ingress", "url", target) } return isReady, nil } diff --git a/test/scripts/gh-actions/setup-deps.sh b/test/scripts/gh-actions/setup-deps.sh index 5acd8cbd75f..529ef1ee10e 100755 --- a/test/scripts/gh-actions/setup-deps.sh +++ b/test/scripts/gh-actions/setup-deps.sh @@ -27,10 +27,14 @@ DEPLOYMENT_MODE="${1:-'serverless'}" ISTIO_VERSION="1.19.4" CERT_MANAGER_VERSION="v1.5.0" YQ_VERSION="v4.28.1" +JQ_VERSION="1.7.1" echo "Installing yq ..." wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq +echo "Installing jq ..." +wget https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-amd64 -O /usr/local/bin/jq && chmod +x /usr/local/bin/jq + echo "Installing Istio ..." mkdir istio_tmp pushd istio_tmp >/dev/null diff --git a/test/scripts/gh-actions/status-check.sh b/test/scripts/gh-actions/status-check.sh index bdd28387b73..451c655d297 100755 --- a/test/scripts/gh-actions/status-check.sh +++ b/test/scripts/gh-actions/status-check.sh @@ -31,7 +31,9 @@ kubectl logs -l control-plane=kserve-controller-manager -n kserve -c manager --t echo "::endgroup::" echo "::group::Predictor Pod logs" -for pod in $(kubectl get pods -l 'component in (predictor)' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do +# Get only pods that are not being deleted. +predictor_pods=$(kubectl get pods -l 'component in (predictor)' -o json -n kserve-ci-e2e-test | jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name') +for pod in $predictor_pods; do echo "===================================== Logs for Predictor Pod: $pod =========================================" kubectl logs "$pod" -c kserve-container -n kserve-ci-e2e-test --tail 500 echo "================================================================================================================" @@ -39,7 +41,9 @@ done echo "::endgroup::" echo "::group::Transformer Pod logs" -for pod in $(kubectl get pods -l 'component in (transformer)' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do +# Get only pods that are not being deleted. +transformer_pods=$(kubectl get pods -l 'component in (transformer)' -o json -n kserve-ci-e2e-test | jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name') +for pod in $transformer_pods; do echo "===================================== Logs for Transformer Pod: $pod =======================================" kubectl logs "$pod" -c kserve-container -n kserve-ci-e2e-test --tail 500 echo "================================================================================================================" @@ -47,7 +51,9 @@ done echo "::endgroup::" echo "::group::Explainer Pod logs" -for pod in $(kubectl get pods -l 'component in (explainer)' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do +# Get only pods that are not being deleted. +explainer_pods=$(kubectl get pods -l 'component in (explainer)' -o json -n kserve-ci-e2e-test | jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name') +for pod in $explainer_pods; do echo "===================================== Logs for Explainer Pod: $pod =========================================" kubectl logs "$pod" -c kserve-container -n kserve-ci-e2e-test --tail 500 echo "================================================================================================================" @@ -55,7 +61,9 @@ done echo "::endgroup::" echo "::group::InferenceGraph Pod logs" -for pod in $(kubectl get pods -l 'serving.kserve.io/inferencegraph=model-chainer' -o jsonpath='{.items[*].metadata.name}' -n kserve-ci-e2e-test); do +# Get only pods that are not being deleted. +graph_pods=$(kubectl get pods -l 'serving.kserve.io/inferencegraph=model-chainer' -o json -n kserve-ci-e2e-test | jq -r '.items[] | select(.metadata.deletionTimestamp == null) | .metadata.name') +for pod in $graph_pods; do echo "===================================== Logs for Graph Pod: $pod =========================================" kubectl logs "$pod" -c user-container -n kserve-ci-e2e-test --tail 500 echo "================================================================================================================"