From 544f0beae4269f97a0574be53b72f508f8ec7021 Mon Sep 17 00:00:00 2001
From: "Kevin H. Luu" <kevin@anyscale.com>
Date: Tue, 19 Nov 2024 18:50:28 -1000
Subject: [PATCH] Run optional tests on nightly build (#51)

* Update test-template-aws.j2

* Update test-template-aws.j2

* Send nightly args

* print nightly

* add nightly args

* Update test-template-aws.j2

* Update ci_aws_bootstrap.sh

* Update test-template-aws.j2

* Update test-template-aws.j2

* just run a100

* remove test steps

* Update ci_aws_bootstrap.sh
---
 scripts/ci_aws_bootstrap.sh  |  9 +++++++--
 scripts/test-template-aws.j2 | 15 ++++++---------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/scripts/ci_aws_bootstrap.sh b/scripts/ci_aws_bootstrap.sh
index b77aad6..27223db 100644
--- a/scripts/ci_aws_bootstrap.sh
+++ b/scripts/ci_aws_bootstrap.sh
@@ -6,6 +6,10 @@ if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
 
+if [[ -z "${NIGHTLY:-}" ]]; then
+    NIGHTLY=0
+fi
+
 upload_pipeline() {
     echo "Uploading pipeline..."
     ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
@@ -25,14 +29,15 @@ upload_pipeline() {
     fi
     if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
         python -m pip install click pydantic
-        python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF"
+        python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY"
         buildkite-agent pipeline upload .buildkite/pipeline.yaml
         exit 0
     fi
     cd .buildkite
     echo "List file diff: $LIST_FILE_DIFF"
     echo "Run all: $RUN_ALL"
-    minijinja-cli test-template.j2 test-pipeline.yaml -D list_file_diff="$LIST_FILE_DIFF" -D run_all="$RUN_ALL" > pipeline.yml
+    echo "Nightly: $NIGHTLY"
+    minijinja-cli test-template.j2 test-pipeline.yaml -D list_file_diff="$LIST_FILE_DIFF" -D run_all="$RUN_ALL" -D nightly="$NIGHTLY" > pipeline.yml
     buildkite-agent pipeline upload pipeline.yml
     exit 0
 }
diff --git a/scripts/test-template-aws.j2 b/scripts/test-template-aws.j2
index 8d92e21..3c7644b 100644
--- a/scripts/test-template-aws.j2
+++ b/scripts/test-template-aws.j2
@@ -39,6 +39,10 @@ steps:
   {% set ns.blocked = 0 %}
   {% endif %}
 
+  {% if nightly == "1" %}
+  {% set ns.blocked = 0 %}
+  {% endif %}
+
   {% if step.source_file_dependencies %}
     {% for source_file in step.source_file_dependencies %}
       {% for file in list_file_diff %}
@@ -51,25 +55,18 @@ steps:
     {% set ns.blocked = 0 %}
   {% endif %}
 
-  {% if step.gpu == "a100" %}
-  {% set ns.blocked = 1 %}
-  {% endif %}
-
-  {% if ns.blocked == 1 or step.optional %}
+  {% if ns.blocked == 1 or (step.optional and nightly != "1") %}
   - block: "Run {{ step.label }}"
     depends_on: image-build
     key: block-{{ step.label | replace(" ", "-") | lower | replace("(", "") | replace(")", "") | replace("%", "") | replace(",", "-") }}
   {% endif %}
 
   - label: "{{ step.label }}"
-    {% if ns.blocked == 1 or step.optional %}
+    {% if ns.blocked == 1 or (step.optional and nightly != "1") %}
     depends_on: block-{{ step.label | replace(" ", "-") | lower | replace("(", "") | replace(")", "") | replace("%", "") | replace(",", "-") }}
     {% else %}
     depends_on: image-build
     {% endif %}
-    {% if step.nightly %}
-    if: build.env("NIGHTLY") == "1"
-    {% endif %}
     agents:
       {% if step.label == "Documentation Build" %}
       queue: small_cpu_queue