From a6faf25db14bbb136301c03a031825e959c5de43 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 6 Dec 2024 08:33:38 -0500 Subject: [PATCH 1/4] use seperate steps for image build and test run on modal --- .github/workflows/tests.yml | 5 +++++ cicd/tests.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e1a4fe924..663098fde0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -164,6 +164,7 @@ jobs: python_version: "3.11" pytorch: 2.4.1 num_gpus: 1 + gpu_type: A10G axolotl_extras: steps: - name: Checkout @@ -184,6 +185,10 @@ jobs: echo "AXOLOTL_EXTRAS=${{ matrix.axolotl_extras}}" >> $GITHUB_ENV echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV + echo "GPU_TYPE=${{ matrix.gpu_type }}" >> $GITHUB_ENV + - name: Build the image on Modal + run: | + modal run cicd/tests.py:app.image_build - name: Run tests job on Modal run: | modal run cicd.tests diff --git a/cicd/tests.py b/cicd/tests.py index f3dbaef105..6507a3bb62 100644 --- a/cicd/tests.py +++ b/cicd/tests.py @@ -52,7 +52,8 @@ N_GPUS = int(os.environ.get("N_GPUS", 1)) -GPU_CONFIG = modal.gpu.A10G(count=N_GPUS) +GPU_TYPE = os.environ.get("GPU_TYPE", "A10G") +GPU_CONFIG = getattr(modal.gpu, GPU_TYPE)(count=N_GPUS) def run_cmd(cmd: str, run_folder: str): @@ -63,6 +64,18 @@ def run_cmd(cmd: str, run_folder: str): exit(exit_code) # pylint: disable=consider-using-sys-exit +@app.function( + image=cicd_image, + gpu=GPU_CONFIG, + timeout=45 * 60, + cpu=8.0, + memory=131072, +) +def image_build(): + # noop to trigger the image build + pass + + @app.function( image=cicd_image, gpu=GPU_CONFIG, From 9d1d5adb361ca0631b6adf6f3d3cf477696d76ed Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 6 Dec 2024 09:08:20 -0500 Subject: [PATCH 2/4] fix syntax for modal entrypoint --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 663098fde0..559f0763aa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -188,7 +188,7 @@ jobs: echo "GPU_TYPE=${{ matrix.gpu_type }}" >> $GITHUB_ENV - name: Build the image on Modal run: | - modal run cicd/tests.py:app.image_build + modal run cicd/tests.py::app.image_build - name: Run tests job on Modal run: | modal run cicd.tests From 03bc7a1bd01fe6d9d5138a92c122b60c22652502 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 6 Dec 2024 09:20:41 -0500 Subject: [PATCH 3/4] modal run doesn't need app namespace --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 559f0763aa..8c9596696a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -188,7 +188,7 @@ jobs: echo "GPU_TYPE=${{ matrix.gpu_type }}" >> $GITHUB_ENV - name: Build the image on Modal run: | - modal run cicd/tests.py::app.image_build + modal run cicd/tests.py::image_build - name: Run tests job on Modal run: | modal run cicd.tests From 1e3400f0c9e2d94343e57bed0d317104f10f5725 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sat, 7 Dec 2024 09:22:12 -0500 Subject: [PATCH 4/4] use module name rather than path to fix template issue --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8c9596696a..05b7b00022 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -188,7 +188,7 @@ jobs: echo "GPU_TYPE=${{ matrix.gpu_type }}" >> $GITHUB_ENV - name: Build the image on Modal run: | - modal run cicd/tests.py::image_build + modal run cicd.tests::image_build - name: Run tests job on Modal run: | modal run cicd.tests