diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
index 5b31520bdbabdf..c4c28025f43c69 100644
--- a/.github/workflows/self-scheduled.yml
+++ b/.github/workflows/self-scheduled.yml
@@ -282,7 +282,18 @@ jobs:
         run: rm -rf /github/home/.cache/torch_extensions/
 
       # To avoid unknown test failures
-      - name: Pre build DeepSpeed *again*
+      - name: Pre build DeepSpeed *again* (for daily CI)
+        if: ${{ inputs.docker == 'huggingface/transformers-pytorch-deepspeed-latest-gpu' }}
+        working-directory: /workspace
+        run: |
+          python3 -m pip uninstall -y deepspeed
+          rm -rf DeepSpeed
+          git clone https://github.com/microsoft/DeepSpeed && cd DeepSpeed && rm -rf build
+          DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install . --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check
+
+      # To avoid unknown test failures
+      - name: Pre build DeepSpeed *again* (for nightly CI)
+        if: ${{ inputs.docker == 'huggingface/transformers-pytorch-deepspeed-nightly-gpu' }}
         working-directory: /workspace
         run: |
           python3 -m pip uninstall -y deepspeed