diff --git a/.github/workflows/no-torch.yml b/.github/workflows/no-torch.yml
index b26b4efd9203..1a13c0f3f4f1 100644
--- a/.github/workflows/no-torch.yml
+++ b/.github/workflows/no-torch.yml
@@ -35,7 +35,7 @@ jobs:
 
       - name: Build deepspeed
         run: |
-          DS_ACCELERATOR="cuda" DS_BUILD_STRING=" " python setup.py sdist
+          DS_BUILD_STRING=" " python setup.py sdist
 
       - name: Open GitHub issue if nightly CI fails
         if: ${{ failure() && (github.event_name == 'schedule') }}
diff --git a/accelerator/cpu_accelerator.py b/accelerator/cpu_accelerator.py
index 0e49bd9f6458..572191ff9c13 100644
--- a/accelerator/cpu_accelerator.py
+++ b/accelerator/cpu_accelerator.py
@@ -3,9 +3,13 @@
 
 # DeepSpeed Team
 
-import torch
 from .abstract_accelerator import DeepSpeedAccelerator
 
+try:
+    import torch
+except ImportError as e:
+    pass
+
 try:
     import oneccl_bindings_for_pytorch  # noqa: F401 # type: ignore
     oneccl_imported_p = True