diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py
index b44a5fde153b12..0107d95ebad99e 100644
--- a/.circleci/create_circleci_config.py
+++ b/.circleci/create_circleci_config.py
@@ -283,6 +283,8 @@ def job_name(self):
         "pip install -U --upgrade-strategy eager .[sklearn,tf-cpu,torch,testing,sentencepiece,torch-speech,vision]",
         "pip install -U --upgrade-strategy eager tensorflow_probability",
         "pip install -U --upgrade-strategy eager -e git+https://github.com/huggingface/accelerate@main#egg=accelerate",
+        # TODO: remove this one after fixing the dependency issue(s) above
+        "pip install -U --upgrade-strategy eager torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu"
     ],
     marker="is_pt_tf_cross_test",
     pytest_options={"rA": None, "durations": 0},
diff --git a/setup.py b/setup.py
index c4a38eccea2f3c..224f36c4a98f00 100644
--- a/setup.py
+++ b/setup.py
@@ -175,9 +175,9 @@
     "timeout-decorator",
     "timm",
     "tokenizers>=0.14,<0.19",
-    "torch<2.2.0",
-    "torchaudio<2.2.0",
-    "torchvision<0.17.0",
+    "torch",
+    "torchaudio",
+    "torchvision",
     "pyctcdecode>=0.4.0",
     "tqdm>=4.27",
     "unidic>=1.0.2",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index 4df2ff8c72957b..d70b717f0d6946 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -80,9 +80,9 @@
     "timeout-decorator": "timeout-decorator",
     "timm": "timm",
     "tokenizers": "tokenizers>=0.14,<0.19",
-    "torch": "torch<2.2.0",
-    "torchaudio": "torchaudio<2.2.0",
-    "torchvision": "torchvision<0.17.0",
+    "torch": "torch",
+    "torchaudio": "torchaudio",
+    "torchvision": "torchvision",
     "pyctcdecode": "pyctcdecode>=0.4.0",
     "tqdm": "tqdm>=4.27",
     "unidic": "unidic>=1.0.2",