Merge main

.circleci/create_circleci_config.py

@@ -132,7 +132,7 @@ def to_dict(self):
             if tests is None:
                 folder = os.environ["test_preparation_dir"]
                 test_file = os.path.join(folder, "filtered_test_list.txt")
-                if os.path.exists(test_file):
+                if os.path.exists(test_file): # We take this job's tests from the filtered test_list.txt
                     with open(test_file) as f:
                         tests = f.read().split(" ")
 
@@ -144,15 +144,26 @@ def to_dict(self):
                 if test.endswith(".py"):
                     expanded_tests.append(test)
                 elif test == "tests/models":
-                    expanded_tests.extend(glob.glob("tests/models/**/test*.py", recursive=True))
+                    if "tokenization" in self.name:
+                        expanded_tests.extend(glob.glob("tests/models/**/test_tokenization*.py", recursive=True))
+                    elif self.name in ["flax","torch","tf"]:
+                        name = self.name if self.name != "torch" else ""
+                        if self.name == "torch":
+                            all_tests = glob.glob(f"tests/models/**/test_modeling_{name}*.py", recursive=True) 
+                            filtered = [k for k in all_tests if ("_tf_") not in k and "_flax_" not in k]
+                            expanded_tests.extend(filtered)
+                        else:
+                            expanded_tests.extend(glob.glob(f"tests/models/**/test_modeling_{name}*.py", recursive=True))
+                    else:
+                        expanded_tests.extend(glob.glob("tests/models/**/test_modeling*.py", recursive=True))
                 elif test == "tests/pipelines":
-                    expanded_tests.extend([os.path.join(test, x) for x in os.listdir(test)])
+                    expanded_tests.extend(glob.glob("tests/models/**/test_modeling*.py", recursive=True)) 
                 else:
                     expanded_tests.append(test)
             tests = " ".join(expanded_tests)
 
             # Each executor to run ~10 tests
-            n_executors = max(len(tests) // 10, 1)
+            n_executors = max(len(expanded_tests) // 10, 1)
             # Avoid empty test list on some executor(s) or launching too many executors
             if n_executors > self.parallelism:
                 n_executors = self.parallelism
@@ -234,6 +245,14 @@ def job_name(self):
     pytest_num_workers=16
 )
 
+tokenization_job = CircleCIJob(
+    "tokenization",
+    docker_image=[{"image": "huggingface/transformers-torch-light"}],
+    install_steps=["uv venv && uv pip install ."],
+    parallelism=6,
+    pytest_num_workers=16
+)
+
 
 tf_job = CircleCIJob(
     "tf",
@@ -291,7 +310,8 @@ def job_name(self):
     additional_env={"OMP_NUM_THREADS": 8},
     cache_name="torch_examples",
     docker_image=[{"image":"huggingface/transformers-examples-torch"}],
-    install_steps=["uv venv && uv pip install ."],
+    # TODO @ArthurZucker remove this once docker is easier to build
+    install_steps=["uv venv && uv pip install . && uv pip install -r examples/pytorch/_tests_requirements.txt"],
     pytest_num_workers=1,
 )
 
@@ -404,6 +424,7 @@ def job_name(self):
     hub_job,
     onnx_job,
     exotic_models_job,
+    tokenization_job
 ]
 EXAMPLES_TESTS = [
     examples_torch_job,

.github/workflows/slack-report.yml

@@ -64,8 +64,8 @@ jobs:
         if: ${{ inputs.job == 'run_models_gpu' }}
         uses: actions/upload-artifact@v4
         with:
-          name: prev_ci_results
-          path: prev_ci_results
+          name: ci_results
+          path: ci_results
 
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4

conftest.py

@@ -71,7 +71,7 @@
     "ModelTester::test_pipeline_",
     "/repo_utils/",
     "/utils/",
-    "/tools/",
+    "/agents/",
 }
 
 # allow having multiple repository checkouts and not needing to remember to rerun
@@ -94,7 +94,7 @@ def pytest_configure(config):
     config.addinivalue_line("markers", "is_pipeline_test: mark test to run only when pipelines are tested")
     config.addinivalue_line("markers", "is_staging_test: mark test to run only in the staging environment")
     config.addinivalue_line("markers", "accelerate_tests: mark test that require accelerate")
-    config.addinivalue_line("markers", "tool_tests: mark the tool tests that are run on their specific schedule")
+    config.addinivalue_line("markers", "agent_tests: mark the agent tests that are run on their specific schedule")
     config.addinivalue_line("markers", "not_device_test: mark the tests always running on cpu")
 
 

docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile

@@ -42,7 +42,7 @@ RUN python3 -m pip uninstall -y deepspeed
 # This has to be run (again) inside the GPU VMs running the tests.
 # The installation works here, but some tests fail, if we don't pre-build deepspeed again in the VMs running the tests.
 # TODO: Find out why test fail.
-RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install deepspeed --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1
+RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 python3 -m pip install "deepspeed<=0.14.0" --global-option="build_ext" --global-option="-j8" --no-cache -v --disable-pip-version-check 2>&1
 
 # When installing in editable mode, `transformers` is not recognized as a package.
 # this line must be added in order for python to be aware of transformers.

docs/source/en/_toctree.yml

@@ -23,7 +23,7 @@
     title: Load and train adapters with 🤗 PEFT
   - local: model_sharing
     title: Share your model
-  - local: transformers_agents
+  - local: agents
     title: Agents
   - local: llm_tutorial
     title: Generation with LLMs
@@ -133,8 +133,6 @@
     title: Notebooks with examples
   - local: community
     title: Community resources
-  - local: custom_tools
-    title: Custom Tools and Prompts
   - local: troubleshooting
     title: Troubleshoot
   - local: hf_quantizer