[DO NOT MERGE] Testing tokenizers 0.19.0rc0

huggingface · Apr 16, 2024 · d272238 · d272238
1 parent 60dea59
commit d272238
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -174,7 +174,7 @@
     "tf2onnx",
     "timeout-decorator",
     "timm",
-    "tokenizers>=0.14,<0.19",
+    "tokenizers==0.19.0rc0",
     "torch",
     "torchaudio",
     "torchvision",

diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
@@ -79,7 +79,7 @@
     "tf2onnx": "tf2onnx",
     "timeout-decorator": "timeout-decorator",
     "timm": "timm",
-    "tokenizers": "tokenizers>=0.14,<0.19",
+    "tokenizers": "tokenizers==0.19.0rc0",
     "torch": "torch",
     "torchaudio": "torchaudio",
     "torchvision": "torchvision",