From 93f87d3cf5d937fc885c2131c6255863e5c1bfc5 Mon Sep 17 00:00:00 2001
From: Arthur <48595927+ArthurZucker@users.noreply.github.com>
Date: Thu, 5 Dec 2024 20:16:02 +0530
Subject: [PATCH] [`tokenizers`] bump to 0.21 (#34972)

bump to 0.21
---
 setup.py                                      | 2 +-
 src/transformers/dependency_versions_table.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 2dd0c50fb8d693..2364e3b677fd6d 100644
--- a/setup.py
+++ b/setup.py
@@ -180,7 +180,7 @@
     "timeout-decorator",
     "tiktoken",
     "timm<=1.0.11",
-    "tokenizers>=0.20,<0.21",
+    "tokenizers>=0.21,<0.22",
     "torch",
     "torchaudio",
     "torchvision",
diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
index dcf216783befaa..85345cc8e5889d 100644
--- a/src/transformers/dependency_versions_table.py
+++ b/src/transformers/dependency_versions_table.py
@@ -85,7 +85,7 @@
     "timeout-decorator": "timeout-decorator",
     "tiktoken": "tiktoken",
     "timm": "timm<=1.0.11",
-    "tokenizers": "tokenizers>=0.20,<0.21",
+    "tokenizers": "tokenizers>=0.21,<0.22",
     "torch": "torch",
     "torchaudio": "torchaudio",
     "torchvision": "torchvision",