From cd5fe6e576076da992d48c46444a8df8bcb2c26f Mon Sep 17 00:00:00 2001
From: computermacgyver <computermacgyver>
Date: Mon, 2 Dec 2024 17:06:03 +0000
Subject: [PATCH] pin tokenizers

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 8d054506..f296eeb6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -137,7 +137,7 @@ threadpoolctl==2.2.0
 tinysegmenter==0.3
 tldextract==3.1.0
 tmkpy==0.1.1
-tokenizers
+tokenizers==0.10.3
 toolz==0.9.0
 torch==1.9.0
 tqdm==4.66.3