From e7e53e83f9cde69080433d088dcf3c7924963a1d Mon Sep 17 00:00:00 2001
From: Itai Smith <itaismith@gmail.com>
Date: Tue, 26 Nov 2024 14:39:51 -0800
Subject: [PATCH] Bound tokenizers version

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index b7b621faf2a0..fc12641a2e13 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,7 +21,8 @@ pypika>=0.48.9
 PyYAML>=6.0.0
 rich>=10.11.0
 tenacity>=8.2.3
-tokenizers>=0.13.2
+--only-binary tokenizers
+tokenizers>=0.13.2,<=0.20.3
 tqdm>=4.65.0
 typer>=0.9.0
 typing_extensions>=4.5.0