From 33a602749e415cfdbc0f1aeef6799e40deaffb8d Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Wed, 27 Nov 2024 13:17:50 +0000
Subject: [PATCH] fix(deps): update rust crate tokenizers to 0.21.0

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index ea10309..fb64249 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,7 @@ path = "src/lib.rs"
 
 [dependencies]
 tch = "0.17.0"
-tokenizers = "0.20.0"
+tokenizers = "0.21.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 rayon = "1.5"