fix(deps): update rust crate tokenizers to 0.21.0

Danite · Nov 27, 2024 · 33a6027 · 33a6027
1 parent ba5b577
commit 33a6027
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,7 +9,7 @@ path = "src/lib.rs"
 
 [dependencies]
 tch = "0.17.0"
-tokenizers = "0.20.0"
+tokenizers = "0.21.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 rayon = "1.5"