Trying the new esaxx version.

huggingface · Oct 5, 2023 · 5ce1e4f · 5ce1e4f
1 parent 82b7b78
commit 5ce1e4f
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 7 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -101,9 +101,6 @@ jobs:
           command: audit
           args: -D warnings -f ./bindings/python/Cargo.lock
 
-      - name: Setup tmate session
-        uses: mxschmitt/action-tmate@v3
-
       - name: Install
         working-directory: ./bindings/python
         run: |

diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -23,9 +23,6 @@ itertools = "0.11"
 [dependencies.tokenizers]
 version = "0.14.1-dev.0"
 path = "../../tokenizers"
-default-features = false
-# features = ["progressbar", "onig", "esaxx_fast"]
-features = ["progressbar", "onig"]
 
 [dev-dependencies]
 tempfile = "3.8"

diff --git a/tokenizers/Cargo.toml b/tokenizers/Cargo.toml
@@ -68,7 +68,7 @@ macro_rules_attribute = "0.2.0"
 thiserror = "1.0.49"
 fancy-regex = { version = "0.11", optional = true}
 getrandom = { version = "0.2.10" }
-esaxx-rs = { version = "0.1", default-features = false, features=[]}
+esaxx-rs = { git = "https://github.com/Narsil/esaxx-rs", default-features = false, features=[]}
 monostate = "0.1.9"
 
 [features]