Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Remove CLI - this was originally intended for local development
Browse files Browse the repository at this point in the history
bryantbiggs committed Jan 23, 2024
1 parent accd065 commit fcb12c0
Showing 3 changed files with 7 additions and 88 deletions.
2 changes: 1 addition & 1 deletion RELEASE.md
Original file line number Diff line number Diff line change
@@ -86,5 +86,5 @@ Simple checklist on how to make releases for `tokenizers`.
If you want to make modifications to the CI/CD of the release GH actions, you need
to :
- **Comment the part that uploads the artifacts** to `crates.io`, `PyPi` or `npm`.
- Change the trigger mecanism so it can trigger every time you push to your branch.
- Change the trigger mechanism so it can trigger every time you push to your branch.
- Keep pushing your changes until the artifacts are properly created.
20 changes: 6 additions & 14 deletions tokenizers/Cargo.toml
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
authors = ["Anthony MOI <[email protected]>", "Nicolas Patry <[email protected]>"]
edition = "2018"
name = "tokenizers"
version = "0.15.1-dev.0"
version = "0.15.2-dev.0"
homepage = "https://github.com/huggingface/tokenizers"
repository = "https://github.com/huggingface/tokenizers"
documentation = "https://docs.rs/tokenizers/"
@@ -20,12 +20,6 @@ name = "tokenizers"
path = "src/lib.rs"
bench = false

[[bin]]
name = "cli"
path = "src/cli.rs"
bench = false
required-features = ["cli"]

[[bench]]
name = "bpe_benchmark"
harness = false
@@ -47,36 +41,34 @@ lazy_static = "1.4"
rand = "0.8"
onig = { version = "6.4", default-features = false, optional = true }
regex = "1.9"
regex-syntax = "0.7"
regex-syntax = "0.8"
rayon = "1.8"
rayon-cond = "0.3"
serde = { version = "1.0", features = [ "derive" ] }
serde_json = "1.0"
clap = { version = "4.4", features=["derive"], optional = true }
unicode-normalization-alignments = "0.1"
unicode_categories = "0.1"
unicode-segmentation = "1.10"
indicatif = {version = "0.17", optional = true}
itertools = "0.11"
itertools = "0.12"
log = "0.4"
derive_builder = "0.12"
derive_builder = "0.13"
spm_precompiled = "0.1"
hf-hub = { version = "0.3.2", optional = true }
aho-corasick = "1.1"
paste = "1.0.14"
macro_rules_attribute = "0.2.0"
thiserror = "1.0.49"
fancy-regex = { version = "0.11", optional = true}
fancy-regex = { version = "0.13", optional = true}
getrandom = { version = "0.2.10" }
esaxx-rs = { version = "0.1.10", default-features = false, features=[]}
monostate = "0.1.9"

[features]
default = ["progressbar", "cli", "onig", "esaxx_fast"]
default = ["progressbar", "onig", "esaxx_fast"]
esaxx_fast = ["esaxx-rs/cpp"]
progressbar = ["indicatif"]
http = ["hf-hub"]
cli = ["clap"]
unstable_wasm = ["fancy-regex", "getrandom/js"]

[dev-dependencies]
73 changes: 0 additions & 73 deletions tokenizers/src/cli.rs

This file was deleted.

0 comments on commit fcb12c0

Please sign in to comment.