From 08a9ecf22336fa21a20b13cd6707af3103b5f8a1 Mon Sep 17 00:00:00 2001 From: Arthur Zucker Date: Wed, 24 Jul 2024 20:12:42 +0200 Subject: [PATCH] revert --- bindings/python/Cargo.toml | 4 ---- tokenizers/benches/bpe_benchmark.rs | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 376523ded..3b1b1bbf1 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -29,7 +29,3 @@ pyo3 = { version = "0.21", features = ["auto-initialize"] } [features] defaut = ["pyo3/extension-module"] - -[profile.release] -debug = true -strip = false \ No newline at end of file diff --git a/tokenizers/benches/bpe_benchmark.rs b/tokenizers/benches/bpe_benchmark.rs index 3589d449a..dd65d233e 100644 --- a/tokenizers/benches/bpe_benchmark.rs +++ b/tokenizers/benches/bpe_benchmark.rs @@ -24,8 +24,8 @@ fn create_gpt2_tokenizer(bpe: BPE) -> Tokenizer { let mut tokenizer = Tokenizer::new(bpe); tokenizer.with_pre_tokenizer(ByteLevel::default()); tokenizer.with_decoder(ByteLevel::default()); - // tokenizer.add_tokens(&[AddedToken::from("ing", false).single_word(false)]); - // tokenizer.add_special_tokens(&[AddedToken::from("[ENT]", true).single_word(true)]); + tokenizer.add_tokens(&[AddedToken::from("ing", false).single_word(false)]); + tokenizer.add_special_tokens(&[AddedToken::from("[ENT]", true).single_word(true)]); tokenizer }