diff --git a/.gitignore b/.gitignore index 3f0956c..0654041 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ Cargo.lock /target/ /crates/*/target/ /crates/*/Cargo.lock -.vscode/ \ No newline at end of file +.vscode/ diff --git a/crates/bpe/benches/counting.rs b/crates/bpe/benches/counting.rs index 250f516..b3c35e2 100644 --- a/crates/bpe/benches/counting.rs +++ b/crates/bpe/benches/counting.rs @@ -34,11 +34,10 @@ fn counting_benchmark(c: &mut Criterion) { } fn encoding_benchmark(c: &mut Criterion) { - for (name, bpe) in [ - ("cl100k", BytePairEncoding::cl100k()), - ("o200k", BytePairEncoding::o200k()), + for (name, bpe, tiktoken) in [ + ("cl100k", BytePairEncoding::cl100k(), tiktoken_rs::cl100k_base().unwrap()), + ("o200k", BytePairEncoding::o200k(), tiktoken_rs::o200k_base().unwrap()), ] { - let tiktoken = tiktoken_rs::cl100k_base().unwrap(); let text = create_test_string(&bpe, 20000); let input = text.as_bytes(); diff --git a/criterion.toml b/criterion.toml new file mode 100644 index 0000000..c0f42f2 --- /dev/null +++ b/criterion.toml @@ -0,0 +1,2 @@ +# save report in this directory, even if a custom target directory is set +criterion_home = "./target/criterion"