Use the right tiktoken encoding for comparison

github · Sep 25, 2024 · 7bf5093 · 7bf5093
1 parent aa14609
commit 7bf5093
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 5 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,4 @@ Cargo.lock
 /target/
 /crates/*/target/
 /crates/*/Cargo.lock
-.vscode/
+.vscode/
diff --git a/crates/bpe/benches/counting.rs b/crates/bpe/benches/counting.rs
@@ -34,11 +34,10 @@ fn counting_benchmark(c: &mut Criterion) {
 }
 
 fn encoding_benchmark(c: &mut Criterion) {
-    for (name, bpe) in [
-        ("cl100k", BytePairEncoding::cl100k()),
-        ("o200k", BytePairEncoding::o200k()),
+    for (name, bpe, tiktoken) in [
+        ("cl100k", BytePairEncoding::cl100k(), tiktoken_rs::cl100k_base().unwrap()),
+        ("o200k", BytePairEncoding::o200k(), tiktoken_rs::o200k_base().unwrap()),
     ] {
-        let tiktoken = tiktoken_rs::cl100k_base().unwrap();
         let text = create_test_string(&bpe, 20000);
         let input = text.as_bytes();
 

diff --git a/criterion.toml b/criterion.toml
@@ -0,0 +1,2 @@
+# save report in this directory, even if a custom target directory is set
+criterion_home = "./target/criterion"