Skip to content

Commit aa14609

Browse files
Remove debug statement
1 parent 9350e9a commit aa14609

File tree

1 file changed

+1
-3
lines changed

1 file changed

+1
-3
lines changed

crates/bpe/src/byte_pair_encoding.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -205,9 +205,7 @@ impl BytePairEncoding {
205205
let mut token_starts = vec![0];
206206
let mut bytes_hash_to_token = FnvHashMap::default();
207207
for (i, token) in iter.enumerate() {
208-
if let Some(j) = bytes_hash_to_token.insert(hash_bytes(&token), i as u32) {
209-
eprintln!("collision: ({i}, {j})");
210-
}
208+
bytes_hash_to_token.insert(hash_bytes(&token), i as u32);
211209
all_tokens_rev.extend(token.iter().copied().rev());
212210
all_tokens.extend(token);
213211
token_starts.push(all_tokens.len() as u32);

0 commit comments

Comments
 (0)