We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 9350e9a commit aa14609Copy full SHA for aa14609
crates/bpe/src/byte_pair_encoding.rs
@@ -205,9 +205,7 @@ impl BytePairEncoding {
205
let mut token_starts = vec![0];
206
let mut bytes_hash_to_token = FnvHashMap::default();
207
for (i, token) in iter.enumerate() {
208
- if let Some(j) = bytes_hash_to_token.insert(hash_bytes(&token), i as u32) {
209
- eprintln!("collision: ({i}, {j})");
210
- }
+ bytes_hash_to_token.insert(hash_bytes(&token), i as u32);
211
all_tokens_rev.extend(token.iter().copied().rev());
212
all_tokens.extend(token);
213
token_starts.push(all_tokens.len() as u32);
0 commit comments