linter

crates/bpe/src/appendable_encoder.rs

@@ -56,7 +56,7 @@ impl<'a> AppendableEncoder<'a> {
     }
 
     /// Returns the number of tokens required to tokenize the input text.
-    /// This operation is O(1) and can be called at any point in time. 
+    /// This operation is O(1) and can be called at any point in time.
     pub fn len(&self) -> usize {
         self.counts.last().copied().unwrap_or(0) as usize
     }
@@ -78,4 +78,4 @@ mod tests {
             enc.push(*c);
         }
     }
-}
+}

crates/bpe/src/backtrack_encoder.rs

@@ -6,7 +6,7 @@ use crate::byte_pair_encoding::BytePairEncoding;
 /// for a given input text.
 /// It keeps track of visited states in a bitfield and only remembers the tokenization
 /// of the currently processed dynamic programming state.
-/// 
+///
 /// The biggest downside of this approach is that the search for the longest leftmost match
 /// has to be reset at every (backtracking) step which is still a net win in practice compared to other approaches.
 pub(crate) struct BacktrackEncoder<'a> {