Merge pull request #10 from github/aneubeck/bpe

linter

crates/bpe/README.md

@@ -191,6 +191,6 @@ We compared our implementations with the tiktoken implementation on a MacBook Pr
 | Heap         | 1900 µs  | ✔          |
 
 As can be seen, our Backtracking implementation beats the TikToken Rust implementation by ~4x.
-And even the fully deynamic programming solution is faster with a more consistent runtime.
+And even the fully dynamic programming solution is faster with a more consistent runtime.
 The tuned heap implementation is still quite competitive to TikToken (especially for smaller inputs).
 If the requirement of correct BPE output can be relaxed, then the Greedy approach or the minimal encoding approach are the clear winners.

crates/bpe/src/appendable_encoder.rs

@@ -56,10 +56,15 @@ impl<'a> AppendableEncoder<'a> {
     }
 
     /// Returns the number of tokens required to tokenize the input text.
-    /// This operation is O(1) and can be called at any point in time. 
+    /// This operation is O(1) and can be called at any point in time.
     pub fn len(&self) -> usize {
         self.counts.last().copied().unwrap_or(0) as usize
     }
+
+    /// Returns true if the structure represents the empty string.
+    pub fn is_empty(&self) -> bool {
+        self.counts.is_empty()
+    }
 }
 
 #[cfg(test)]
@@ -78,4 +83,4 @@ mod tests {
             enc.push(*c);
         }
     }
-}
+}

crates/bpe/src/backtrack_encoder.rs

@@ -6,7 +6,7 @@ use crate::byte_pair_encoding::BytePairEncoding;
 /// for a given input text.
 /// It keeps track of visited states in a bitfield and only remembers the tokenization
 /// of the currently processed dynamic programming state.
-/// 
+///
 /// The biggest downside of this approach is that the search for the longest leftmost match
 /// has to be reset at every (backtracking) step which is still a net win in practice compared to other approaches.
 pub(crate) struct BacktrackEncoder<'a> {

crates/bpe/src/interval_encoding.rs

@@ -57,7 +57,7 @@ impl<'a> IntervalEncoding<'a> {
     /// Thereby it reencodes the prefix with the `BacktrackEncoder` until the encoding sequence becomes
     /// compatible with the precomputed tables. Once that's the case, the remainder of the range becomes
     /// a simple O(1) lookup.
-    /// 
+    ///
     /// Note: in the worst-case the complexity is O(n). This happens for instance for a whitespace input
     /// where the encoding changes when the starting position changes.
     pub fn count(&self, range: Range<usize>) -> usize {