Merge pull request #15 from github/aneubeck/dict

Make encoders clonable

Author: aneubeck

crates/bpe/src/appendable_encoder.rs

@@ -1,12 +1,14 @@
 use crate::byte_pair_encoding::BytePairEncoding;
 
+#[derive(Clone)]
 struct State {
     state: u32,
     last_token: u32,
     count: u32,
 }
 
 /// Encoder which keeps track of the encoding length while appending characters.
+#[derive(Clone)]
 pub struct AppendableEncoder<'a> {
     bpe: &'a BytePairEncoding,
     states: Vec<State>,

crates/bpe/src/prependable_encoder.rs

@@ -1,12 +1,14 @@
 use crate::byte_pair_encoding::BytePairEncoding;
 
+#[derive(Clone)]
 struct State {
     state: u32,
     prev_token: u32,
     count: u32,
 }
 
 /// Encoder which keeps track of the encoding length while prepending characters.
+#[derive(Clone)]
 pub struct PrependableEncoder<'a> {
     bpe: &'a BytePairEncoding,
     states: Vec<State>,