diff --git a/crates/bpe/src/appendable_encoder.rs b/crates/bpe/src/appendable_encoder.rs index c4e7eef..f75fde8 100644 --- a/crates/bpe/src/appendable_encoder.rs +++ b/crates/bpe/src/appendable_encoder.rs @@ -1,5 +1,6 @@ use crate::byte_pair_encoding::BytePairEncoding; +#[derive(Clone)] struct State { state: u32, last_token: u32, @@ -7,6 +8,7 @@ struct State { } /// Encoder which keeps track of the encoding length while appending characters. +#[derive(Clone)] pub struct AppendableEncoder<'a> { bpe: &'a BytePairEncoding, states: Vec, diff --git a/crates/bpe/src/prependable_encoder.rs b/crates/bpe/src/prependable_encoder.rs index 61d60ca..f229d32 100644 --- a/crates/bpe/src/prependable_encoder.rs +++ b/crates/bpe/src/prependable_encoder.rs @@ -1,5 +1,6 @@ use crate::byte_pair_encoding::BytePairEncoding; +#[derive(Clone)] struct State { state: u32, prev_token: u32, @@ -7,6 +8,7 @@ struct State { } /// Encoder which keeps track of the encoding length while prepending characters. +#[derive(Clone)] pub struct PrependableEncoder<'a> { bpe: &'a BytePairEncoding, states: Vec,