From f76623694845c2d921094e665083804f85773207 Mon Sep 17 00:00:00 2001
From: Alexander Neubeck <aneubeck@github.com>
Date: Mon, 19 Aug 2024 11:11:44 +0000
Subject: [PATCH] make encoders clonable

---
 crates/bpe/src/appendable_encoder.rs  | 2 ++
 crates/bpe/src/prependable_encoder.rs | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/crates/bpe/src/appendable_encoder.rs b/crates/bpe/src/appendable_encoder.rs
index c4e7eef..f75fde8 100644
--- a/crates/bpe/src/appendable_encoder.rs
+++ b/crates/bpe/src/appendable_encoder.rs
@@ -1,5 +1,6 @@
 use crate::byte_pair_encoding::BytePairEncoding;
 
+#[derive(Clone)]
 struct State {
     state: u32,
     last_token: u32,
@@ -7,6 +8,7 @@ struct State {
 }
 
 /// Encoder which keeps track of the encoding length while appending characters.
+#[derive(Clone)]
 pub struct AppendableEncoder<'a> {
     bpe: &'a BytePairEncoding,
     states: Vec<State>,
diff --git a/crates/bpe/src/prependable_encoder.rs b/crates/bpe/src/prependable_encoder.rs
index 61d60ca..f229d32 100644
--- a/crates/bpe/src/prependable_encoder.rs
+++ b/crates/bpe/src/prependable_encoder.rs
@@ -1,5 +1,6 @@
 use crate::byte_pair_encoding::BytePairEncoding;
 
+#[derive(Clone)]
 struct State {
     state: u32,
     prev_token: u32,
@@ -7,6 +8,7 @@ struct State {
 }
 
 /// Encoder which keeps track of the encoding length while prepending characters.
+#[derive(Clone)]
 pub struct PrependableEncoder<'a> {
     bpe: &'a BytePairEncoding,
     states: Vec<State>,