Merge pull request #15 from github/aneubeck/dict

Make encoders clonable
github · Aug 19, 2024 · 2112a0e · 2112a0e
2 parents 9feb662 + f766236
commit 2112a0e
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/crates/bpe/src/appendable_encoder.rs b/crates/bpe/src/appendable_encoder.rs
@@ -1,12 +1,14 @@
 use crate::byte_pair_encoding::BytePairEncoding;
 
+#[derive(Clone)]
 struct State {
     state: u32,
     last_token: u32,
     count: u32,
 }
 
 /// Encoder which keeps track of the encoding length while appending characters.
+#[derive(Clone)]
 pub struct AppendableEncoder<'a> {
     bpe: &'a BytePairEncoding,
     states: Vec<State>,

diff --git a/crates/bpe/src/prependable_encoder.rs b/crates/bpe/src/prependable_encoder.rs
@@ -1,12 +1,14 @@
 use crate::byte_pair_encoding::BytePairEncoding;
 
+#[derive(Clone)]
 struct State {
     state: u32,
     prev_token: u32,
     count: u32,
 }
 
 /// Encoder which keeps track of the encoding length while prepending characters.
+#[derive(Clone)]
 pub struct PrependableEncoder<'a> {
     bpe: &'a BytePairEncoding,
     states: Vec<State>,