Fixes.

tokenizers/src/tokenizer/added_vocabulary.rs

@@ -139,7 +139,7 @@ fn space_rightmost_at_start(sentence: &str) -> usize {
 /// exist as required.
 ///
 #[derive(Clone, Debug)]
-pub(super) struct AddedVocabulary {
+pub struct AddedVocabulary {
     /// Contains the mapping from String (token content) to ID. This map contains both special
     /// tokens and classic added tokens that were added to the this vocabulary.
     added_tokens_map: HashMap<String, u32>,
@@ -192,6 +192,11 @@ impl AddedVocabulary {
         self.added_tokens_map.len()
     }
 
+    /// Whether or not this vocabulary is empty
+    pub fn is_empty(&self) -> bool {
+        self.added_tokens_map.is_empty()
+    }
+
     /// Get the additional vocabulary
     pub fn get_vocab(&self) -> &HashMap<String, u32> {
         &self.added_tokens_map
@@ -487,6 +492,12 @@ impl AddedVocabulary {
     }
 }
 
+impl Default for AddedVocabulary {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub(super) struct AddedTokenWithId {
     /// The id assigned to this token

tokenizers/src/tokenizer/mod.rs

@@ -606,7 +606,7 @@ where
 
     /// Set the added vocabulary.
     pub fn with_added_vocabulary(&mut self, added_vocabulary: AddedVocabulary) -> &mut Self {
-        self.added_vocabulary = added_vocabulary.into();
+        self.added_vocabulary = added_vocabulary;
         self
     }