Skip to content

Commit

Permalink
nit clippy
Browse files Browse the repository at this point in the history
  • Loading branch information
ArthurZucker committed Jul 15, 2024
1 parent 52a6f2e commit 5d9ea9f
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions tokenizers/src/pre_tokenizers/metaspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ impl Default for Metaspace {

impl PreTokenizer for Metaspace {
fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
if self.prepend_scheme == PrependScheme::First {
let result = if self.prepend_scheme == PrependScheme::First {
let normalized = pretokenized.splits[0].normalized.clone();
if !normalized.get().starts_with(self.replacement)
&& normalized.offsets_original().0 == 0
Expand All @@ -128,7 +128,7 @@ impl PreTokenizer for Metaspace {
NormalizedString::from(self.replacement.to_string()).into(),
);
}
pretokenized.split(|_, mut normalized| {
let _ = pretokenized.split(|_, mut normalized| {
normalized.replace(' ', &self.str_rep)?;
if self.split {
normalized.split(self.replacement, SplitDelimiterBehavior::MergedWithNext)
Expand All @@ -155,7 +155,8 @@ impl PreTokenizer for Metaspace {
Ok(vec![normalized])
}
})
}
};
result
}
}

Expand Down

0 comments on commit 5d9ea9f

Please sign in to comment.