Skip to content

Commit

Permalink
nits
Browse files Browse the repository at this point in the history
  • Loading branch information
ArthurZucker committed Nov 14, 2023
1 parent eb2d0d8 commit e68bcc3
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 8 deletions.
2 changes: 1 addition & 1 deletion bindings/python/src/pre_tokenizers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -510,7 +510,7 @@ impl PyMetaspace {
"always" => PrependScheme::Always,
_ => {
return Err(exceptions::PyValueError::new_err(format!(
"Unknown variant: {}",
"Unknown variant: {}, should be one of First, Never, Always",
prepend_scheme
)))
}
Expand Down
13 changes: 6 additions & 7 deletions tokenizers/src/pre_tokenizers/metaspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use crate::tokenizer::{Decoder, PreTokenizedString, PreTokenizer, Result, SplitD
use serde::{Deserialize, Deserializer, Serialize};

/// Enum representing options for the metaspace prepending scheme.
#[derive(Debug, Clone, PartialEq, Serialize, Eq, Deserialize)]
#[derive(Debug, Clone, PartialEq, Serialize, Eq, Deserialize, Copy)]
#[serde(rename_all = "snake_case")]
pub enum PrependScheme {
/// Specifies that the scheme should be prepended only once, on the first split.
Expand Down Expand Up @@ -63,12 +63,11 @@ impl<'de> Deserialize<'de> for Metaspace {

impl Metaspace {
pub fn new(replacement: char, add_prefix_space: bool) -> Self {
Self {
Self::new_with_prepend_scheme(
replacement,
str_rep: replacement.to_string(),
add_prefix_space,
prepend_scheme: PrependScheme::Always, // always prepend for legacy purpose
}
PrependScheme::Always, // always prepend for legacy purpose
)
}

pub fn new_with_prepend_scheme(
Expand All @@ -93,8 +92,8 @@ impl Metaspace {
self.str_rep = replacement.to_string();
}

pub fn get_prepend_scheme(&self) -> PrependScheme {
self.prepend_scheme.clone()
pub fn get_prepend_scheme(&self) -> &PrependScheme {
&self.prepend_scheme
}

pub fn set_prepend_scheme(&mut self, scheme: PrependScheme) {
Expand Down

0 comments on commit e68bcc3

Please sign in to comment.