Do not add token types for FalconTokenizer

huggingface · Oct 3, 2023 · 8f30a28 · 8f30a28
1 parent 965a715
commit 8f30a28
Showing 1 changed file with 1 addition and 6 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
 export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
 export class MPNetTokenizer extends PreTrainedTokenizer { }
 
-export class FalconTokenizer extends PreTrainedTokenizer {
-    /** @type {add_token_types} */
-    prepare_model_inputs(inputs) {
-        return add_token_types(inputs);
-    }
-}
+export class FalconTokenizer extends PreTrainedTokenizer { }
 
 export class GPTNeoXTokenizer extends PreTrainedTokenizer { }