diff --git a/src/tokenizers.js b/src/tokenizers.js index 49c3a71ca..74d179a6d 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { } export class XLMRobertaTokenizer extends PreTrainedTokenizer { } export class MPNetTokenizer extends PreTrainedTokenizer { } -export class FalconTokenizer extends PreTrainedTokenizer { - /** @type {add_token_types} */ - prepare_model_inputs(inputs) { - return add_token_types(inputs); - } -} +export class FalconTokenizer extends PreTrainedTokenizer { } export class GPTNeoXTokenizer extends PreTrainedTokenizer { }