From 8f30a282460c0aaf6791563c8b5f3bf5fd64c2f5 Mon Sep 17 00:00:00 2001
From: Joshua Lochner <admin@xenova.com>
Date: Tue, 3 Oct 2023 11:53:22 +0200
Subject: [PATCH] Do not add token types for `FalconTokenizer`

---
 src/tokenizers.js | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/tokenizers.js b/src/tokenizers.js
index 49c3a71ca..74d179a6d 100644
--- a/src/tokenizers.js
+++ b/src/tokenizers.js
@@ -2705,12 +2705,7 @@ export class CodeLlamaTokenizer extends PreTrainedTokenizer { }
 export class XLMRobertaTokenizer extends PreTrainedTokenizer { }
 export class MPNetTokenizer extends PreTrainedTokenizer { }
 
-export class FalconTokenizer extends PreTrainedTokenizer {
-    /** @type {add_token_types} */
-    prepare_model_inputs(inputs) {
-        return add_token_types(inputs);
-    }
-}
+export class FalconTokenizer extends PreTrainedTokenizer { }
 
 export class GPTNeoXTokenizer extends PreTrainedTokenizer { }