Update tokenizers.js

huggingface · Jan 12, 2024 · 2df14fc · 2df14fc
1 parent 935274f
commit 2df14fc
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2463,7 +2463,7 @@ export class PreTrainedTokenizer extends Callable {
         this.sep_token = this.getToken('sep_token');
         this.sep_token_id = this.model.tokens_to_ids.get(this.sep_token);
 
-        this.unk_token = this.getToken(tokenizerConfig, 'unk_token');
+        this.unk_token = this.getToken('unk_token');
         this.unk_token_id = this.model.tokens_to_ids.get(this.unk_token);
 
         this.model_max_length = tokenizerConfig.model_max_length;