Return empty tokens array if text is empty after normalization

huggingface · Jan 25, 2024 · 5a9fc6a · 5a9fc6a
1 parent b07336d
commit 5a9fc6a
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2744,6 +2744,12 @@ export class PreTrainedTokenizer extends Callable {
                     x = this.normalizer(x);
                 }
 
+                // If, after normalization, this section is empty (e.g., trimming whitespace),
+                // we return an empty array
+                if (x.length === 0) {
+                    return [];
+                }
+
                 const sectionTokens = (this.pre_tokenizer !== null) ? this.pre_tokenizer(x, {
                     section_index,
                 }) : [x];