Merge branch 'main' into add-depth-anything

huggingface · Jan 25, 2024 · 87b6ba2 · 87b6ba2
2 parents 84ddd85 + 4a6adda
commit 87b6ba2
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/src/tokenizers.js b/src/tokenizers.js
@@ -2744,6 +2744,12 @@ export class PreTrainedTokenizer extends Callable {
                     x = this.normalizer(x);
                 }
 
+                // If, after normalization, this section is empty (e.g., trimming whitespace),
+                // we return an empty array
+                if (x.length === 0) {
+                    return [];
+                }
+
                 const sectionTokens = (this.pre_tokenizer !== null) ? this.pre_tokenizer(x, {
                     section_index,
                 }) : [x];