Merge branch 'main' into add-depth-estimation

README.md

@@ -15,10 +15,13 @@
         <img alt="NPM" src="https://img.shields.io/npm/v/@xenova/transformers">
     </a>
     <a href="https://www.npmjs.com/package/@xenova/transformers">
-        <img alt="Downloads" src="https://img.shields.io/npm/dw/@xenova/transformers">
+        <img alt="NPM Downloads" src="https://img.shields.io/npm/dw/@xenova/transformers">
+    </a>
+    <a href="https://www.jsdelivr.com/package/npm/@xenova/transformers">
+        <img alt="jsDelivr Hits" src="https://img.shields.io/jsdelivr/npm/hw/@xenova/transformers">
     </a>
     <a href="https://github.com/xenova/transformers.js/blob/main/LICENSE">
-        <img alt="License" src="https://img.shields.io/github/license/xenova/transformers.js">
+        <img alt="License" src="https://img.shields.io/github/license/xenova/transformers.js?color=blue">
     </a>
     <a href="https://huggingface.co/docs/transformers.js/index">
         <img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/transformers.js/index.svg?down_color=red&down_message=offline&up_message=online">

docs/scripts/build_readme.py

@@ -17,10 +17,13 @@
         <img alt="NPM" src="https://img.shields.io/npm/v/@xenova/transformers">
     </a>
     <a href="https://www.npmjs.com/package/@xenova/transformers">
-        <img alt="Downloads" src="https://img.shields.io/npm/dw/@xenova/transformers">
+        <img alt="NPM Downloads" src="https://img.shields.io/npm/dw/@xenova/transformers">
+    </a>
+    <a href="https://www.jsdelivr.com/package/npm/@xenova/transformers">
+        <img alt="jsDelivr Hits" src="https://img.shields.io/jsdelivr/npm/hw/@xenova/transformers">
     </a>
     <a href="https://github.com/xenova/transformers.js/blob/main/LICENSE">
-        <img alt="License" src="https://img.shields.io/github/license/xenova/transformers.js">
+        <img alt="License" src="https://img.shields.io/github/license/xenova/transformers.js?color=blue">
     </a>
     <a href="https://huggingface.co/docs/transformers.js/index">
         <img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/transformers.js/index.svg?down_color=red&down_message=offline&up_message=online">

src/tokenizers.js

@@ -270,6 +270,7 @@ class WordPieceTokenizer extends TokenizerModel {
      * @param {Object} config.vocab A mapping of tokens to ids.
      * @param {string} config.unk_token The unknown token string.
      * @param {string} config.continuing_subword_prefix The prefix to use for continuing subwords.
+     * @param {number} [config.max_input_chars_per_word=100] The maximum number of characters per word.
      */
     constructor(config) {
         super(config);
@@ -291,6 +292,12 @@ class WordPieceTokenizer extends TokenizerModel {
          */
         this.unk_token = config.unk_token;
 
+        /**
+         * The maximum number of characters allowed per word.
+         * @type {number}
+         */
+        this.max_input_chars_per_word = config.max_input_chars_per_word ?? 100;
+
         /**
          * An array of tokens.
          * @type {string[]}
@@ -310,10 +317,10 @@ class WordPieceTokenizer extends TokenizerModel {
         let outputTokens = [];
         for (let token of tokens) {
             let chars = [...token];
-            // TODO add
-            // if len(chars) > self.max_input_chars_per_word:
-            //     output_tokens.append(self.unk_token)
-            //     continue
+            if (chars.length > this.max_input_chars_per_word) {
+                outputTokens.push(this.unk_token);
+                continue;
+            }
 
             let isUnknown = false;
             let start = 0;