From 63d6ff46b1c45f56dde71bebb5b68e4f4ea02640 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 25 Nov 2024 16:04:08 +0100 Subject: [PATCH] [JS] Add sources for nodejs package of tokenizers (#312) * Add sources for nodejs package of tokenizers * Update version in package.json * Download binaries script refactoring * Add .npmignore * Replace path to linux archive at the storage to centos7 * Update binary version in package.json to 2024.5.0 * Add openvino-node as dependency and use binary manager from it * Update openvino-node package to 2024.5.0 * Specify vesion of openvino-tokenizers-node package as preview 2024.5.0-preview --------- Co-authored-by: Mikhail Ryzhov --- js/.gitignore | 2 + js/.npmignore | 5 + js/README.md | 30 ++ js/openvino-tokenizers.js | 60 ++++ js/package-lock.json | 425 +++++++++++++++++++++++++++ js/package.json | 38 +++ js/scripts/download-runtime.js | 91 ++++++ js/tests/openvino-tokenizers.test.js | 103 +++++++ 8 files changed, 754 insertions(+) create mode 100644 js/.gitignore create mode 100644 js/.npmignore create mode 100644 js/README.md create mode 100644 js/openvino-tokenizers.js create mode 100644 js/package-lock.json create mode 100644 js/package.json create mode 100644 js/scripts/download-runtime.js create mode 100644 js/tests/openvino-tokenizers.test.js diff --git a/js/.gitignore b/js/.gitignore new file mode 100644 index 00000000..8901c5b4 --- /dev/null +++ b/js/.gitignore @@ -0,0 +1,2 @@ +bin +node_modules diff --git a/js/.npmignore b/js/.npmignore new file mode 100644 index 00000000..67caf5ca --- /dev/null +++ b/js/.npmignore @@ -0,0 +1,5 @@ +bin +tests +thirdparty + +*.tgz diff --git a/js/README.md b/js/README.md new file mode 100644 index 00000000..194cce90 --- /dev/null +++ b/js/README.md @@ -0,0 +1,30 @@ +# OpenVINO Tokenizers + +This package contains binaries extension for +[openvino-node](https://www.npmjs.com/package/openvino-node) package + +## Installation + +```bash +npm install openvino-tokenizers-node +``` +After package installation script that downloads and extracts binaries will be executed. Binaries will be placed in the `node_modules/openvino-tokenizers-node/bin` directory. + +This package is a part of [openvino-node](https://www.npmjs.com/package/openvino-node) package and should be used with it. Version of this package should be the same as version of openvino-node package. + +## Usage + +```javascript +const { addon: ov } = require('openvino-node'); +const openvinoTokenizers = require('openvino-tokenizers-node'); + +const core = new ov.Core(); +core.addExtension(openvinoTokenizers.path); // Add tokenizers extension + +// Load tokenizers model +// ... +``` + +[License](https://github.com/openvinotoolkit/openvino/blob/master/LICENSE) + +Copyright © 2018-2024 Intel Corporation diff --git a/js/openvino-tokenizers.js b/js/openvino-tokenizers.js new file mode 100644 index 00000000..b850cc4c --- /dev/null +++ b/js/openvino-tokenizers.js @@ -0,0 +1,60 @@ +const os = require('node:os'); +const path = require('node:path'); + +module.exports = { + path: getPathToBinary({ + platform: os.platform(), + arch: os.arch(), + }), + getPathToBinary, +}; + +function getPathToBinary(osProps) { + const { arch, platform } = osProps; + + if (platform === 'win32' && arch !== 'x64') + throw new Error(`Version for windows and '${arch}' is not supported.`); + + return path.join( + __dirname, + 'bin/runtime', + libOrBin(platform), + getDirnameByArch(arch), + platform === 'linux' ? '' : 'Release', + getBinaryFilename(platform), + ); +} + +function getDirnameByArch(arch) { + switch (arch) { + case 'x64': + return 'intel64'; + case 'arm64': + case 'armhf': + return 'arm64'; + default: + throw new Error(`Unsupported architecture: ${arch}`); + } +} + +function libOrBin(platform) { + switch (platform) { + case 'win32': + return 'bin'; + default: + return 'lib'; + } +} + +function getBinaryFilename(platform) { + switch (platform) { + case 'win32': + return 'openvino_tokenizers.dll'; + case 'linux': + return 'libopenvino_tokenizers.so'; + case 'darwin': + return 'libopenvino_tokenizers.dylib'; + default: + throw new Error(`Unsupported platform: ${platform}`); + } +} diff --git a/js/package-lock.json b/js/package-lock.json new file mode 100644 index 00000000..88a7eb2f --- /dev/null +++ b/js/package-lock.json @@ -0,0 +1,425 @@ +{ + "name": "openvino-tokenizers-node", + "version": "2025.0.0-preview", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openvino-tokenizers-node", + "version": "2025.0.0-preview", + "hasInstallScript": true, + "license": "Apache-2.0", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "openvino-node": "2024.5.0-0" + } + }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, + "node_modules/bare-events": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.0.tgz", + "integrity": "sha512-/E8dDe9dsbLyh2qrZ64PEPadOQ0F4gbl1sUJOrmph7xOiIxfY8vwab/4bFLh4Y88/Hk/ujKcrQKc+ps0mv873A==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.5.tgz", + "integrity": "sha512-SlE9eTxifPDJrT6YgemQ1WGFleevzwY+XAP1Xqgl56HtcrisC2CHCZ2tq6dBpcH2TnNxwUEUGhweo+lrQtYuiw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.4.tgz", + "integrity": "sha512-z3UiI2yi1mK0sXeRdc4O1Kk8aOa/e+FNWZcTiPB/dfTWyLypuE99LibgRaQki914Jq//yAWylcAt+mknKdixRQ==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", + "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.3.2.tgz", + "integrity": "sha512-EFZHSIBkDgSHIwj2l2QZfP4U5OcD4xFAOwhSb/vlr9PIqyGJGvB/nfClJbcnh3EY4jtPE4zsb5ztae96bVF79A==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.20.0" + } + }, + "node_modules/browserify-zlib": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.1.4.tgz", + "integrity": "sha512-19OEpq7vWgsH6WkvkBJQDFvJS1uPcbFOQ4v9CU839dO+ZZXUZO6XpE6hNCqvlIIj+4fZvRiJ6DsAQ382GwiyTQ==", + "license": "MIT", + "dependencies": { + "pako": "~0.2.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/gunzip-maybe": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", + "integrity": "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==", + "license": "MIT", + "dependencies": { + "browserify-zlib": "^0.1.4", + "is-deflate": "^1.0.0", + "is-gzip": "^1.0.0", + "peek-stream": "^1.1.0", + "pumpify": "^1.3.3", + "through2": "^2.0.3" + }, + "bin": { + "gunzip-maybe": "bin.js" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is-deflate": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-deflate/-/is-deflate-1.0.0.tgz", + "integrity": "sha512-YDoFpuZWu1VRXlsnlYMzKyVRITXj7Ej/V9gXQ2/pAe7X1J7M/RNOqaIYi6qUn+B7nGyB9pDXrv02dsB58d2ZAQ==", + "license": "MIT" + }, + "node_modules/is-gzip": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-gzip/-/is-gzip-1.0.0.tgz", + "integrity": "sha512-rcfALRIb1YewtnksfRIHGcIY93QnK8BIQ/2c9yDYcG/Y6+vRoJuTWBmmSEbyLLYtXm7q35pHOHbZFQBaLrhlWQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/openvino-node": { + "version": "2024.5.0-0", + "resolved": "https://registry.npmjs.org/openvino-node/-/openvino-node-2024.5.0-0.tgz", + "integrity": "sha512-SgvHH3OdOXyMu5iZx0oBFWn7yIu3uB54IIfmXFKlyhHbSjO+3ph+DauUdlUkp2DGETR7bzq7+cPyyroeOF7qqQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "gunzip-maybe": "^1.4.2", + "https-proxy-agent": "^7.0.2", + "tar-fs": "^3.0.4" + }, + "engines": { + "node": ">=21.0.0" + } + }, + "node_modules/pako": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/pako/-/pako-0.2.9.tgz", + "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", + "license": "MIT" + }, + "node_modules/peek-stream": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", + "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "duplexify": "^3.5.0", + "through2": "^2.0.3" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "license": "MIT", + "dependencies": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + } + }, + "node_modules/pumpify/node_modules/pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, + "node_modules/streamx": { + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.1.tgz", + "integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/tar-fs": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", + "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.1.tgz", + "integrity": "sha512-x9v3H/lTKIJKQQe7RPQkLfKAnc9lUTkWDypIQgTzPJAq+5/GCDHonmshfvlsNSj58yyshbIJJDLmU15qNERrXQ==", + "license": "Apache-2.0" + }, + "node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "license": "MIT", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/js/package.json b/js/package.json new file mode 100644 index 00000000..2da4ff7b --- /dev/null +++ b/js/package.json @@ -0,0 +1,38 @@ +{ + "name": "openvino-tokenizers-node", + "version": "2024.5.0-preview", + "description": "OpenVINO™ Tokenizers adds text processing operations to openvino-node package", + "repository": { + "url": "https://github.com/openvinotoolkit/openvino.git", + "type": "git" + }, + "license": "Apache-2.0", + "main": "openvino-tokenizers.js", + "os": [ + "win32", + "darwin", + "linux" + ], + "scripts": { + "postinstall": "npm run install_runtime", + "download_runtime": "node ./scripts/download-runtime.js", + "install_runtime": "npm run download_runtime -- --ignore-if-exists", + "test": "node --test ./tests/*.test.js" + }, + "binary": { + "version": "2024.5.0.0", + "module_path": "./bin/", + "remote_path": "./repositories/openvino_tokenizers/packages/{version}/", + "package_name": "openvino_tokenizers_{platform}_{version}_{arch}.{extension}", + "host": "https://storage.openvinotoolkit.org" + }, + "keywords": [ + "OpenVINO", + "tokenizers", + "OpenVINO tokenizers" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "openvino-node": "2024.5.0-0" + } +} diff --git a/js/scripts/download-runtime.js b/js/scripts/download-runtime.js new file mode 100644 index 00000000..2e46e7b2 --- /dev/null +++ b/js/scripts/download-runtime.js @@ -0,0 +1,91 @@ + +const os = require('node:os'); +const AdmZip = require('adm-zip'); +const { join } = require('node:path'); +const BinaryManager = require('openvino-node/scripts/lib/binary-manager'); + +const packageJson = require('../package.json'); + +class TokenizersBinaryManager extends BinaryManager { + getVersion() { + return `${ this.binaryConfig.version || this.version }.0`; + } + + getPlatformLabel() { + return this.convertPlatformLabel(os.platform()); + } + + getArchLabel() { + return this.convertArchLabel(os.arch()); + } + + getExtension() { + return os.platform() === 'win32' ? 'zip' : 'tar.gz'; + } + + convertPlatformLabel(platform) { + switch(platform) { + case 'win32': + return 'windows'; + case 'linux': + return 'centos7'; + case 'darwin': + return 'macos_12_6'; + } + } + + convertArchLabel(arch) { + switch(arch) { + case 'x64': + return 'x86_64'; + case 'arm64': + case 'armhf': + return 'arm64'; + } + } + + /** + * Unarchive tar, tar.gz or zip archives. + * + * @function unarchive + * @param {string} archivePath - Path to archive. + * @param {string} dest - Path where to unpack. + * @returns {Promise} + */ + static unarchive(archivePath, dest) { + if (archivePath.endsWith('.zip')) { + const zip = new AdmZip(archivePath); + + return new Promise((resolve, reject) => { + zip.extractAllToAsync(dest, true, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } + + return BinaryManager.unarchive(archivePath, dest); + } +} + +if (require.main === module) main(); + +async function main() { + if (!TokenizersBinaryManager.isCompatible()) process.exit(1); + + const force = process.argv.includes('-f'); + const ignoreIfExists = process.argv.includes('--ignore-if-exists'); + + const { env } = process; + const proxy = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; + + await TokenizersBinaryManager.prepareBinary( + join(__dirname, '..'), + packageJson.binary.version || packageJson.version, + packageJson.binary, + { force, ignoreIfExists, proxy }, + ); +} diff --git a/js/tests/openvino-tokenizers.test.js b/js/tests/openvino-tokenizers.test.js new file mode 100644 index 00000000..26fefb29 --- /dev/null +++ b/js/tests/openvino-tokenizers.test.js @@ -0,0 +1,103 @@ +const assert = require('node:assert/strict'); +const { test, describe } = require('node:test'); + +const { getPathToBinary } = require('../openvino-tokenizers'); + +const parentDir = __dirname.split('/').slice(0, -1).join('/'); + +describe('getPathToBinary for x64', () => { + test('should map win32 to windows and x64 to intel64', () => { + const path = getPathToBinary({ platform: 'win32', arch: 'x64' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/bin/intel64/Release/openvino_tokenizers.dll', + ); + }); + + test('should map linux to linux and x64 to intel64', () => { + const path = getPathToBinary({ platform: 'linux', arch: 'x64' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/lib/intel64/libopenvino_tokenizers.so', + ); + }); + + test('should map darwin to macos and x64 to intel64', () => { + const path = getPathToBinary({ platform: 'darwin', arch: 'x64' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/lib/intel64/Release/libopenvino_tokenizers.dylib', + ); + }); +}); + +describe('getPathToBinary for arm64', () => { + test('should map win32 to windows and arm64 to arm64', () => { + assert.throws( + () => getPathToBinary({ platform: 'win32', arch: 'arm64' }), + new Error(`Version for windows and 'arm64' is not supported.`), + ); + }); + + test('should map linux to linux and arm64 to arm64', () => { + const path = getPathToBinary({ platform: 'linux', arch: 'arm64' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/lib/arm64/libopenvino_tokenizers.so', + ); + }); + + test('should map darwin to macos and arm64 to arm64', () => { + const path = getPathToBinary({ platform: 'darwin', arch: 'arm64' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/lib/arm64/Release/libopenvino_tokenizers.dylib', + ); + }); +}); + +describe('getPathToBinary for armhf', () => { + test('should map win32 to windows and armhf to arm64', () => { + assert.throws( + () => getPathToBinary({ platform: 'win32', arch: 'armhf' }), + new Error(`Version for windows and 'armhf' is not supported.`), + ); + }); + + test('should map linux to linux and armhf to arm64', () => { + const path = getPathToBinary({ platform: 'linux', arch: 'armhf' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/lib/arm64/libopenvino_tokenizers.so', + ); + }); + + test('should map darwin to macos and armhf to arm64', () => { + const path = getPathToBinary({ platform: 'darwin', arch: 'armhf' }); + const [, relatedFromBin] = path.split(parentDir); + + assert.equal( + relatedFromBin, + '/bin/runtime/lib/arm64/Release/libopenvino_tokenizers.dylib', + ); + }); +}); + +describe('getPathToBinary for unsupported platform', () => { + test('should throw an error for unsupported platform', () => { + assert.throws(() => getPathToBinary({ platform: 'unsupported', arch: 'x64' })); + }); +}); +