From 190e7f81987ec94875829b26fcba75a50e142c8f Mon Sep 17 00:00:00 2001 From: Antoni Orfin Date: Wed, 1 Aug 2018 10:45:45 +0200 Subject: [PATCH] Implements PDF.js as a primary pdf extraction library --- README.md | 2 +- lib/extractors/pdf-js.js | 81 ++++++++++++++++++++++++++++++++++++++++ lib/extractors/pdf.js | 2 +- package.json | 19 +++++----- yarn.lock | 80 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 11 deletions(-) create mode 100644 lib/extractors/pdf-js.js diff --git a/README.md b/README.md index 5b013cb..5036c57 100644 --- a/README.md +++ b/README.md @@ -41,11 +41,11 @@ npm install textract Note, if any of the requirements below are missing, textract will run and extract all files for types it is capable. Not having these items installed does not prevent you from using textract, it just prevents you from extracting those specific files. -* `PDF` extraction requires `pdftotext` be installed, [link](http://www.foolabs.com/xpdf/download.html) * `DOC` extraction requires `antiword` be installed, [link](http://www.winfield.demon.nl/), unless on OSX in which case textutil (installed by default) is used. * `RTF` extraction requires `unrtf` be installed, [link](https://www.gnu.org/software/unrtf/), unless on OSX in which case textutil (installed by default) is used. * `PNG`, `JPG` and `GIF` require `tesseract` to be available, [link](http://code.google.com/p/tesseract-ocr/). Images need to be pretty clear, high DPI and made almost entirely of just text for `tesseract` to be able to accurately extract the text. * `DXF` extraction requires `drawingtotext` be available, [link](https://github.com/davidworkman9/drawingtotext) +* By default, PDF are extracted with PDF.js library that is not relying on any 3rd party binaries. However to use pdftotext, pass "application/pdf-pdftotext" as a document type. This extraction requires `pdftotext` to be installed, [link](http://www.foolabs.com/xpdf/download.html) ## Configuration diff --git a/lib/extractors/pdf-js.js b/lib/extractors/pdf-js.js new file mode 100644 index 0000000..f9f40ad --- /dev/null +++ b/lib/extractors/pdf-js.js @@ -0,0 +1,81 @@ +var path = require( 'path' ) + , pdfjs = require( 'pdfjs-dist' ); + +function _createError( filePath, e ) { + return new Error( 'Error extracting PDF text for file at [[ ' + + path.basename( filePath ) + ' ]], error: ' + e.message ); +} + +function _getTextFromPage( page, preserveLineBreaks ) { + return page.getTextContent().then( ( textContent ) => { + var text = '' + , lastY = -1; + + textContent.items.forEach( ( item ) => { + var itemStr = item.str + , itemY = item.transform[5]; + + if ( itemStr === '' ) return; + + if ( preserveLineBreaks && lastY !== itemY ) { + text = text.trim(); + text += '\n'; + } + + text += itemStr; + lastY = itemY; + }); + + return text.trim(); + }); +} + +function _isArrayBuffer( v ) { + return typeof v === 'object' && v !== null && v.byteLength !== undefined; +} + +function _createOptions( filePath, baseOptions ) { + var pdftotextOptions = baseOptions.pdftotextOptions || {} + , options = { + password: pdftotextOptions.userPassword, + }; + + if ( _isArrayBuffer( filePath ) ) { + options.data = filePath; + } else { + options.url = filePath; + } + + return options; +} + +function extractText( filePath, baseOptions, cb ) { + var options = baseOptions || {} + , pdfJsOptions = _createOptions( filePath, options ) + , preserveLineBreaks = options.preserveLineBreaks; + + pdfjs.getDocument( pdfJsOptions ).then( ( pdf ) => { + var pagePromises = [] + , pageNumber; + + for ( pageNumber = 1; pageNumber <= pdf.numPages; ++pageNumber ) { + pagePromises.push( pdf.getPage( pageNumber ) ); + } + + return Promise.all( pagePromises ); + }) + .then( ( pages ) => Promise.all( pages.map( ( page ) => + _getTextFromPage( page, preserveLineBreaks ) + ) ) ) + .then( ( textFromPages ) => { + cb( null, textFromPages.join( '\n' ) ); + }) + .catch( ( e ) => { + cb( _createError( filePath, e ), null ); + }); +} + +module.exports = { + types: ['application/pdf'], + extract: extractText +}; diff --git a/lib/extractors/pdf.js b/lib/extractors/pdf.js index 656729d..6273724 100644 --- a/lib/extractors/pdf.js +++ b/lib/extractors/pdf.js @@ -37,7 +37,7 @@ function testForBinary( options, cb ) { } module.exports = { - types: ['application/pdf'], + types: ['application/pdf-pdftotext'], extract: extractText, test: testForBinary }; diff --git a/package.json b/package.json index a0c9662..4e22a92 100644 --- a/package.json +++ b/package.json @@ -46,27 +46,28 @@ "otg" ], "dependencies": { - "mime": "2.2.0", - "pdf-text-extract": "1.3.1", - "xpath": "0.0.23", - "xmldom": "0.1.27", - "j": "0.4.3", "cheerio": "0.22.0", - "marked": "0.3.17", - "meow": "3.7.0", "got": "5.7.1", "html-entities": "1.2.0", "iconv-lite": "0.4.15", + "j": "0.4.3", "jschardet": "1.4.1", + "marked": "0.3.17", + "meow": "3.7.0", + "mime": "2.2.0", + "pdf-text-extract": "1.3.1", + "pdfjs-dist": "^2.0.489", + "xmldom": "0.1.27", + "xpath": "0.0.23", "yauzl": "2.7.0" }, "devDependencies": { "chai": "1.5.0", "eslint": "2.11.1", "eslint-config-airbnb": "^9.0.1", - "eslint-plugin-react": "^5.1.1", - "eslint-plugin-jsx-a11y": "^1.2.0", "eslint-plugin-import": "^1.7.0 ", + "eslint-plugin-jsx-a11y": "^1.2.0", + "eslint-plugin-react": "^5.1.1", "mocha": "1.9.0" }, "scripts": { diff --git a/yarn.lock b/yarn.lock index ef2b317..559af1b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,6 +23,10 @@ ajv-keywords@^1.0.0: version "1.5.1" resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-1.5.1.tgz#314dd0a4b3368fad3dfcdc54ede6171b886daf3c" +ajv-keywords@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-3.2.0.tgz#e86b819c602cf8821ad637413698f1dec021847a" + ajv@^4.7.0: version "4.11.8" resolved "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" @@ -30,6 +34,15 @@ ajv@^4.7.0: co "^4.6.0" json-stable-stringify "^1.0.1" +ajv@^6.1.0: + version "6.5.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.5.2.tgz#678495f9b82f7cca6be248dd92f59bff5e1f4360" + dependencies: + fast-deep-equal "^2.0.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.1" + ansi-escapes@^1.1.0: version "1.4.0" resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz#d3a8a83b319aa67793662b13e761c7911422306e" @@ -78,6 +91,10 @@ balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" +big.js@^3.1.3: + version "3.2.0" + resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e" + boolbase@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" @@ -376,6 +393,10 @@ duplexer2@^0.1.4: dependencies: readable-stream "^2.0.2" +emojis-list@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389" + entities@^1.1.1, entities@~1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0" @@ -584,6 +605,14 @@ exit-on-epipe@, exit-on-epipe@~1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" +fast-deep-equal@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" + +fast-json-stable-stringify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" + fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -903,12 +932,20 @@ jschardet@1.4.1: version "1.4.1" resolved "https://registry.npmjs.org/jschardet/-/jschardet-1.4.1.tgz#5e0f8966ddbe897f6d287e2196bfe0cf3a0090ec" +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + json-stable-stringify@^1.0.0, json-stable-stringify@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" dependencies: jsonify "~0.0.0" +json5@^0.5.0: + version "0.5.1" + resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -944,6 +981,14 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +loader-utils@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.1.0.tgz#c98aef488bcceda2ffb5e2de646d6a754429f5cd" + dependencies: + big.js "^3.1.3" + emojis-list "^2.0.0" + json5 "^0.5.0" + lodash.assignin@^4.0.9: version "4.2.0" resolved "https://registry.npmjs.org/lodash.assignin/-/lodash.assignin-4.2.0.tgz#ba8df5fb841eb0a3e8044232b0e263a8dc6a28a2" @@ -1106,6 +1151,10 @@ next-tick@1: version "1.0.0" resolved "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" +node-ensure@^0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" + node-status-codes@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz#5ae5541d024645d32a58fcddc9ceecea7ae3ac2f" @@ -1200,6 +1249,13 @@ pdf-text-extract@1.3.1: dependencies: yargs "^1.2.5" +pdfjs-dist@^2.0.489: + version "2.0.489" + resolved "https://registry.yarnpkg.com/pdfjs-dist/-/pdfjs-dist-2.0.489.tgz#63e54b292a86790a454697eb44d4347b8fbfad27" + dependencies: + node-ensure "^0.0.0" + worker-loader "^1.1.1" + pend@~1.2.0: version "1.2.0" resolved "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" @@ -1254,6 +1310,10 @@ progress@^1.1.8: version "1.1.8" resolved "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be" +punycode@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + read-all-stream@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz#35c3e177f2078ef789ee4bfafa4373074eaef4fa" @@ -1353,6 +1413,13 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" +schema-utils@^0.4.0: + version "0.4.5" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.5.tgz#21836f0608aac17b78f9e3e24daff14a5ca13a3e" + dependencies: + ajv "^6.1.0" + ajv-keywords "^3.1.0" + "semver@2 || 3 || 4 || 5": version "5.5.0" resolved "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" @@ -1497,6 +1564,12 @@ unzip-response@^1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz#b984f0877fc0a89c2c773cc1ef7b5b232b5b06fe" +uri-js@^4.2.1: + version "4.2.2" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.2.2.tgz#94c540e1ff772956e2299507c010aea6c8838eb0" + dependencies: + punycode "^2.1.0" + url-parse-lax@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73" @@ -1528,6 +1601,13 @@ wordwrap@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" +worker-loader@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-1.1.1.tgz#920d74ddac6816fc635392653ed8b4af1929fd92" + dependencies: + loader-utils "^1.0.0" + schema-utils "^0.4.0" + wrappy@1: version "1.0.2" resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"