From c0493dde8e34247dbe2f06cb1d76c184d97422f4 Mon Sep 17 00:00:00 2001 From: alice-was-here Date: Tue, 25 Jun 2024 15:00:57 +0800 Subject: [PATCH 1/6] Add support for @aws-sdk version 3 This should resolve this issue: https://github.com/ZJONSSON/node-unzipper/issues/241 I have used a variation of @Sljux's solution to make the interface compatible with the current `stream` interface. --- lib/Open/index.js | 36 ++++++++++++++++++++++++++++++++++++ package.json | 3 +++ 2 files changed, 39 insertions(+) diff --git a/lib/Open/index.js b/lib/Open/index.js index 83c349b..ab0da12 100644 --- a/lib/Open/index.js +++ b/lib/Open/index.js @@ -1,6 +1,7 @@ const fs = require('graceful-fs'); const directory = require('./directory'); const Stream = require('stream'); +const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); module.exports = { buffer: function(buffer, options) { @@ -93,7 +94,42 @@ module.exports = { return directory(source, options); }, + s3_v3: function (client, params, options) { + const source = { + size: async () => { + const head = await client.send( + new HeadObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + }) + ); + + return head.ContentLength ?? 0; + }, + stream: (offset, length) => { + const stream = Stream.PassThrough(); + const end = length ? offset + length : ""; + client + .send( + new GetObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + Range: `bytes=${offset}-${end}`, + }) + ) + .then((response) => { + response.Body.pipe(stream); + }) + .catch((error) => { + stream.emit("error", error); + }); + return stream; + }, + }; + + return directory(source, options); + }, custom: function(source, options) { return directory(source, options); } diff --git a/package.json b/package.json index 528f510..ab009c1 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,9 @@ "tap": "^12.7.0", "temp": ">= 0.4.0 < 1" }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.0.0" + }, "directories": { "example": "examples", "test": "test" From b71f8b09232078541bf67634e70375da95aeb699 Mon Sep 17 00:00:00 2001 From: alice-was-here Date: Mon, 8 Jul 2024 08:58:22 +0800 Subject: [PATCH 2/6] Address PR comments --- lib/Open/index.js | 104 +++++++++++++++++++++++----------------------- package.json | 2 +- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/lib/Open/index.js b/lib/Open/index.js index ab0da12..ddecbe0 100644 --- a/lib/Open/index.js +++ b/lib/Open/index.js @@ -1,100 +1,98 @@ -const fs = require('graceful-fs'); -const directory = require('./directory'); -const Stream = require('stream'); -const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); +const fs = require("graceful-fs"); +const directory = require("./directory"); +const Stream = require("stream"); module.exports = { - buffer: function(buffer, options) { + buffer: function (buffer, options) { const source = { - stream: function(offset, length) { + stream: function (offset, length) { const stream = Stream.PassThrough(); const end = length ? offset + length : undefined; stream.end(buffer.slice(offset, end)); return stream; }, - size: function() { + size: function () { return Promise.resolve(buffer.length); - } + }, }; return directory(source, options); }, - file: function(filename, options) { + file: function (filename, options) { const source = { - stream: function(start, length) { + stream: function (start, length) { const end = length ? start + length : undefined; - return fs.createReadStream(filename, {start, end}); + return fs.createReadStream(filename, { start, end }); }, - size: function() { - return new Promise(function(resolve, reject) { - fs.stat(filename, function(err, d) { - if (err) - reject(err); - else - resolve(d.size); + size: function () { + return new Promise(function (resolve, reject) { + fs.stat(filename, function (err, d) { + if (err) reject(err); + else resolve(d.size); }); }); - } + }, }; return directory(source, options); }, - url: function(request, params, options) { - if (typeof params === 'string') - params = {url: params}; - if (!params.url) - throw 'URL missing'; + url: function (request, params, options) { + if (typeof params === "string") params = { url: params }; + if (!params.url) throw "URL missing"; params.headers = params.headers || {}; const source = { - stream : function(offset, length) { + stream: function (offset, length) { const options = Object.create(params); - const end = length ? offset + length : ''; + const end = length ? offset + length : ""; options.headers = Object.create(params.headers); - options.headers.range = 'bytes='+offset+'-' + end; + options.headers.range = "bytes=" + offset + "-" + end; return request(options); }, - size: function() { - return new Promise(function(resolve, reject) { + size: function () { + return new Promise(function (resolve, reject) { const req = request(params); - req.on('response', function(d) { - req.abort(); - if (!d.headers['content-length']) - reject(new Error('Missing content length header')); - else - resolve(d.headers['content-length']); - }).on('error', reject); + req + .on("response", function (d) { + req.abort(); + if (!d.headers["content-length"]) + reject(new Error("Missing content length header")); + else resolve(d.headers["content-length"]); + }) + .on("error", reject); }); - } + }, }; return directory(source, options); }, - s3 : function(client, params, options) { + s3: function (client, params, options) { const source = { - size: function() { - return new Promise(function(resolve, reject) { - client.headObject(params, function(err, d) { - if (err) - reject(err); - else - resolve(d.ContentLength); + size: function () { + return new Promise(function (resolve, reject) { + client.headObject(params, function (err, d) { + if (err) reject(err); + else resolve(d.ContentLength); }); }); }, - stream: function(offset, length) { + stream: function (offset, length) { const d = {}; - for (const key in params) - d[key] = params[key]; - const end = length ? offset + length : ''; - d.Range = 'bytes='+offset+'-' + end; + for (const key in params) d[key] = params[key]; + const end = length ? offset + length : ""; + d.Range = "bytes=" + offset + "-" + end; return client.getObject(d).createReadStream(); - } + }, }; return directory(source, options); }, s3_v3: function (client, params, options) { + const { + GetObjectCommand, + HeadObjectCommand, + } = require("@aws-sdk/client-s3"); + const source = { size: async () => { const head = await client.send( @@ -130,7 +128,7 @@ module.exports = { return directory(source, options); }, - custom: function(source, options) { + custom: function (source, options) { return directory(source, options); - } + }, }; diff --git a/package.json b/package.json index ab009c1..148ea85 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "tap": "^12.7.0", "temp": ">= 0.4.0 < 1" }, - "peerDependencies": { + "optionalDependencies": { "@aws-sdk/client-s3": "^3.0.0" }, "directories": { From 5d05b72534fe780a082bce9c81b4404cfe079cbe Mon Sep 17 00:00:00 2001 From: alice-was-here Date: Tue, 9 Jul 2024 08:23:19 +0800 Subject: [PATCH 3/6] Revert "Address PR comments" This reverts commit b71f8b09232078541bf67634e70375da95aeb699. --- lib/Open/index.js | 104 +++++++++++++++++++++++----------------------- package.json | 2 +- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/lib/Open/index.js b/lib/Open/index.js index ddecbe0..ab0da12 100644 --- a/lib/Open/index.js +++ b/lib/Open/index.js @@ -1,98 +1,100 @@ -const fs = require("graceful-fs"); -const directory = require("./directory"); -const Stream = require("stream"); +const fs = require('graceful-fs'); +const directory = require('./directory'); +const Stream = require('stream'); +const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); module.exports = { - buffer: function (buffer, options) { + buffer: function(buffer, options) { const source = { - stream: function (offset, length) { + stream: function(offset, length) { const stream = Stream.PassThrough(); const end = length ? offset + length : undefined; stream.end(buffer.slice(offset, end)); return stream; }, - size: function () { + size: function() { return Promise.resolve(buffer.length); - }, + } }; return directory(source, options); }, - file: function (filename, options) { + file: function(filename, options) { const source = { - stream: function (start, length) { + stream: function(start, length) { const end = length ? start + length : undefined; - return fs.createReadStream(filename, { start, end }); + return fs.createReadStream(filename, {start, end}); }, - size: function () { - return new Promise(function (resolve, reject) { - fs.stat(filename, function (err, d) { - if (err) reject(err); - else resolve(d.size); + size: function() { + return new Promise(function(resolve, reject) { + fs.stat(filename, function(err, d) { + if (err) + reject(err); + else + resolve(d.size); }); }); - }, + } }; return directory(source, options); }, - url: function (request, params, options) { - if (typeof params === "string") params = { url: params }; - if (!params.url) throw "URL missing"; + url: function(request, params, options) { + if (typeof params === 'string') + params = {url: params}; + if (!params.url) + throw 'URL missing'; params.headers = params.headers || {}; const source = { - stream: function (offset, length) { + stream : function(offset, length) { const options = Object.create(params); - const end = length ? offset + length : ""; + const end = length ? offset + length : ''; options.headers = Object.create(params.headers); - options.headers.range = "bytes=" + offset + "-" + end; + options.headers.range = 'bytes='+offset+'-' + end; return request(options); }, - size: function () { - return new Promise(function (resolve, reject) { + size: function() { + return new Promise(function(resolve, reject) { const req = request(params); - req - .on("response", function (d) { - req.abort(); - if (!d.headers["content-length"]) - reject(new Error("Missing content length header")); - else resolve(d.headers["content-length"]); - }) - .on("error", reject); + req.on('response', function(d) { + req.abort(); + if (!d.headers['content-length']) + reject(new Error('Missing content length header')); + else + resolve(d.headers['content-length']); + }).on('error', reject); }); - }, + } }; return directory(source, options); }, - s3: function (client, params, options) { + s3 : function(client, params, options) { const source = { - size: function () { - return new Promise(function (resolve, reject) { - client.headObject(params, function (err, d) { - if (err) reject(err); - else resolve(d.ContentLength); + size: function() { + return new Promise(function(resolve, reject) { + client.headObject(params, function(err, d) { + if (err) + reject(err); + else + resolve(d.ContentLength); }); }); }, - stream: function (offset, length) { + stream: function(offset, length) { const d = {}; - for (const key in params) d[key] = params[key]; - const end = length ? offset + length : ""; - d.Range = "bytes=" + offset + "-" + end; + for (const key in params) + d[key] = params[key]; + const end = length ? offset + length : ''; + d.Range = 'bytes='+offset+'-' + end; return client.getObject(d).createReadStream(); - }, + } }; return directory(source, options); }, s3_v3: function (client, params, options) { - const { - GetObjectCommand, - HeadObjectCommand, - } = require("@aws-sdk/client-s3"); - const source = { size: async () => { const head = await client.send( @@ -128,7 +130,7 @@ module.exports = { return directory(source, options); }, - custom: function (source, options) { + custom: function(source, options) { return directory(source, options); - }, + } }; diff --git a/package.json b/package.json index 148ea85..ab009c1 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "tap": "^12.7.0", "temp": ">= 0.4.0 < 1" }, - "optionalDependencies": { + "peerDependencies": { "@aws-sdk/client-s3": "^3.0.0" }, "directories": { From 3927030b906c210211adced8c793b7029d243ab7 Mon Sep 17 00:00:00 2001 From: alice-was-here Date: Tue, 9 Jul 2024 08:56:16 +0800 Subject: [PATCH 4/6] Add node16 + tests, make import conditional, revert formatting changes --- lib/Open/index.js | 8 ++++++-- package.json | 1 + test/openS3_v3.js | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 test/openS3_v3.js diff --git a/lib/Open/index.js b/lib/Open/index.js index ab0da12..81f95d5 100644 --- a/lib/Open/index.js +++ b/lib/Open/index.js @@ -1,7 +1,6 @@ const fs = require('graceful-fs'); const directory = require('./directory'); const Stream = require('stream'); -const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); module.exports = { buffer: function(buffer, options) { @@ -95,6 +94,7 @@ module.exports = { return directory(source, options); }, s3_v3: function (client, params, options) { + const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); const source = { size: async () => { const head = await client.send( @@ -104,7 +104,11 @@ module.exports = { }) ); - return head.ContentLength ?? 0; + if(!head.ContentLength) { + return 0; + } + + return head.ContentLength; }, stream: (offset, length) => { const stream = Stream.PassThrough(); diff --git a/package.json b/package.json index ab009c1..47aedc9 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "devDependencies": { "@eslint/js": "^9.2.0", "aws-sdk": "^2.1636.0", + "@aws-sdk/client-s3": "^3.0.0", "dirdiff": ">= 0.0.1 < 1", "eslint": "^9.2.0", "globals": "^15.2.0", diff --git a/test/openS3_v3.js b/test/openS3_v3.js new file mode 100644 index 0000000..6cb720b --- /dev/null +++ b/test/openS3_v3.js @@ -0,0 +1,25 @@ +const test = require('tap').test; +const fs = require('fs'); +const path = require('path'); +const unzip = require('../unzip'); + +const version = +process.version.replace('v', '').split('.')[0]; + +test("get content of a single file entry out of a zip", { skip: version < 16 }, function(t) { + const { S3Client } = require('@aws-sdk/client-s3'); + const client = new S3Client({ region: 'us-east-1' }); + + return unzip.Open.s3_v3(client, { Bucket: 'unzipper', Key: 'archive.zip' }) + .then(function(d) { + const file = d.files.filter(function(file) { + return file.path == 'file.txt'; + })[0]; + + return file.buffer() + .then(function(str) { + const fileStr = fs.readFileSync(path.join(__dirname, '../testData/compressed-standard/inflated/file.txt'), 'utf8'); + t.equal(str.toString(), fileStr); + t.end(); + }); + }); +}); From 2828d20fa743b94c034d639b6a9fd104e38f2c69 Mon Sep 17 00:00:00 2001 From: alice-was-here <82787803+alice-was-here@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:33:49 +0800 Subject: [PATCH 5/6] CI fixes --- package.json | 4 ++-- test/openS3_v3.js | 48 ++++++++++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 47aedc9..73a62e7 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "iconv-lite": "^0.4.24", "request": "^2.88.0", "stream-buffers": ">= 0.2.5 < 1", - "tap": "^12.7.0", + "tap": "^16.3.10", "temp": ">= 0.4.0 < 1" }, "peerDependencies": { @@ -60,6 +60,6 @@ ], "main": "unzip.js", "scripts": { - "test": "npx tap test/*.js --coverage-report=html --reporter=dot" + "test": "npx tap@^16.3.10 test/*.js --no-coverage --reporter=dot" } } diff --git a/test/openS3_v3.js b/test/openS3_v3.js index 6cb720b..378bf0d 100644 --- a/test/openS3_v3.js +++ b/test/openS3_v3.js @@ -1,25 +1,35 @@ -const test = require('tap').test; -const fs = require('fs'); -const path = require('path'); -const unzip = require('../unzip'); +const test = require("tap").test; +const unzip = require("../unzip"); -const version = +process.version.replace('v', '').split('.')[0]; +const version = +process.version.replace("v", "").split(".")[0]; -test("get content of a single file entry out of a zip", { skip: version < 16 }, function(t) { - const { S3Client } = require('@aws-sdk/client-s3'); - const client = new S3Client({ region: 'us-east-1' }); +test( + "get content of a single file entry out of a zip", + { skip: version < 16 }, + function (t) { + const { S3Client } = require("@aws-sdk/client-s3"); - return unzip.Open.s3_v3(client, { Bucket: 'unzipper', Key: 'archive.zip' }) - .then(function(d) { - const file = d.files.filter(function(file) { - return file.path == 'file.txt'; + const client = new S3Client({ + region: "us-east-1", + signer: { sign: async (request) => request }, + }); + + // These files are provided by AWS's open data registry project. + // https://github.com/awslabs/open-data-registry + + return unzip.Open.s3_v3(client, { + Bucket: "wikisum", + Key: "WikiSumDataset.zip", + }).then(function (d) { + const file = d.files.filter(function (file) { + return file.path == "WikiSumDataset/LICENSE.txt"; })[0]; - return file.buffer() - .then(function(str) { - const fileStr = fs.readFileSync(path.join(__dirname, '../testData/compressed-standard/inflated/file.txt'), 'utf8'); - t.equal(str.toString(), fileStr); - t.end(); - }); + return file.buffer().then(function (b) { + const firstLine = b.toString().split("\n")[0]; + t.equal(firstLine, "Attribution-NonCommercial-ShareAlike 3.0 Unported"); + t.end(); + }); }); -}); + } +); From 1afcab33094ccd5c4ff6303195e518e1c93db1a0 Mon Sep 17 00:00:00 2001 From: alice-was-here Date: Wed, 10 Jul 2024 21:51:54 +0800 Subject: [PATCH 6/6] Remove peer dep and tweak coverage --- package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package.json b/package.json index 73a62e7..63bab94 100644 --- a/package.json +++ b/package.json @@ -42,9 +42,6 @@ "tap": "^16.3.10", "temp": ">= 0.4.0 < 1" }, - "peerDependencies": { - "@aws-sdk/client-s3": "^3.0.0" - }, "directories": { "example": "examples", "test": "test" @@ -60,6 +57,6 @@ ], "main": "unzip.js", "scripts": { - "test": "npx tap@^16.3.10 test/*.js --no-coverage --reporter=dot" + "test": "npx tap test/*.js --coverage-report=html --lines=90 --functions=85 --statements=90 --branches=80 --reporter=dot" } }