diff --git a/lib/Open/index.js b/lib/Open/index.js index 83c349b..81f95d5 100644 --- a/lib/Open/index.js +++ b/lib/Open/index.js @@ -93,7 +93,47 @@ module.exports = { return directory(source, options); }, + s3_v3: function (client, params, options) { + const { GetObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); + const source = { + size: async () => { + const head = await client.send( + new HeadObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + }) + ); + + if(!head.ContentLength) { + return 0; + } + + return head.ContentLength; + }, + stream: (offset, length) => { + const stream = Stream.PassThrough(); + const end = length ? offset + length : ""; + client + .send( + new GetObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + Range: `bytes=${offset}-${end}`, + }) + ) + .then((response) => { + response.Body.pipe(stream); + }) + .catch((error) => { + stream.emit("error", error); + }); + return stream; + }, + }; + + return directory(source, options); + }, custom: function(source, options) { return directory(source, options); } diff --git a/package.json b/package.json index 528f510..63bab94 100644 --- a/package.json +++ b/package.json @@ -32,13 +32,14 @@ "devDependencies": { "@eslint/js": "^9.2.0", "aws-sdk": "^2.1636.0", + "@aws-sdk/client-s3": "^3.0.0", "dirdiff": ">= 0.0.1 < 1", "eslint": "^9.2.0", "globals": "^15.2.0", "iconv-lite": "^0.4.24", "request": "^2.88.0", "stream-buffers": ">= 0.2.5 < 1", - "tap": "^12.7.0", + "tap": "^16.3.10", "temp": ">= 0.4.0 < 1" }, "directories": { @@ -56,6 +57,6 @@ ], "main": "unzip.js", "scripts": { - "test": "npx tap test/*.js --coverage-report=html --reporter=dot" + "test": "npx tap test/*.js --coverage-report=html --lines=90 --functions=85 --statements=90 --branches=80 --reporter=dot" } } diff --git a/test/openS3_v3.js b/test/openS3_v3.js new file mode 100644 index 0000000..378bf0d --- /dev/null +++ b/test/openS3_v3.js @@ -0,0 +1,35 @@ +const test = require("tap").test; +const unzip = require("../unzip"); + +const version = +process.version.replace("v", "").split(".")[0]; + +test( + "get content of a single file entry out of a zip", + { skip: version < 16 }, + function (t) { + const { S3Client } = require("@aws-sdk/client-s3"); + + const client = new S3Client({ + region: "us-east-1", + signer: { sign: async (request) => request }, + }); + + // These files are provided by AWS's open data registry project. + // https://github.com/awslabs/open-data-registry + + return unzip.Open.s3_v3(client, { + Bucket: "wikisum", + Key: "WikiSumDataset.zip", + }).then(function (d) { + const file = d.files.filter(function (file) { + return file.path == "WikiSumDataset/LICENSE.txt"; + })[0]; + + return file.buffer().then(function (b) { + const firstLine = b.toString().split("\n")[0]; + t.equal(firstLine, "Attribution-NonCommercial-ShareAlike 3.0 Unported"); + t.end(); + }); + }); + } +);