From e76692d2f3c1d2e04f1bda0d09eb1b899d83fdce Mon Sep 17 00:00:00 2001 From: soulgalore Date: Sun, 29 Dec 2024 22:14:37 +0100 Subject: [PATCH 1/2] Replace recursive readdir --- lib/plugins/gcs/index.js | 11 ++++------- lib/plugins/s3/index.js | 10 +++------- lib/plugins/scp/index.js | 8 ++------ lib/support/fileUtil.js | 23 +++++++++++++++++++++++ npm-shrinkwrap.json | 12 ------------ package.json | 1 - 6 files changed, 32 insertions(+), 33 deletions(-) create mode 100644 lib/support/fileUtil.js diff --git a/lib/plugins/gcs/index.js b/lib/plugins/gcs/index.js index 5ba09fc0cf..56d13befd7 100644 --- a/lib/plugins/gcs/index.js +++ b/lib/plugins/gcs/index.js @@ -2,18 +2,15 @@ import path from 'node:path'; import fs from 'node:fs'; import { SitespeedioPlugin } from '@sitespeed.io/plugin'; -import readdir from 'recursive-readdir'; + // Documentation of @google-cloud/storage: https://cloud.google.com/nodejs/docs/reference/storage/2.3.x/Bucket#upload import { Storage } from '@google-cloud/storage'; import intel from 'intel'; import { throwIfMissing } from '../../support/util.js'; +import { recursiveReaddir} from '../../support/fileUtil.js'; const log = intel.getLogger('sitespeedio.plugin.gcs'); -function ignoreDirectories(file, stats) { - return stats.isDirectory(); -} - async function uploadLatestFiles(dir, gcsOptions, prefix) { const config = { projectId: gcsOptions.projectId, @@ -23,7 +20,7 @@ async function uploadLatestFiles(dir, gcsOptions, prefix) { const storage = new Storage(config); const bucket = storage.bucket(gcsOptions.bucketname); - const files = await readdir(dir, [ignoreDirectories]); + const files = await recursiveReaddir(dir, true); const promises = []; for (let file of files) { @@ -33,7 +30,7 @@ async function uploadLatestFiles(dir, gcsOptions, prefix) { } async function upload(dir, gcsOptions, prefix) { - const files = await readdir(dir); + const files = await recursiveReaddir(dir); const promises = []; const storage = new Storage({ diff --git a/lib/plugins/s3/index.js b/lib/plugins/s3/index.js index dac7b430dd..a72357fc6d 100644 --- a/lib/plugins/s3/index.js +++ b/lib/plugins/s3/index.js @@ -1,21 +1,17 @@ import path from 'node:path'; import { promises as fsPromises } from 'node:fs'; -import readdir from 'recursive-readdir'; import pLimit from 'p-limit'; import intel from 'intel'; import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; import { SitespeedioPlugin } from '@sitespeed.io/plugin'; import { throwIfMissing } from '../../support/util.js'; +import { recursiveReaddir } from '../../support/fileUtil.js'; import { getContentType } from './contentType.js'; const log = intel.getLogger('sitespeedio.plugin.s3'); -function ignoreDirectories(file, stats) { - return stats.isDirectory(); -} - async function uploadFile(file, s3Client, s3Options, prefix, baseDir) { const stream = await fsPromises.readFile(file); const contentType = getContentType(file); @@ -89,7 +85,7 @@ export default class S3Plugin extends SitespeedioPlugin { ); try { - const files = await readdir(baseDir); + const files = await recursiveReaddir(baseDir); const limit = pLimit(s3Options.maxAsyncS3 || 20); const uploadPromises = files.map(file => limit(() => @@ -108,7 +104,7 @@ export default class S3Plugin extends SitespeedioPlugin { const rootPath = path.join(baseDir, '..'); const directoriesAsArray = rootPath.split(path.sep); const rootName = directoriesAsArray.at(-1); - const latestFiles = await readdir(rootPath, [ignoreDirectories]); + const latestFiles = await recursiveReaddir(rootPath, true); const latestUploadPromises = latestFiles.map(file => limit(() => uploadFile(file, this.s3Client, s3Options, rootName, rootPath) diff --git a/lib/plugins/scp/index.js b/lib/plugins/scp/index.js index fbb318bf49..ffb9d89483 100644 --- a/lib/plugins/scp/index.js +++ b/lib/plugins/scp/index.js @@ -3,9 +3,9 @@ import fs from 'node:fs'; import { SitespeedioPlugin } from '@sitespeed.io/plugin'; import { Client } from 'node-scp'; -import readdir from 'recursive-readdir'; import intel from 'intel'; import { throwIfMissing } from '../../support/util.js'; +import { recursiveReaddir } from '../../support/fileUtil.js'; const log = intel.getLogger('sitespeedio.plugin.scp'); @@ -75,12 +75,8 @@ async function uploadFiles(files, scpOptions, prefix) { } } -function ignoreDirectories(file, stats) { - return stats.isDirectory(); -} - async function uploadLatestFiles(dir, scpOptions, prefix) { - const files = await readdir(dir, [ignoreDirectories]); + const files = await recursiveReaddir(dir, true); return uploadFiles(files, scpOptions, prefix); } diff --git a/lib/support/fileUtil.js b/lib/support/fileUtil.js new file mode 100644 index 0000000000..383a09d324 --- /dev/null +++ b/lib/support/fileUtil.js @@ -0,0 +1,23 @@ +import { readdir } from 'node:fs/promises'; +import path from 'node:path'; + +export async function recursiveReaddir(dir, includeDirectories = false) { + const results = []; + const entries = await readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + + if (entry.isDirectory()) { + if (includeDirectories) { + results.push(fullPath); + } + const subPaths = await recursiveReaddir(fullPath, includeDirectories); + results.push(...subPaths); + } else { + results.push(fullPath); + } + } + + return results; +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index e8a974d033..9edb85e8db 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -46,7 +46,6 @@ "os-name": "6.0.0", "p-limit": "6.1.0", "pug": "3.0.3", - "recursive-readdir": "2.2.3", "simplecrawler": "1.1.9", "tape": "5.8.1", "yargs": "17.7.2" @@ -9528,17 +9527,6 @@ "node": ">=8.10.0" } }, - "node_modules/recursive-readdir": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.3.tgz", - "integrity": "sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==", - "dependencies": { - "minimatch": "^3.0.5" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/regenerator-runtime": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", diff --git a/package.json b/package.json index 87e54549a3..925c61d8d4 100644 --- a/package.json +++ b/package.json @@ -116,7 +116,6 @@ "os-name": "6.0.0", "p-limit": "6.1.0", "pug": "3.0.3", - "recursive-readdir": "2.2.3", "simplecrawler": "1.1.9", "@sitespeed.io/plugin": "0.0.6", "tape": "5.8.1", From b4067e98146b6576233c2389a8af670d6454d223 Mon Sep 17 00:00:00 2001 From: soulgalore Date: Sun, 29 Dec 2024 22:20:03 +0100 Subject: [PATCH 2/2] lint --- lib/plugins/gcs/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/plugins/gcs/index.js b/lib/plugins/gcs/index.js index 56d13befd7..d7a3de4d91 100644 --- a/lib/plugins/gcs/index.js +++ b/lib/plugins/gcs/index.js @@ -7,7 +7,7 @@ import { SitespeedioPlugin } from '@sitespeed.io/plugin'; import { Storage } from '@google-cloud/storage'; import intel from 'intel'; import { throwIfMissing } from '../../support/util.js'; -import { recursiveReaddir} from '../../support/fileUtil.js'; +import { recursiveReaddir } from '../../support/fileUtil.js'; const log = intel.getLogger('sitespeedio.plugin.gcs');