Replace recursive readdir (#4377)

* Replace recursive readdir

* lint

lib/plugins/gcs/index.js

@@ -2,18 +2,15 @@ import path from 'node:path';
 import fs from 'node:fs';
 
 import { SitespeedioPlugin } from '@sitespeed.io/plugin';
-import readdir from 'recursive-readdir';
+
 // Documentation of @google-cloud/storage: https://cloud.google.com/nodejs/docs/reference/storage/2.3.x/Bucket#upload
 import { Storage } from '@google-cloud/storage';
 import intel from 'intel';
 import { throwIfMissing } from '../../support/util.js';
+import { recursiveReaddir } from '../../support/fileUtil.js';
 
 const log = intel.getLogger('sitespeedio.plugin.gcs');
 
-function ignoreDirectories(file, stats) {
-  return stats.isDirectory();
-}
-
 async function uploadLatestFiles(dir, gcsOptions, prefix) {
   const config = {
     projectId: gcsOptions.projectId,
@@ -23,7 +20,7 @@ async function uploadLatestFiles(dir, gcsOptions, prefix) {
   const storage = new Storage(config);
   const bucket = storage.bucket(gcsOptions.bucketname);
 
-  const files = await readdir(dir, [ignoreDirectories]);
+  const files = await recursiveReaddir(dir, true);
   const promises = [];
 
   for (let file of files) {
@@ -33,7 +30,7 @@ async function uploadLatestFiles(dir, gcsOptions, prefix) {
 }
 
 async function upload(dir, gcsOptions, prefix) {
-  const files = await readdir(dir);
+  const files = await recursiveReaddir(dir);
   const promises = [];
 
   const storage = new Storage({

lib/plugins/s3/index.js

@@ -1,21 +1,17 @@
 import path from 'node:path';
 import { promises as fsPromises } from 'node:fs';
 
-import readdir from 'recursive-readdir';
 import pLimit from 'p-limit';
 import intel from 'intel';
 import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
 
 import { SitespeedioPlugin } from '@sitespeed.io/plugin';
 import { throwIfMissing } from '../../support/util.js';
+import { recursiveReaddir } from '../../support/fileUtil.js';
 import { getContentType } from './contentType.js';
 
 const log = intel.getLogger('sitespeedio.plugin.s3');
 
-function ignoreDirectories(file, stats) {
-  return stats.isDirectory();
-}
-
 async function uploadFile(file, s3Client, s3Options, prefix, baseDir) {
   const stream = await fsPromises.readFile(file);
   const contentType = getContentType(file);
@@ -89,7 +85,7 @@ export default class S3Plugin extends SitespeedioPlugin {
       );
 
       try {
-        const files = await readdir(baseDir);
+        const files = await recursiveReaddir(baseDir);
         const limit = pLimit(s3Options.maxAsyncS3 || 20);
         const uploadPromises = files.map(file =>
           limit(() =>
@@ -108,7 +104,7 @@ export default class S3Plugin extends SitespeedioPlugin {
           const rootPath = path.join(baseDir, '..');
           const directoriesAsArray = rootPath.split(path.sep);
           const rootName = directoriesAsArray.at(-1);
-          const latestFiles = await readdir(rootPath, [ignoreDirectories]);
+          const latestFiles = await recursiveReaddir(rootPath, true);
           const latestUploadPromises = latestFiles.map(file =>
             limit(() =>
               uploadFile(file, this.s3Client, s3Options, rootName, rootPath)

lib/plugins/scp/index.js

@@ -3,9 +3,9 @@ import fs from 'node:fs';
 
 import { SitespeedioPlugin } from '@sitespeed.io/plugin';
 import { Client } from 'node-scp';
-import readdir from 'recursive-readdir';
 import intel from 'intel';
 import { throwIfMissing } from '../../support/util.js';
+import { recursiveReaddir } from '../../support/fileUtil.js';
 
 const log = intel.getLogger('sitespeedio.plugin.scp');
 
@@ -75,12 +75,8 @@ async function uploadFiles(files, scpOptions, prefix) {
   }
 }
 
-function ignoreDirectories(file, stats) {
-  return stats.isDirectory();
-}
-
 async function uploadLatestFiles(dir, scpOptions, prefix) {
-  const files = await readdir(dir, [ignoreDirectories]);
+  const files = await recursiveReaddir(dir, true);
 
   return uploadFiles(files, scpOptions, prefix);
 }

lib/support/fileUtil.js

@@ -0,0 +1,23 @@
+import { readdir } from 'node:fs/promises';
+import path from 'node:path';
+
+export async function recursiveReaddir(dir, includeDirectories = false) {
+  const results = [];
+  const entries = await readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      if (includeDirectories) {
+        results.push(fullPath);
+      }
+      const subPaths = await recursiveReaddir(fullPath, includeDirectories);
+      results.push(...subPaths);
+    } else {
+      results.push(fullPath);
+    }
+  }
+
+  return results;
+}

package.json

@@ -116,7 +116,6 @@
     "os-name": "6.0.0",
     "p-limit": "6.1.0",
     "pug": "3.0.3",
-    "recursive-readdir": "2.2.3",
     "simplecrawler": "1.1.9",
     "@sitespeed.io/plugin": "0.0.6",
     "tape": "5.8.1",