fix: resolve the issue with invalid zip data

When the zip file can not be unzipped we attempt to use `tar` as
fallback mechanism. After testing this for a while, can confirm the
fallback is good enough but it cannot replace the zip handler altogether
like attempted before in:

- #1487
- #1495

Instead this approach will only use it if we receive the invalid zip
exception from fflate.

package.json

@@ -26,7 +26,8 @@
     "lint": "eslint '**/*.ts' '**/*.tsx'",
     "lint:fix": "eslint --fix '**/*.ts' '**/*.tsx'",
     "purge-js": "rm `find . -name '*.js'|grep -v node_modules`",
-    "test:mock": "FORCE_BUILD=true ts-node lib/notion/_mock/build-mock-data.tsx"
+    "test:mock": "FORCE_BUILD=true ts-node lib/notion/_mock/build-mock-data.tsx",
+    "dev-cleanup": "git clean -fd && find . -name '*.js.map' -type f -delete"
   },
   "license": "MIT",
   "dependencies": {
@@ -97,4 +98,4 @@
     "ts-node": "^10.9.1",
     "typescript": "^5.2.2"
   }
-}
+}

src/lib/parser/DeckParser.ts

@@ -2,7 +2,7 @@ import cheerio from 'cheerio';
 
 import preserveNewlinesIfApplicable from '../../services/NotionService/helpers/preserveNewlinesIfApplicable';
 import sanitizeTags from '../anki/sanitizeTags';
-import { File } from '../anki/zip';
+import { File } from '../zip/zip';
 import Deck from './Deck';
 import Note from './Note';
 import Settings from './Settings';

src/lib/parser/experimental/FallbackParser.ts

@@ -1,6 +1,6 @@
 import cheerio from 'cheerio';
 
-import { File } from '../../anki/zip';
+import { File } from '../../zip/zip';
 import {
   isHTMLFile,
   isMarkdownFile,

src/lib/parser/exporters/embedFile.ts

@@ -1,4 +1,4 @@
-import { File } from '../../anki/zip';
+import { File } from '../../zip/zip';
 import { SuffixFrom } from '../../misc/file';
 import getUniqueFileName from '../../misc/getUniqueFileName';
 import CustomExporter from './CustomExporter';

src/lib/parser/getFileContents.ts

@@ -1,6 +1,6 @@
 import { isHTMLFile, isMarkdownFile } from '../storage/checks';
 import { markdownToHTML } from '../markdown';
-import { File } from '../anki/zip';
+import { File } from '../zip/zip';
 
 export function getFileContents(file: File | undefined, convertToHTML = true) {
   const contents = file?.contents;

src/lib/storage/checks.ts

@@ -21,4 +21,4 @@ export const isImageFileEmbedable = (url: string) =>
 
 export const isCSVFile = (fileName: string) => /.csv$/i.exec(fileName);
 
-export const isPDFFile = (fileName: string) => /.pdf$/i.exec(fileName);
+export const isPDFFile = (fileName: string) => /.pdf$/i.exec(fileName);

src/lib/storage/types.ts

@@ -1,4 +1,4 @@
-import { File } from '../anki/zip';
+import { File } from '../zip/zip';
 
 // Multer types
 export interface UploadedFile extends Express.Multer.File {

src/lib/zip/fallback/decompress.ts

@@ -0,0 +1,8 @@
+import { unpack } from './unpack';
+import { File } from './types';
+import { writeFile } from './writeFile';
+
+export function decompress(byteArray: Uint8Array): Promise<File[]> {
+  const { workspace, filePath } = writeFile(byteArray);
+  return unpack(filePath, workspace.location);
+}

src/lib/zip/fallback/listFiles.ts

@@ -0,0 +1,28 @@
+import fs from 'fs';
+
+import { File } from './types';
+export async function listFiles(workspace: string) {
+  const files: File[] = [];
+
+  async function explorePath(currentPath: string) {
+    const dir = await fs.promises.readdir(currentPath);
+    for (const fileName of dir) {
+      const filePath = `${currentPath}/${fileName}`;
+      const stats = await fs.promises.stat(filePath);
+
+      if (stats.isFile()) {
+        const buffer = await fs.promises.readFile(filePath);
+        files.push({
+          name: filePath,
+          contents: new Uint8Array(buffer),
+        });
+      } else if (stats.isDirectory()) {
+        await explorePath(filePath); // Recursively explore subdirectories
+      }
+    }
+  }
+
+  await explorePath(workspace);
+  console.log('files', files);
+  return files;
+}

src/lib/zip/fallback/processAndPrepareArchiveData.tsx

@@ -0,0 +1,42 @@
+import { renderToStaticMarkup } from 'react-dom/server';
+import { getUploadLimits } from '../../misc/getUploadLimits';
+import { decompress } from './decompress';
+import { isZipContentFileSupported } from '../../../usecases/uploads/isZipContentFileSupported';
+
+export const processAndPrepareArchiveData = async (
+  byteArray: Uint8Array,
+  isPatron: boolean = false
+) => {
+  const size = Buffer.byteLength(byteArray);
+  const limits = getUploadLimits(isPatron);
+
+  if (size > limits.fileSize) {
+    throw new Error(
+      renderToStaticMarkup(
+        <>
+          Your upload is too big, there is a max of {size} / ${limits.fileSize}{' '}
+          currently. <a href="https://alemayhu.com/patreon">Become a patron</a>{' '}
+          to remove default limit or{' '}
+          <a href="https://2anki.net/login#login">login</a>.
+        </>
+      )
+    );
+  }
+
+  const decompressedData = await decompress(byteArray);
+  const fileNames = decompressedData.map((z) => z.name);
+  const files = [];
+
+  for (const name of fileNames) {
+    const file = decompressedData.find((z) => z.name === name);
+    let contents = file?.contents;
+    if (isZipContentFileSupported(name) && contents) {
+      const s = new TextDecoder().decode(contents as Uint8Array);
+      files.push({ name, contents: s });
+    } else if (contents) {
+      files.push({ name, contents });
+    }
+  }
+
+  return files;
+};

src/lib/zip/fallback/types.ts

@@ -0,0 +1,4 @@
+export interface File {
+        name: string;
+        contents?: string | Uint8Array;
+      }

src/lib/zip/fallback/unpack.ts

@@ -0,0 +1,24 @@
+import { spawn } from 'node:child_process';
+
+import { listFiles } from './listFiles';
+import { File } from './types';
+
+const TAR_PATH = '/usr/bin/bsdtar';
+
+export function unpack(filePath: string, workspace: string): Promise<File[]> {
+  return new Promise((resolve, reject) => {
+    const decompressProcess = spawn(TAR_PATH, ['xvf', filePath], {
+      cwd: workspace,
+    });
+    decompressProcess.stdout.on('data', (data) => {
+      console.log(`tar output: ${data}`);
+    });
+    decompressProcess.stderr.on('data', (data) => {
+      console.error(`tar error: ${data}`);
+    });
+    decompressProcess.on('close', () => {
+      // We are not reading the status code because we support partial extraction
+      listFiles(workspace).then(resolve).catch(reject);
+    });
+  });
+}

src/lib/zip/fallback/writeFile.ts

@@ -0,0 +1,12 @@
+import { getRandomUUID } from '../../../shared/helpers/getRandomUUID';
+import Workspace from '../../parser/WorkSpace';
+import path from 'path';
+import fs from 'fs';
+
+export function writeFile(compressedData: Uint8Array) {
+  const uuid = getRandomUUID();
+  const workspace = new Workspace(true, 'fs');
+  const p = path.join(workspace.location, uuid);
+  fs.writeFileSync(p, compressedData);
+  return { workspace, filePath: p };
+}

src/lib/zip/zip.tsx

@@ -0,0 +1,90 @@
+import { strFromU8, unzipSync } from 'fflate';
+import { Body } from 'aws-sdk/clients/s3';
+import { renderToStaticMarkup } from 'react-dom/server';
+import { getUploadLimits } from '../misc/getUploadLimits';
+import { isHTMLFile, isMarkdownFile, isPDFFile } from '../storage/checks';
+import { processAndPrepareArchiveData } from './fallback/processAndPrepareArchiveData';
+
+interface File {
+  name: string;
+  contents?: Body | string;
+}
+
+class ZipHandler {
+  files: File[];
+  zipFileCount: number;
+  maxZipFiles: number;
+
+  constructor(maxNestedZipFiles: number) {
+    this.files = [];
+    this.zipFileCount = 0;
+    this.maxZipFiles = maxNestedZipFiles;
+  }
+
+  async build(zipData: Uint8Array, paying: boolean) {
+    const size = Buffer.byteLength(zipData);
+    const limits = getUploadLimits(paying);
+
+    if (size > limits.fileSize) {
+      throw new Error(
+        renderToStaticMarkup(
+          <>
+            Your upload is too big, there is a max of {size} / $
+            {limits.fileSize} currently.{' '}
+            <a href="https://alemayhu.com/patreon">Become a patron</a> to remove
+            default limit.
+          </>
+        )
+      );
+    }
+
+    await this.processZip(zipData, paying);
+  }
+
+  private async processZip(zipData: Uint8Array, paying: boolean) {
+    if (this.zipFileCount >= this.maxZipFiles) {
+      throw new Error('Too many zip files in the upload.');
+    }
+
+    try {
+      const loadedZip = unzipSync(zipData, {
+        filter(file) {
+          return !file.name.endsWith('/');
+        },
+      });
+
+      for (const name in loadedZip) {
+        const file = loadedZip[name];
+        let contents = file;
+
+        if (name.includes('__MACOSX/') || isPDFFile(name)) {
+          continue;
+        }
+
+        if (name.endsWith('.zip')) {
+          this.zipFileCount++;
+          await this.processZip(file, paying);
+        } else if ((isHTMLFile(name) || isMarkdownFile(name)) && contents) {
+          this.files.push({ name, contents: strFromU8(file) });
+        } else if (contents) {
+          this.files.push({ name, contents });
+        }
+      }
+
+    } catch (error: unknown) {
+      if ((error as { code?: number }).code === 13) {
+        const foundFiles = await processAndPrepareArchiveData(zipData, paying);
+        this.files.push(...foundFiles);
+        console.log('files', this.files);
+      } else {
+      throw error;
+      }
+    }
+  }
+
+  getFileNames() {
+    return this.files.map((file) => file.name);
+  }
+}
+
+export { ZipHandler, File };