Zipping file input (multiple files) and sending as formData

[ryanwinchester]

First of all this library looks pretty badass, and I can't believe it took me this long to discover it (found from a comment in JSZip issues).

Explanation

I want to get file input, zip it, and send it to AWS S3 using a pre-signed URL. My JS is weak, and I've been researching how to do this for a couple of days now and was going to attempt to do it with JSZip, as I've seen some comments saying you should be able to do it something like:

zip.generateAsync({type: 'blob'})
   .then((content) => formData.append("file", content))

And I think that to submit it with xhr / formData it needs to be blob.

I'm using some Phoenix Live View example code, from their docs:

let Uploaders = {}

Uploaders.S3 = function(entries, onViewError){
  entries.forEach(entry => {
    let formData = new FormData()
    let {url, fields} = entry.meta

    Object.entries(fields).forEach(([key, val]) => formData.append(key, val))

    formData.append("file", entry.file)

    let xhr = new XMLHttpRequest()

    onViewError(() => xhr.abort())
    xhr.onload = () => xhr.status === 204 || entry.error()
    xhr.onerror = () => entry.error()

    xhr.upload.addEventListener("progress", (event) => {
      if(event.lengthComputable){
        let percent = Math.round((event.loaded / event.total) * 100)
        entry.progress(percent)
      }
    })

    xhr.open("POST", url, true)
    xhr.send(formData)
  })
}

let liveSocket = new LiveSocket("/live", Socket, {
  uploaders: Uploaders,
  params: {_csrf_token: csrfToken}
})

The Question

How can I modify the above JS code to use fflate to zip all the files from the form input to post to the S3 pre-signed URL?

Any help whatsoever would be greatly appreciated.

[101arrowz]

Looking at the code you provided, each individual file has its own pre-signed URL and fields that need to go with it. If you have a single URL to upload to, it'll be one of the following (depending on how much you care about compatibility with old browsers):

// If you use a transpiler like Babel, this works in even IE11
const zipOut = [];
const zipFile = new fflate.Zip();
zipFile.ondata = (err, dat, final) => {
  if (err) throw err;
  zipOut.push(dat);
  if (final) {
    const blob = new Blob(zipOut, { type: 'application/zip' });
    // Do whatever you need to upload here
  }
}

const ALREADY_COMPRESSED = [
  'zip', 'gz', 'png', 'jpg', 'jpeg', 'pdf', 'doc', 'docx',
  'ppt', 'pptx', 'xls', 'xlsx', 'heic', 'heif', '7z', 'bz2',
  'rar', 'gif', 'webp', 'webm', 'mp4', 'mov', 'mp3', 'aifc'
   // Add whatever extensions you don't want to compress
];

for (const entry of entries) {
  const { file } = entry;
  // If file is a File object, use this:
  const ext = file.name.slice(file.name.lastIndexOf('.') + 1);
  const defl = ALREADY_COMPRESSED.indexOf(ext) == -1
    ? file.size < 512000
      ? new fflate.AsyncZipDeflate(file.name, { level: 1 })
      : new fflate.ZipDeflate(file.name, { level: 1 })
    : new fflate.ZipPassThrough(file.name);
  zip.add(defl);
  let ind = 262144;
  const fr = new FileReader();
  fr.onload = () => {
    const dat = new Uint8Array(fr.result);
    if (ind >= file.size) {
      defl.push(dat, true);
    } else {
      defl.push(dat);
      fr.readAsArrayBuffer(file.slice(ind, ind += 262144));
    }
  }
  fr.readAsArrayBuffer(file.slice(0, ind));
}
zip.end();

Alternatively, if you're supporting modern browsers only, follow the guide in the wiki.

This code is a bit unwieldy because it maximizes performance. This may be more easy to understand, if a bit slower:

const ALREADY_COMPRESSED = [
  'zip', 'gz', 'png', 'jpg', 'jpeg', 'pdf', 'doc', 'docx',
  'ppt', 'pptx', 'xls', 'xlsx', 'heic', 'heif', '7z', 'bz2',
  'rar', 'gif', 'webp', 'webm', 'mp4', 'mov', 'mp3', 'aifc'
   // Add whatever extensions you don't want to compress
];

const readFile = file => {
  return new Promise((resolve, reject) => {
    const fr = new FileReader();
    fr.onload = () => {
      const level = ALREADY_COMPRESSED.indexOf(file) == -1
        ? 1
        : 0;
      resolve([
        file.name,
        [new Uint8Array(fr.result), { level }]
      ]);
    }
    fr.onerror = reject;
    fr.readAsArrayBuffer(file);
  });
}

const upload = async entries => {
  const files = await Promise.all(entries.map(entry => readFile(entry.file)));
  const toZip = files.reduce((acc, [fn, dat]) => (acc[fn] = dat, acc), {});
  const zipData = await new Promise((resolve, reject) => {
    fflate.zip(toZip, (err, dat) => err ? reject(err) : resolve(dat));
  });
  const blob = new Blob([zipData], { type: 'application/zip' });
  // upload logic here
}

Alternatively, if you just want to compress your uploads but don't necessarily care how, you should use GZIP. However, note that the files will need to be decompressed on download now. You can set Content-Encoding to gzip if you want auto-decompression, but then you can't upload the FormData and must only upload the Blob data (i.e. xhr.send(blob), not xhr.send(formData)).

const ALREADY_COMPRESSED = [
  'zip', 'gz', 'png', 'jpg', 'jpeg', 'pdf', 'doc', 'docx',
  'ppt', 'pptx', 'xls', 'xlsx', 'heic', 'heif', '7z', 'bz2',
  'rar', 'gif', 'webp', 'webm', 'mp4', 'mov', 'mp3', 'aifc'
   // Add whatever extensions you don't want to compress
];

const Uploaders = {};

Uploaders.S3 = (entries, onViewError) => {
  entries.forEach(entry => {
    const { file, meta: {url, fields} } = entry;
    const formData = new FormData(fields);
    const ext = file.name.slice(file.name.lastIndexOf('.') + 1);
    let ind = 262144;
    const gz = new fflate.AsyncGzip({
      level: ALREADY_COMPRESSED.indexOf(ext) == -1
        ? 1
        : 0
    });
    const fr = new FileReader();
    fr.onload = () => {
      const dat = new Uint8Array(fr.result);
      if (ind >= file.size) {
        gz.push(dat, true);
      } else {
        gz.push(dat);
        fr.readAsArrayBuffer(file.slice(ind, ind += 262144));
      }
    }
    const chunks = [];
    gz.ondata = (err, dat, final) => {
      if (err) throw err;
      chunks.push(dat);
      if (final) {
        const blob = new Blob([chunks]);
        formData.append("file", blob);
        let xhr = new XMLHttpRequest();
        onViewError(() => xhr.abort());
        xhr.onload = () => xhr.status === 204 || entry.error();
        xhr.onerror = () => entry.error();

        xhr.upload.addEventListener("progress", event => {
          if (event.lengthComputable) {
            let percent = Math.round((event.loaded / event.total) * 100);
            entry.progress(percent)
          }
        });

        xhr.open("POST", url, true);
        xhr.send(formData);
      }
    }
    fr.readAsArrayBuffer(file.slice(0, ind));
  });
}

const liveSocket = new LiveSocket("/live", Socket, {
  uploaders: Uploaders,
  params: {_csrf_token: csrfToken}
});

[ryanwinchester]

Nice! Thank you! First hurdle is done. The code didn't throw any errors...

I got it to upload a "zip" file to S3, but if I try to download it and open it it says it's an unsopported format.

EDIT: Update there were CORS issues and one bad zip, but now it seems to be working 🙌

const Uploaders = {}

Uploaders.S3 = function(entries, onViewError) {
  console.log({entries});

  const zipOut = [];
  const zipFile = new fflate.Zip();
  zipFile.ondata = (err, dat, final) => {
    if (err) throw err;
    zipOut.push(dat);
    if (final) {
      const entry = entries[entries.length-1];
      const blob = new Blob(zipOut, { type: 'application/zip' });
      // Do whatever you need to upload here
      const formData = new FormData();
      const {url, fields} = entry.meta;

      Object.entries(fields).forEach(([key, val]) => formData.append(key, val));

      formData.append("file", blob);

      const xhr = new XMLHttpRequest();

      onViewError(() => xhr.abort());
      xhr.onload = () => xhr.status === 204 || entry.error();
      xhr.onerror = () => entry.error();

      xhr.upload.addEventListener("progress", (event) => {
        if(event.lengthComputable) {
          const percent = Math.round((event.loaded / event.total) * 100);
          entry.progress(percent);
        }
      })

      xhr.open("POST", url, true);
      xhr.send(formData);
    }
  }

  const ALREADY_COMPRESSED = [
    'zip', 'gz', 'png', 'jpg', 'jpeg', 'pdf', 'doc', 'docx',
    'ppt', 'pptx', 'xls', 'xlsx', 'heic', 'heif', '7z', 'bz2',
    'rar', 'gif', 'webp', 'webm', 'mp4', 'mov', 'mp3', 'aifc'
    // Add whatever extensions you don't want to compress
  ];

  const addChunks = (file, ind) => {

  }

  for (const entry of entries) {
    const { file } = entry;
    const ext = file.name.slice(file.name.lastIndexOf('.') + 1);
    const defl = ALREADY_COMPRESSED.indexOf(ext) == -1
      ? file.size < 512000
        ? new fflate.AsyncZipDeflate(file.name, { level: 1 })
        : new fflate.ZipDeflate(file.name, { level: 1 })
      : new fflate.ZipPassThrough(file.name);
    zipFile.add(defl);
    let ind = 262144;
    const fr = new FileReader();
    fr.onload = () => {
      const dat = new Uint8Array(fr.result);
      if (ind >= file.size) {
        defl.push(dat, true);
      } else {
        defl.push(dat);
        fr.readAsArrayBuffer(file.slice(ind, ind += 262144));
      }
    }
    fr.readAsArrayBuffer(file.slice(0, ind));
  }
  zipFile.end();
}

[ryanwinchester]

I'm not sure what this part was intended for:

const addChunks = (file, ind) => {
    // empty?
}

[ryanwinchester]

After solving some annoying S3 CORS issues, I managed to get it to work!

Thank you for the library and your help. 🙏

[101arrowz]

The addChunks was a typo, but I'm glad the rest of the code works. Hope it works well for you!