From 187da28e22d9f6448b0d2fbdbeec914e92a58114 Mon Sep 17 00:00:00 2001 From: Ayobami Akingbade Date: Mon, 20 Nov 2023 12:23:31 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(uploads):=20install=20formidab?= =?UTF-8?q?le?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 78 ++++++++++++++++++- package.json | 5 +- src/backend/actions/actions.service.ts | 2 +- .../validations/implementations/index.ts | 2 + .../implementations/raw-request.ts | 7 ++ src/backend/lib/request/validations/types.ts | 1 + src/backend/uploads/constants.ts | 21 +++++ src/backend/uploads/parse.ts | 52 +++++++++++++ .../components/Form/FormFileInput/index.tsx | 10 ++- src/frontend/lib/data/makeRequest.ts | 2 +- src/pages/api/upload/index.ts | 78 +++++++------------ 11 files changed, 197 insertions(+), 61 deletions(-) create mode 100644 src/backend/lib/request/validations/implementations/raw-request.ts create mode 100644 src/backend/uploads/constants.ts create mode 100644 src/backend/uploads/parse.ts diff --git a/package-lock.json b/package-lock.json index 9d09236e6..bf010aa4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@dashpress/bacteria": "^0.0.9", "@tanstack/react-table": "^8.7.9", "@types/cryptr": "^4.0.1", + "@types/formidable": "^3.4.5", "@types/jsonwebtoken": "^8.5.8", "@types/lodash": "^4.14.182", "@types/microseconds": "^0.2.0", @@ -32,6 +33,7 @@ "execa": "^6.1.0", "final-form": "^4.20.7", "final-form-arrays": "^3.0.2", + "formidable": "^3.5.1", "fs-extra": "^10.1.0", "immer": "9.0.3", "jsonwebtoken": "^8.5.1", @@ -8532,6 +8534,14 @@ "@types/range-parser": "*" } }, + "node_modules/@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/fs-extra": { "version": "9.0.13", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", @@ -10282,8 +10292,7 @@ "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", - "dev": true + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "node_modules/asn1.js": { "version": "5.4.1", @@ -14702,6 +14711,15 @@ "detect-port": "bin/detect-port.js" } }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -17520,6 +17538,19 @@ "node": ">= 14.17" } }, + "node_modules/formidable": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz", + "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==", + "dependencies": { + "dezalgo": "^1.0.4", + "hexoid": "^1.0.0", + "once": "^1.4.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -18639,6 +18670,14 @@ "dev": true, "license": "MIT" }, + "node_modules/hexoid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", + "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", + "engines": { + "node": ">=8" + } + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -39157,6 +39196,14 @@ "@types/range-parser": "*" } }, + "@types/formidable": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz", + "integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==", + "requires": { + "@types/node": "*" + } + }, "@types/fs-extra": { "version": "9.0.13", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", @@ -40540,8 +40587,7 @@ "asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", - "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", - "dev": true + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "asn1.js": { "version": "5.4.1", @@ -43872,6 +43918,15 @@ "debug": "4" } }, + "dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "requires": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -46004,6 +46059,16 @@ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==" }, + "formidable": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz", + "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==", + "requires": { + "dezalgo": "^1.0.4", + "hexoid": "^1.0.0", + "once": "^1.4.0" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -46849,6 +46914,11 @@ "integrity": "sha512-lOhQU7iG3AMcjmb8NIWCa+KwfJw5bY44BoWPtrj5A4iDbSD3ylGf5QcYr0ZyQnhkKQ2GgWNLdF2rfrXtXlF3nQ==", "dev": true }, + "hexoid": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", + "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", diff --git a/package.json b/package.json index ed71e9dfe..88fbfc8f9 100644 --- a/package.json +++ b/package.json @@ -35,10 +35,10 @@ "@dashpress/bacteria": "^0.0.9", "@tanstack/react-table": "^8.7.9", "@types/cryptr": "^4.0.1", + "@types/formidable": "^3.4.5", "@types/jsonwebtoken": "^8.5.8", "@types/lodash": "^4.14.182", "@types/microseconds": "^0.2.0", - "@types/multer": "^1.4.7", "@types/nodemailer": "^6.4.6", "@types/prismjs": "^1.26.0", "@types/qs": "^6.9.7", @@ -55,6 +55,7 @@ "execa": "^6.1.0", "final-form": "^4.20.7", "final-form-arrays": "^3.0.2", + "formidable": "^3.5.1", "fs-extra": "^10.1.0", "immer": "9.0.3", "jsonwebtoken": "^8.5.1", @@ -62,11 +63,9 @@ "latest-version": "^7.0.0", "lodash": "^4.17.21", "microseconds": "^0.2.0", - "multer": "^1.4.5-lts.1", "mustache": "^4.2.0", "nanoid": "^4.0.0", "next": "12.3.1", - "next-connect": "^0.13.0", "nodemailer": "^6.8.0", "path": "^0.12.7", "polished": "^4.2.2", diff --git a/src/backend/actions/actions.service.ts b/src/backend/actions/actions.service.ts index 04a156994..13a1a635e 100644 --- a/src/backend/actions/actions.service.ts +++ b/src/backend/actions/actions.service.ts @@ -41,7 +41,7 @@ export class ActionsApiService implements IApplicationService { await this._actionInstancesPersistenceService.setup(); } - // TODO: job queue + // TODO: job queue https://github.com/bee-queue/bee-queue async runAction( entity: string, formAction: string, diff --git a/src/backend/lib/request/validations/implementations/index.ts b/src/backend/lib/request/validations/implementations/index.ts index de9b511a7..7c2e3ff77 100644 --- a/src/backend/lib/request/validations/implementations/index.ts +++ b/src/backend/lib/request/validations/implementations/index.ts @@ -16,6 +16,7 @@ import { canUserValidationImpl as canUser } from "./can-user"; import { withPasswordValidationImpl as withPassword } from "./with-password"; import { authenticatedUserValidationImpl as authenticatedUser } from "./authenticated-user"; import { requestQueriesValidationImpl as requestQueries } from "./request-queries"; +import { rawRequestValidationImpl as rawRequest } from "./raw-request"; import { ValidationImplType } from "./types"; import { PortalValidationImpl } from "./portal"; @@ -32,6 +33,7 @@ export const ValidationImpl: Record< anyBody, requestQuery, canUser, + rawRequest, requestQueries, authenticatedUser, configBody, diff --git a/src/backend/lib/request/validations/implementations/raw-request.ts b/src/backend/lib/request/validations/implementations/raw-request.ts new file mode 100644 index 000000000..ccb3965e7 --- /dev/null +++ b/src/backend/lib/request/validations/implementations/raw-request.ts @@ -0,0 +1,7 @@ +import { ValidationImplType } from "./types"; + +export const rawRequestValidationImpl: ValidationImplType< + Record +> = async (req) => { + return req as unknown as Record; +}; diff --git a/src/backend/lib/request/validations/types.ts b/src/backend/lib/request/validations/types.ts index 91e74c1f9..db7b04f00 100644 --- a/src/backend/lib/request/validations/types.ts +++ b/src/backend/lib/request/validations/types.ts @@ -9,6 +9,7 @@ export type ValidationKeys = { | "entity" | "authenticatedUser" | "configKey" + | "rawRequest" | "paginationFilter" | "canUser" | "crudEnabled" diff --git a/src/backend/uploads/constants.ts b/src/backend/uploads/constants.ts new file mode 100644 index 000000000..01066eff7 --- /dev/null +++ b/src/backend/uploads/constants.ts @@ -0,0 +1,21 @@ +export const FORMIDABLE_ERRORS = { + aborted: 1002, + biggerThanMaxFileSize: 1016, + biggerThanTotalMaxFileSize: 1009, + cannotCreateDir: 1018, + filenameNotString: 1005, + malformedMultipart: 1012, + maxFieldsExceeded: 1007, + maxFieldsSizeExceeded: 1006, + maxFilesExceeded: 1015, + missingContentType: 1011, + missingMultipartBoundary: 1013, + missingPlugin: 1000, + noEmptyFiles: 1010, + noParser: 1003, + pluginFailed: 1017, + pluginFunction: 1001, + smallerThanMinFileSize: 1008, + uninitializedParser: 1004, + unknownTransferEncoding: 1014, +}; diff --git a/src/backend/uploads/parse.ts b/src/backend/uploads/parse.ts new file mode 100644 index 000000000..287876de1 --- /dev/null +++ b/src/backend/uploads/parse.ts @@ -0,0 +1,52 @@ +import formidable from "formidable"; +import { mkdir, stat } from "fs/promises"; +import { join } from "path"; +import { sluggify } from "shared/lib/strings"; +import { format } from "date-fns"; +import { NextApiRequest } from "next"; +import { nanoid } from "nanoid"; + +export async function parseForm( + req: NextApiRequest +): Promise<{ fields: formidable.Fields; files: formidable.Files }> { + const UPLOAD_CONFIG = { + entity: sluggify("posts"), + maxFileSize: 1024 * 1024 * 5, + fileType: "image", + rootDir: process.cwd(), + }; + + const uploadDir = join( + UPLOAD_CONFIG.rootDir || process.cwd(), + `/uploads/${UPLOAD_CONFIG.entity}/${format(Date.now(), "dd-MM-Y")}` + ); + + try { + await stat(uploadDir); + } catch (e: any) { + if (e.code === "ENOENT") { + await mkdir(uploadDir, { recursive: true }); + } else { + throw e; + } + } + + const form = formidable({ + maxFiles: 1, + maxFileSize: UPLOAD_CONFIG.maxFileSize, + uploadDir, + filename: (_name, _ext, part) => { + return `${nanoid()}.${part.originalFilename.split(".").pop()}`; + }, + filter: (part) => { + return ( + part.name === "file" && + (part.mimetype?.includes(UPLOAD_CONFIG.fileType) || false) + ); + }, + }); + + const [fields, files] = await form.parse(req); + + return { fields, files }; +} diff --git a/src/frontend/design-system/components/Form/FormFileInput/index.tsx b/src/frontend/design-system/components/Form/FormFileInput/index.tsx index 37e05af8e..b20a35e31 100644 --- a/src/frontend/design-system/components/Form/FormFileInput/index.tsx +++ b/src/frontend/design-system/components/Form/FormFileInput/index.tsx @@ -1,6 +1,7 @@ import React, { useCallback, useState } from "react"; import { useDropzone } from "react-dropzone"; import axios, { AxiosProgressEvent } from "axios"; +import { getRequestHeaders } from "frontend/lib/data/makeRequest"; import { ISharedFormInput } from "../_types"; import { generateClassNames, wrapLabelAndError } from "../_wrapForm"; import { Presentation } from "./Presentation"; @@ -9,7 +10,6 @@ interface IFormFileInput extends ISharedFormInput { uploadUrl: string; metadata?: Record; maxSize?: number; - requestHeaders?: Record; } function FileInput({ @@ -19,7 +19,6 @@ function FileInput({ uploadUrl, metadata, maxSize, - requestHeaders, }: IFormFileInput) { const [progress, setProgress] = useState(0); const [error, setError] = useState(""); @@ -43,7 +42,7 @@ function FileInput({ const { fileUrl } = ( await axios.post(uploadUrl, formData, { headers: { - ...requestHeaders, + ...getRequestHeaders(), "Content-Type": "multipart/form-data", }, onUploadProgress: (progressEvent: AxiosProgressEvent) => { @@ -55,8 +54,11 @@ function FileInput({ }) ).data; input.onChange(fileUrl); + setError(null); } catch (e) { - setError("Ooops, something wrong happened."); + setError( + e.response.data.message || "Ooops, something wrong happened." + ); } setProgress(0); }); diff --git a/src/frontend/lib/data/makeRequest.ts b/src/frontend/lib/data/makeRequest.ts index ebaf3fba8..0d9f14479 100644 --- a/src/frontend/lib/data/makeRequest.ts +++ b/src/frontend/lib/data/makeRequest.ts @@ -11,7 +11,7 @@ const pathWithBaseUrl = (path: string) => { return (process.env.NEXT_PUBLIC_BASE_URL || "") + path; }; -const getRequestHeaders = () => { +export const getRequestHeaders = () => { const authToken = getAuthToken(); const headers: Record = { "Content-Type": "application/json", diff --git a/src/pages/api/upload/index.ts b/src/pages/api/upload/index.ts index 82c02407c..cc59abcbf 100644 --- a/src/pages/api/upload/index.ts +++ b/src/pages/api/upload/index.ts @@ -1,59 +1,41 @@ -import { NextApiRequest, NextApiResponse, PageConfig } from "next"; -import nc from "next-connect"; +import { PageConfig } from "next"; +import { requestHandler } from "backend/lib/request"; +import { BadRequestError } from "backend/lib/errors"; +import { FORMIDABLE_ERRORS } from "backend/uploads/constants"; +import { parseForm } from "backend/uploads/parse"; -const multer = require("multer"); +export default requestHandler({ + POST: async (getValidatedRequest): Promise> => { + try { + if (process.env.NEXT_PUBLIC_IS_DEMO) { + throw new Error("File uploads will not work on demo site"); + } -const upload = multer({ - dest: "uploads/", - filename: (_, file, callback) => { - // originalname is the uploaded file's name with extn - callback(null, file.originalname); - }, -}); + const { rawRequest: req } = await getValidatedRequest(["rawRequest"]); -const handler = nc({ - onError: (_1, _2, res) => { - res.status(500).end("Something broke!"); - }, - onNoMatch: (_, res) => { - res.status(404).end("Page is not found"); - }, -}) - .use((req, res, next) => { - res.setHeader("Access-Control-Allow-Origin", "*"); - res.setHeader( - "Access-Control-Allow-Headers", - "Origin, X-Requested-With, Content-Type, Accept, Authorization" - ); - if (req.method === "OPTIONS") { - res.setHeader( - "Access-Control-Allow-Methods", - "PUT, POST, PATCH, DELETE, GET" - ); - return res.status(200).json({}); - } + const { files } = await parseForm(req); + + if (files.file.length === 0) { + throw new BadRequestError("An invalid file was submitted"); + } - next(); - }) - .use(upload.single("file")) - .options((_, res) => { - res.json({}); - }) - .post((req, res) => { - // Move the file - res.json({ hello: "world", fileUrl: (req as any).file.path }); - }); + const fileUrl = files.file[0].filepath; -export default handler; + return { + fileUrl, + }; + } catch (error) { + if ([FORMIDABLE_ERRORS.biggerThanTotalMaxFileSize].includes(error.code)) { + throw new BadRequestError(error.message); + } + + throw error; + } + }, +}); export const config: PageConfig = { api: { bodyParser: false, }, }; - -// TODO: -// disable file upload in DEMO -// Max Size -// file type -// auth validation