diff --git a/.gitignore b/.gitignore index e5ff3b7c..68cbae86 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,9 @@ vercel-build testbed/*/netlify examples/basic/pkg/basic.tar.gz +# Local Wrangler folder +.wrangler + # Fastly output main.wasm diff --git a/packages/base/multipart/readme.md b/packages/base/multipart/readme.md index b4a575dd..1ae5ae3b 100644 --- a/packages/base/multipart/readme.md +++ b/packages/base/multipart/readme.md @@ -1,7 +1,5 @@ # `@hattip/multipart` -> ⚠️ This package is work in progress. Please don't use in user-facing production code as it may have security issues. - Multipart parser for HatTip. It can be used to parse multipart requests, especially `multipart/form-data` for handling file uploads. The web standards offer [`Request.prototype.formData`](https://developer.mozilla.org/en-US/docs/Web/API/Request/formData) for parsing `multipart/form-data` requests, but it offers no way of enforcing size limits or controlling where the files are stored. Most implementations simply store the files in memory, which is not ideal for large files. diff --git a/packages/base/multipart/src/form-data.ts b/packages/base/multipart/src/form-data.ts index 6dae7b23..27303f62 100644 --- a/packages/base/multipart/src/form-data.ts +++ b/packages/base/multipart/src/form-data.ts @@ -21,16 +21,16 @@ export interface FormDataParserOptions { */ handleFile: FileHandler; /** Create the error to throw when a limit is exceeded */ - createLimitError?(name: string, value: number, limit: number): Error; + createLimitError?(name: string, value: number, limit: number): any; /** * Create the error to throw when the Content-Type header is not multipart * form-data with a boundary. */ - createTypeError?(): Error; + createTypeError?(): any; /** * Create the error to throw when the Content-Disposition header is * invalid. */ - createContentDispositionError?(): Error; + createContentDispositionError?(): any; /** The maximum number of headers @default 16 */ maxHeaderCount?: number; /** The maximum size of a header in bytes @default 1024 */ diff --git a/packages/bundler/bundler-cloudflare-workers/readme.md b/packages/bundler/bundler-cloudflare-workers/readme.md index 0f1a2c49..9f7671fe 100644 --- a/packages/bundler/bundler-cloudflare-workers/readme.md +++ b/packages/bundler/bundler-cloudflare-workers/readme.md @@ -26,7 +26,7 @@ export default { }; ``` -The output is a Clourflare Workers bundle that can be deployed with `wrangler` or tested with `wrangler dev --local`. +The output is a Clourflare Workers bundle that can be deployed with `wrangler` or tested with `wrangler dev`. ## JavaScript API diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3fbb8e34..61f0e812 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1081,6 +1081,9 @@ importers: '@hattip/graphql': specifier: workspace:* version: link:../../packages/middleware/graphql + '@hattip/multipart': + specifier: workspace:* + version: link:../../packages/base/multipart '@hattip/polyfills': specifier: workspace:* version: link:../../packages/base/polyfills diff --git a/testbed/basic/ci.test.ts b/testbed/basic/ci.test.ts index 4d3fab34..dd8faf9e 100644 --- a/testbed/basic/ci.test.ts +++ b/testbed/basic/ci.test.ts @@ -23,6 +23,7 @@ let cases: Array<{ skipCryptoTest?: boolean; skipStaticFileTest?: boolean; skipAdvancedStaticFileTest?: boolean; + skipMultipartTest?: boolean; tryStreamingWithoutCompression?: boolean; }>; @@ -77,6 +78,7 @@ if (process.env.CI === "true") { name: "Node with node-fetch", command: `node ${noFetchFlag} entry-node.js`, skipCryptoTest: nodeVersionMajor < 16, + skipMultipartTest: true, // node-fetch doesn't support streaming }, fetchAvailable && { name: "Node with @whatwg-node/fetch", @@ -140,6 +142,7 @@ if (process.env.CI === "true") { name: "Lagon", command: "lagon dev entry-lagon.js -p public --port 3000", skipAdvancedStaticFileTest: true, + skipMultipartTest: true, // Seems like a btoa bug in Lagon }, { name: "Google Cloud Functions", @@ -184,12 +187,13 @@ describe.each(cases)( command, envOverride, fetch = globalThis.fetch, - skipStreamingTest, requiresForwardedIp, + tryStreamingWithoutCompression, + skipStreamingTest, skipCryptoTest, skipStaticFileTest, skipAdvancedStaticFileTest, - tryStreamingWithoutCompression, + skipMultipartTest: skipMultipartTest, }) => { beforeAll(async () => { const original = fetch; @@ -450,7 +454,7 @@ describe.each(cases)( expect(text).toEqual('"bar"'); }); - test.skip("GraphQL", async () => { + test("GraphQL", async () => { function g(query: string) { return fetch(host + "/graphql", { method: "POST", @@ -472,6 +476,31 @@ describe.each(cases)( expect(r3).toStrictEqual({ data: { sum: 3 } }); }); + test.failsIf(skipMultipartTest)("multipart form data works", async () => { + const fd = new FormData(); + const data = Uint8Array.from( + new Array(300).fill(0).map((_, i) => i & 0xff), + ); + fd.append("file", new File([data], "hello.txt", { type: "text/plain" })); + fd.append("text", "Hello world! 😊"); + + const r = await fetch(host + "/form", { + method: "POST", + body: fd, + }).then((r) => r.json()); + + expect(r).toEqual({ + text: "Hello world! 😊", + file: { + name: "file", + filename: "hello.txt", + unsanitizedFilename: "hello.txt", + contentType: "text/plain", + body: Buffer.from(data).toString("base64"), + }, + }); + }); + test.failsIf(skipCryptoTest)("session", async () => { const response = await fetch(host + "/session"); const text = await response.text(); diff --git a/testbed/basic/fastly/.gitignore b/testbed/basic/fastly/.gitignore index 15c1f99b..fa095e2a 100644 --- a/testbed/basic/fastly/.gitignore +++ b/testbed/basic/fastly/.gitignore @@ -5,4 +5,4 @@ /src/statics.d.ts /src/statics-metadata.js /src/statics-metadata.d.ts -/src/static-content +/src/static-content \ No newline at end of file diff --git a/testbed/basic/index.js b/testbed/basic/index.js index 13512447..01f6d6cf 100644 --- a/testbed/basic/index.js +++ b/testbed/basic/index.js @@ -4,6 +4,7 @@ import { html, json, text } from "@hattip/response"; import { cookie } from "@hattip/cookie"; import { yoga, createSchema } from "@hattip/graphql"; import { session, EncryptedCookieStore } from "@hattip/session"; +import { parseMultipartFormData } from "@hattip/multipart"; const app = createRouter(); @@ -115,6 +116,8 @@ app.use( defaultQuery: `query { hello }`, }, + // TODO: Understand why this is needed + // @ts-expect-error schema, }), ); @@ -140,13 +143,64 @@ app.use("/session", async (ctx) => { return sessionMiddleware(ctx); }); -app.use("/session", (ctx) => { +app.get("/session", (ctx) => { // @ts-ignore ctx.session.data.count++; // @ts-ignore return text(`You have visited this page ${ctx.session.data.count} time(s).`); }); +/** @type {(stream: ReadableStream) => Promise} */ +async function readAll(stream) { + const reader = stream.getReader(); + /** @type {Uint8Array[]} */ + const chunks = []; + let totalLength = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + totalLength += value.length; + } + + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + + return result; +} + +/** @type {(data: ArrayBuffer) => string} */ +function toBase64(data) { + if (typeof Buffer !== "undefined") { + return Buffer.from(data).toString("base64"); + } + + return btoa(String.fromCharCode(...new Uint8Array(data))); +} + +app.post("/form", async (ctx) => { + const fd = await parseMultipartFormData(ctx.request, { + handleFile: async (fileInfo) => ({ + ...fileInfo, + body: toBase64(await readAll(fileInfo.body)), + }), + createTypeError: () => text("Unsupported media type", { status: 415 }), + createContentDispositionError: () => + text("Invalid content disposition", { status: 400 }), + createLimitError: (name, value, limit) => + text(`Field ${name} is too long (max ${limit} bytes)`, { status: 400 }), + }); + + return json(Object.fromEntries(fd)); +}); + app.get("/platform", (ctx) => { // @ts-expect-error return text(`Platform: ${ctx.platform.name}`); diff --git a/testbed/basic/package.json b/testbed/basic/package.json index 9326f777..9837e3f8 100644 --- a/testbed/basic/package.json +++ b/testbed/basic/package.json @@ -7,7 +7,7 @@ "start": "node --no-experimental-fetch entry-node.js", "start:native-fetch": "node --experimental-fetch entry-node-native-fetch.js", "build:cfw": "hattip-cloudflare-workers -e entry-cfw.js dist/cloudflare-workers-bundle/index.js", - "start:cfw": "wrangler dev --local --port 3000", + "start:cfw": "wrangler dev --port 3000", "build:netlify-functions": "rimraf .netlify/edge-functions-dist && hattip-netlify -c --staticDir public --func entry-netlify-function.js", "build:netlify-edge": "hattip-netlify -c --staticDir public --edge entry-netlify-edge.js", "start:netlify": "cross-env BROWSER=none netlify serve -op 3000", @@ -63,6 +63,7 @@ "@hattip/compose": "workspace:*", "@hattip/cookie": "workspace:*", "@hattip/graphql": "workspace:*", + "@hattip/multipart": "workspace:*", "@hattip/polyfills": "workspace:*", "@hattip/response": "workspace:*", "@hattip/router": "workspace:*", diff --git a/testbed/basic/tsconfig.json b/testbed/basic/tsconfig.json index cc557afc..9f78b71d 100644 --- a/testbed/basic/tsconfig.json +++ b/testbed/basic/tsconfig.json @@ -7,7 +7,8 @@ "strict": true, "skipLibCheck": true, "moduleResolution": "Node", - "checkJs": true + "checkJs": true, + "noEmit": true }, "include": ["index.js"] } diff --git a/testbed/basic/wrangler.toml b/testbed/basic/wrangler.toml index 0c18ac2b..2b231517 100644 --- a/testbed/basic/wrangler.toml +++ b/testbed/basic/wrangler.toml @@ -1,5 +1,5 @@ -compatibility_date = "2021-11-01" +compatibility_date = "2024-01-01" main = "dist/cloudflare-workers-bundle/index.js" name = "hattip-basic" usage_model = 'bundled'