From 9d7809718ba5c2ba92f8c9ef44d70b55897c5ff0 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 10 Oct 2024 11:48:23 +0200 Subject: [PATCH 1/6] enhance: use `Object.fromEntries()` --- functions/_common/grapherRenderer.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 5937f2a706..0f96ed4108 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -27,7 +27,6 @@ import LatoMedium from "../_common/fonts/LatoLatin-Medium.ttf.bin" import LatoBold from "../_common/fonts/LatoLatin-Bold.ttf.bin" import PlayfairSemiBold from "../_common/fonts/PlayfairDisplayLatin-SemiBold.ttf.bin" import { Env } from "./env.js" -import { fromPairs } from "lodash" declare global { // eslint-disable-next-line no-var @@ -498,7 +497,7 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) { originalChartUrl: grapher.canonicalUrl, selection: grapher.selectedEntityNames, }, - columns: fromPairs(columns), + columns: Object.fromEntries(columns), } return fullMetadata From 192cf7de88679cd665e426657670ac4335d101ac Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 10 Oct 2024 11:52:12 +0200 Subject: [PATCH 2/6] fix: use `canonicalUrl` over `originUrl` --- functions/_common/readmeTools.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index 1c189706f4..08559be287 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -246,7 +246,7 @@ export function constructReadme( if (isSingleColumn) readme = `# ${grapher.title} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${grapher.originUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}. +This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}. ## CSV Structure @@ -274,7 +274,7 @@ ${sources.join("\n")} else readme = `# ${grapher.title} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${grapher.originUrl}) on the Our World in Data website. +This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. ## CSV Structure From 665c4384ce30156d4b525238d59fa0dd1687f437 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 10 Oct 2024 11:58:33 +0200 Subject: [PATCH 3/6] fix: set correct content-type for readme --- functions/_common/grapherRenderer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 0f96ed4108..f5d9c3ef83 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -622,7 +622,7 @@ export async function fetchReadmeForGrapher( const readme = assembleReadme(grapher) return new Response(readme, { headers: { - "Content-Type": "text/markdown", + "Content-Type": "text/markdown; charset=utf-8", }, }) } From 49eb9068112e8ed421320ea7efa1abac65532a8e Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 10 Oct 2024 11:59:16 +0200 Subject: [PATCH 4/6] chore(deps): dedupe --- yarn.lock | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/yarn.lock b/yarn.lock index ac7a3bf197..901f7d96a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2938,7 +2938,7 @@ __metadata: languageName: node linkType: hard -"@npmcli/package-json@npm:5.2.0, @npmcli/package-json@npm:^5.1.0": +"@npmcli/package-json@npm:5.2.0, @npmcli/package-json@npm:^5.0.0, @npmcli/package-json@npm:^5.1.0": version: 5.2.0 resolution: "@npmcli/package-json@npm:5.2.0" dependencies: @@ -2953,21 +2953,6 @@ __metadata: languageName: node linkType: hard -"@npmcli/package-json@npm:^5.0.0": - version: 5.1.0 - resolution: "@npmcli/package-json@npm:5.1.0" - dependencies: - "@npmcli/git": "npm:^5.0.0" - glob: "npm:^10.2.2" - hosted-git-info: "npm:^7.0.0" - json-parse-even-better-errors: "npm:^3.0.0" - normalize-package-data: "npm:^6.0.0" - proc-log: "npm:^4.0.0" - semver: "npm:^7.5.3" - checksum: 10/0e5cb5eff32cf80234525160a702c91a38e4b98ab74e34e2632b43c4350dbad170bd835989cc7d6e18d24798e3242e45b60f3d5e26bd128fe1c4529931105f8e - languageName: node - linkType: hard - "@npmcli/promise-spawn@npm:^7.0.0": version: 7.0.2 resolution: "@npmcli/promise-spawn@npm:7.0.2" @@ -8556,7 +8541,7 @@ __metadata: languageName: node linkType: hard -"dedent@npm:1.5.3": +"dedent@npm:1.5.3, dedent@npm:^1.0.0": version: 1.5.3 resolution: "dedent@npm:1.5.3" peerDependencies: @@ -8568,18 +8553,6 @@ __metadata: languageName: node linkType: hard -"dedent@npm:^1.0.0": - version: 1.5.1 - resolution: "dedent@npm:1.5.1" - peerDependencies: - babel-plugin-macros: ^3.1.0 - peerDependenciesMeta: - babel-plugin-macros: - optional: true - checksum: 10/fc00a8bc3dfb7c413a778dc40ee8151b6c6ff35159d641f36ecd839c1df5c6e0ec5f4992e658c82624a1a62aaecaffc23b9c965ceb0bbf4d698bfc16469ac27d - languageName: node - linkType: hard - "deep-extend@npm:^0.6.0": version: 0.6.0 resolution: "deep-extend@npm:0.6.0" @@ -8853,14 +8826,7 @@ __metadata: languageName: node linkType: hard -"dotenv@npm:^16.0.1, dotenv@npm:^16.0.3": - version: 16.3.1 - resolution: "dotenv@npm:16.3.1" - checksum: 10/dbb778237ef8750e9e3cd1473d3c8eaa9cc3600e33a75c0e36415d0fa0848197f56c3800f77924c70e7828f0b03896818cd52f785b07b9ad4d88dba73fbba83f - languageName: node - linkType: hard - -"dotenv@npm:^16.4.4, dotenv@npm:~16.4.5": +"dotenv@npm:^16.0.1, dotenv@npm:^16.0.3, dotenv@npm:^16.4.4, dotenv@npm:~16.4.5": version: 16.4.5 resolution: "dotenv@npm:16.4.5" checksum: 10/55a3134601115194ae0f924e54473459ed0d9fc340ae610b676e248cca45aa7c680d86365318ea964e6da4e2ea80c4514c1adab5adb43d6867fb57ff068f95c8 From 0615d5dff6ac25b8d706877c7476330aafdf78d3 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 10 Oct 2024 12:29:41 +0200 Subject: [PATCH 5/6] enhance: use `littlezipper` instead of `jszip` it's very lightweight, and uses the built-in `CompressionStream` for compression (available in CF Workers) --- functions/_common/grapherRenderer.ts | 20 ++++++----- functions/package.json | 2 +- yarn.lock | 51 +++++----------------------- 3 files changed, 20 insertions(+), 53 deletions(-) diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index f5d9c3ef83..867440f30c 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -17,7 +17,7 @@ import { constructReadme } from "./readmeTools" import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm" import { TimeLogger } from "./timeLogger" import { png, StatusError } from "itty-router" -import JSZip from "jszip" +import { createZip, File } from "littlezipper" import svg2png_wasm from "../../node_modules/svg2png-wasm/svg2png_wasm_bg.wasm" @@ -544,14 +544,16 @@ export async function fetchZipForGrapher( const readme = assembleReadme(grapher) const csv = assembleCsv(grapher, searchParams) console.log("Fetched the parts, creating zip file") - const zip = new JSZip() - zip.file( - `${identifier.id}.metadata.json`, - JSON.stringify(metadata, undefined, 2) - ) - zip.file(`${identifier.id}.csv`, csv) - zip.file("readme.md", readme) - const content = await zip.generateAsync({ type: "arraybuffer" }) + + const zipContent: File[] = [ + { + path: `${identifier.id}.metadata.json`, + data: JSON.stringify(metadata, undefined, 2), + }, + { path: `${identifier.id}.csv`, data: csv }, + { path: "readme.md", data: readme }, + ] + const content = await createZip(zipContent) console.log("Generated content, returning response") return new Response(content, { headers: { diff --git a/functions/package.json b/functions/package.json index 15600d6356..33faae7d58 100644 --- a/functions/package.json +++ b/functions/package.json @@ -4,7 +4,7 @@ "@ourworldindata/grapher": "workspace:^", "@ourworldindata/utils": "workspace:^", "itty-router": "^5.0.17", - "jszip": "^3.10.1", + "littlezipper": "^0.1.4", "stripe": "^14.20.0", "svg2png-wasm": "^1.4.1" }, diff --git a/yarn.lock b/yarn.lock index 901f7d96a9..7833e4923c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11583,13 +11583,6 @@ __metadata: languageName: node linkType: hard -"immediate@npm:~3.0.5": - version: 3.0.6 - resolution: "immediate@npm:3.0.6" - checksum: 10/f9b3486477555997657f70318cc8d3416159f208bec4cca3ff3442fd266bc23f50f0c9bd8547e1371a6b5e82b821ec9a7044a4f7b944798b25aa3cc6d5e63e62 - languageName: node - linkType: hard - "immutable@npm:^4.0.0, immutable@npm:^4.3.6": version: 4.3.6 resolution: "immutable@npm:4.3.6" @@ -13209,18 +13202,6 @@ __metadata: languageName: node linkType: hard -"jszip@npm:^3.10.1": - version: 3.10.1 - resolution: "jszip@npm:3.10.1" - dependencies: - lie: "npm:~3.3.0" - pako: "npm:~1.0.2" - readable-stream: "npm:~2.3.6" - setimmediate: "npm:^1.0.5" - checksum: 10/bfbfbb9b0a27121330ac46ab9cdb3b4812433faa9ba4a54742c87ca441e31a6194ff70ae12acefa5fe25406c432290e68003900541d948a169b23d30c34dd984 - languageName: node - linkType: hard - "just-diff-apply@npm:^5.2.0": version: 5.5.0 resolution: "just-diff-apply@npm:5.5.0" @@ -13486,15 +13467,6 @@ __metadata: languageName: node linkType: hard -"lie@npm:~3.3.0": - version: 3.3.0 - resolution: "lie@npm:3.3.0" - dependencies: - immediate: "npm:~3.0.5" - checksum: 10/f335ce67fe221af496185d7ce39c8321304adb701e122942c495f4f72dcee8803f9315ee572f5f8e8b08b9e8d7195da91b9fad776e8864746ba8b5e910adf76e - languageName: node - linkType: hard - "lilconfig@npm:3.0.0": version: 3.0.0 resolution: "lilconfig@npm:3.0.0" @@ -13578,6 +13550,13 @@ __metadata: languageName: node linkType: hard +"littlezipper@npm:^0.1.4": + version: 0.1.4 + resolution: "littlezipper@npm:0.1.4" + checksum: 10/72b14b035a4cd2b5ac9999b955c5f17dcfa67859d83857f1c90d82a70cea4980cdc894915a8f94ed215cb5d207fcd64aa358081581349a167f7e2a261e3c5a04 + languageName: node + linkType: hard + "load-json-file@npm:6.2.0": version: 6.2.0 resolution: "load-json-file@npm:6.2.0" @@ -15297,7 +15276,7 @@ __metadata: "@ourworldindata/grapher": "workspace:^" "@ourworldindata/utils": "workspace:^" itty-router: "npm:^5.0.17" - jszip: "npm:^3.10.1" + littlezipper: "npm:^0.1.4" stripe: "npm:^14.20.0" svg2png-wasm: "npm:^1.4.1" languageName: unknown @@ -15473,13 +15452,6 @@ __metadata: languageName: node linkType: hard -"pako@npm:~1.0.2": - version: 1.0.11 - resolution: "pako@npm:1.0.11" - checksum: 10/1ad07210e894472685564c4d39a08717e84c2a68a70d3c1d9e657d32394ef1670e22972a433cbfe48976cb98b154ba06855dcd3fcfba77f60f1777634bec48c0 - languageName: node - linkType: hard - "papaparse@npm:^5.3.1": version: 5.3.1 resolution: "papaparse@npm:5.3.1" @@ -18080,13 +18052,6 @@ __metadata: languageName: node linkType: hard -"setimmediate@npm:^1.0.5": - version: 1.0.5 - resolution: "setimmediate@npm:1.0.5" - checksum: 10/76e3f5d7f4b581b6100ff819761f04a984fa3f3990e72a6554b57188ded53efce2d3d6c0932c10f810b7c59414f85e2ab3c11521877d1dea1ce0b56dc906f485 - languageName: node - linkType: hard - "setprototypeof@npm:1.2.0": version: 1.2.0 resolution: "setprototypeof@npm:1.2.0" From 6195555eeb25f1edf5b0e635f9beff7d562ed676 Mon Sep 17 00:00:00 2001 From: Marcel Gerber Date: Thu, 10 Oct 2024 12:42:20 +0200 Subject: [PATCH 6/6] enhance: centralize `columnsToIgnore` --- functions/_common/grapherRenderer.ts | 35 +++++++++++----------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 867440f30c..4dd28fceb3 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -317,7 +317,7 @@ async function initGrapher( return grapher } -function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) { +const getColumnsForMetadata = (grapher: Grapher) => { const columnsToIgnore = new Set( [ OwidTableSlugs.entityId, @@ -330,12 +330,19 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) { ].map((slug) => slug.toString()) ) - const columnsToGet = grapher.inputTable.columnSlugs.filter( + const colsToGet = grapher.inputTable.columnSlugs.filter( (col) => !columnsToIgnore.has(col) ) + + return grapher.inputTable.getColumns(colsToGet) +} + +function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) { const useShortNames = searchParams.get("useColumnShortNames") === "true" console.log("useShortNames", useShortNames) + const metadataCols = getColumnsForMetadata(grapher) + const columns: [ string, { @@ -366,7 +373,7 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) { >[] shortName: string }, - ][] = grapher.inputTable.getColumns(columnsToGet).map((col) => { + ][] = metadataCols.map((col) => { console.log("mapping col", col.name) const { descriptionShort, @@ -630,26 +637,10 @@ export async function fetchReadmeForGrapher( } function assembleReadme(grapher: Grapher): string { - const columnsToIgnore = new Set( - [ - OwidTableSlugs.entityId, - OwidTableSlugs.time, - OwidTableSlugs.entityColor, - OwidTableSlugs.entityName, - OwidTableSlugs.entityCode, - OwidTableSlugs.year, - OwidTableSlugs.day, - ].map((slug) => slug.toString()) - ) - - const columnsToGet = grapher.inputTable.columnSlugs.filter( - (col) => !columnsToIgnore.has(col) - ) - - const columns = grapher.inputTable.getColumns(columnsToGet) - - return constructReadme(grapher, columns) + const metadataCols = getColumnsForMetadata(grapher) + return constructReadme(grapher, metadataCols) } + async function fetchAndRenderGrapherToSvg( identifier: GrapherIdentifier, options: ImageOptions,