Skip to content

Commit

Permalink
Marcel's enhancements to server-side csv generation (#4051)
Browse files Browse the repository at this point in the history 
Enhancements on top of #3613, see the commit messages for details.

The biggest change is replacing `jszip` with [`littlezipper`](https://www.npmjs.com/package/littlezipper), which (mostly) uses the browser-inbuilt (and Workers-inbuilt!) `CompressionStream` API for zip generation, which gives us fast, native compression.
It's also a good option if we ever should want to create a zip file directly in the browser - it uses `CompressionStream` if available ( = most modern browsers), or otherwise creates an uncompressed zip file as a fallback.
  • Loading branch information
@danyx23
danyx23 authored Oct 10, 2024
2 parents 5744b4b + 6195555 commit e08b424
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 117 deletions.
60 changes: 26 additions & 34 deletions functions/_common/grapherRenderer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { constructReadme } from "./readmeTools"
import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm"
import { TimeLogger } from "./timeLogger"
import { png, StatusError } from "itty-router"
import JSZip from "jszip"
import { createZip, File } from "littlezipper"

import svg2png_wasm from "../../node_modules/svg2png-wasm/svg2png_wasm_bg.wasm"

Expand All @@ -27,7 +27,6 @@ import LatoMedium from "../_common/fonts/LatoLatin-Medium.ttf.bin"
import LatoBold from "../_common/fonts/LatoLatin-Bold.ttf.bin"
import PlayfairSemiBold from "../_common/fonts/PlayfairDisplayLatin-SemiBold.ttf.bin"
import { Env } from "./env.js"
import { fromPairs } from "lodash"

declare global {
// eslint-disable-next-line no-var
Expand Down Expand Up @@ -318,7 +317,7 @@ async function initGrapher(
return grapher
}

function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
const getColumnsForMetadata = (grapher: Grapher) => {
const columnsToIgnore = new Set(
[
OwidTableSlugs.entityId,
Expand All @@ -331,12 +330,19 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
].map((slug) => slug.toString())
)

const columnsToGet = grapher.inputTable.columnSlugs.filter(
const colsToGet = grapher.inputTable.columnSlugs.filter(
(col) => !columnsToIgnore.has(col)
)

return grapher.inputTable.getColumns(colsToGet)
}

function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
const useShortNames = searchParams.get("useColumnShortNames") === "true"
console.log("useShortNames", useShortNames)

const metadataCols = getColumnsForMetadata(grapher)

const columns: [
string,
{
Expand Down Expand Up @@ -367,7 +373,7 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
>[]
shortName: string
},
][] = grapher.inputTable.getColumns(columnsToGet).map((col) => {
][] = metadataCols.map((col) => {
console.log("mapping col", col.name)
const {
descriptionShort,
Expand Down Expand Up @@ -498,7 +504,7 @@ function assembleMetadata(grapher: Grapher, searchParams: URLSearchParams) {
originalChartUrl: grapher.canonicalUrl,
selection: grapher.selectedEntityNames,
},
columns: fromPairs(columns),
columns: Object.fromEntries(columns),
}

return fullMetadata
Expand Down Expand Up @@ -545,14 +551,16 @@ export async function fetchZipForGrapher(
const readme = assembleReadme(grapher)
const csv = assembleCsv(grapher, searchParams)
console.log("Fetched the parts, creating zip file")
const zip = new JSZip()
zip.file(
`${identifier.id}.metadata.json`,
JSON.stringify(metadata, undefined, 2)
)
zip.file(`${identifier.id}.csv`, csv)
zip.file("readme.md", readme)
const content = await zip.generateAsync({ type: "arraybuffer" })

const zipContent: File[] = [
{
path: `${identifier.id}.metadata.json`,
data: JSON.stringify(metadata, undefined, 2),
},
{ path: `${identifier.id}.csv`, data: csv },
{ path: "readme.md", data: readme },
]
const content = await createZip(zipContent)
console.log("Generated content, returning response")
return new Response(content, {
headers: {
Expand Down Expand Up @@ -623,32 +631,16 @@ export async function fetchReadmeForGrapher(
const readme = assembleReadme(grapher)
return new Response(readme, {
headers: {
"Content-Type": "text/markdown",
"Content-Type": "text/markdown; charset=utf-8",
},
})
}

function assembleReadme(grapher: Grapher): string {
const columnsToIgnore = new Set(
[
OwidTableSlugs.entityId,
OwidTableSlugs.time,
OwidTableSlugs.entityColor,
OwidTableSlugs.entityName,
OwidTableSlugs.entityCode,
OwidTableSlugs.year,
OwidTableSlugs.day,
].map((slug) => slug.toString())
)

const columnsToGet = grapher.inputTable.columnSlugs.filter(
(col) => !columnsToIgnore.has(col)
)

const columns = grapher.inputTable.getColumns(columnsToGet)

return constructReadme(grapher, columns)
const metadataCols = getColumnsForMetadata(grapher)
return constructReadme(grapher, metadataCols)
}

async function fetchAndRenderGrapherToSvg(
identifier: GrapherIdentifier,
options: ImageOptions,
Expand Down
4 changes: 2 additions & 2 deletions functions/_common/readmeTools.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ export function constructReadme(
if (isSingleColumn)
readme = `# ${grapher.title} - Data package
This data package contains the data that powers the chart ["${grapher.title}"](${grapher.originUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}.
This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}.
## CSV Structure
Expand Down Expand Up @@ -274,7 +274,7 @@ ${sources.join("\n")}
else
readme = `# ${grapher.title} - Data package
This data package contains the data that powers the chart ["${grapher.title}"](${grapher.originUrl}) on the Our World in Data website.
This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website.
## CSV Structure
Expand Down
2 changes: 1 addition & 1 deletion functions/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"@ourworldindata/grapher": "workspace:^",
"@ourworldindata/utils": "workspace:^",
"itty-router": "^5.0.17",
"jszip": "^3.10.1",
"littlezipper": "^0.1.4",
"stripe": "^14.20.0",
"svg2png-wasm": "^1.4.1"
},
Expand Down
91 changes: 11 additions & 80 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2938,7 +2938,7 @@ __metadata:
languageName: node
linkType: hard

"@npmcli/package-json@npm:5.2.0, @npmcli/package-json@npm:^5.1.0":
"@npmcli/package-json@npm:5.2.0, @npmcli/package-json@npm:^5.0.0, @npmcli/package-json@npm:^5.1.0":
version: 5.2.0
resolution: "@npmcli/package-json@npm:5.2.0"
dependencies:
Expand All @@ -2953,21 +2953,6 @@ __metadata:
languageName: node
linkType: hard

"@npmcli/package-json@npm:^5.0.0":
version: 5.1.0
resolution: "@npmcli/package-json@npm:5.1.0"
dependencies:
"@npmcli/git": "npm:^5.0.0"
glob: "npm:^10.2.2"
hosted-git-info: "npm:^7.0.0"
json-parse-even-better-errors: "npm:^3.0.0"
normalize-package-data: "npm:^6.0.0"
proc-log: "npm:^4.0.0"
semver: "npm:^7.5.3"
checksum: 10/0e5cb5eff32cf80234525160a702c91a38e4b98ab74e34e2632b43c4350dbad170bd835989cc7d6e18d24798e3242e45b60f3d5e26bd128fe1c4529931105f8e
languageName: node
linkType: hard

"@npmcli/promise-spawn@npm:^7.0.0":
version: 7.0.2
resolution: "@npmcli/promise-spawn@npm:7.0.2"
Expand Down Expand Up @@ -8556,7 +8541,7 @@ __metadata:
languageName: node
linkType: hard

"dedent@npm:1.5.3":
"dedent@npm:1.5.3, dedent@npm:^1.0.0":
version: 1.5.3
resolution: "dedent@npm:1.5.3"
peerDependencies:
Expand All @@ -8568,18 +8553,6 @@ __metadata:
languageName: node
linkType: hard

"dedent@npm:^1.0.0":
version: 1.5.1
resolution: "dedent@npm:1.5.1"
peerDependencies:
babel-plugin-macros: ^3.1.0
peerDependenciesMeta:
babel-plugin-macros:
optional: true
checksum: 10/fc00a8bc3dfb7c413a778dc40ee8151b6c6ff35159d641f36ecd839c1df5c6e0ec5f4992e658c82624a1a62aaecaffc23b9c965ceb0bbf4d698bfc16469ac27d
languageName: node
linkType: hard

"deep-extend@npm:^0.6.0":
version: 0.6.0
resolution: "deep-extend@npm:0.6.0"
Expand Down Expand Up @@ -8853,14 +8826,7 @@ __metadata:
languageName: node
linkType: hard

"dotenv@npm:^16.0.1, dotenv@npm:^16.0.3":
version: 16.3.1
resolution: "dotenv@npm:16.3.1"
checksum: 10/dbb778237ef8750e9e3cd1473d3c8eaa9cc3600e33a75c0e36415d0fa0848197f56c3800f77924c70e7828f0b03896818cd52f785b07b9ad4d88dba73fbba83f
languageName: node
linkType: hard

"dotenv@npm:^16.4.4, dotenv@npm:~16.4.5":
"dotenv@npm:^16.0.1, dotenv@npm:^16.0.3, dotenv@npm:^16.4.4, dotenv@npm:~16.4.5":
version: 16.4.5
resolution: "dotenv@npm:16.4.5"
checksum: 10/55a3134601115194ae0f924e54473459ed0d9fc340ae610b676e248cca45aa7c680d86365318ea964e6da4e2ea80c4514c1adab5adb43d6867fb57ff068f95c8
Expand Down Expand Up @@ -11617,13 +11583,6 @@ __metadata:
languageName: node
linkType: hard

"immediate@npm:~3.0.5":
version: 3.0.6
resolution: "immediate@npm:3.0.6"
checksum: 10/f9b3486477555997657f70318cc8d3416159f208bec4cca3ff3442fd266bc23f50f0c9bd8547e1371a6b5e82b821ec9a7044a4f7b944798b25aa3cc6d5e63e62
languageName: node
linkType: hard

"immutable@npm:^4.0.0, immutable@npm:^4.3.6":
version: 4.3.6
resolution: "immutable@npm:4.3.6"
Expand Down Expand Up @@ -13243,18 +13202,6 @@ __metadata:
languageName: node
linkType: hard

"jszip@npm:^3.10.1":
version: 3.10.1
resolution: "jszip@npm:3.10.1"
dependencies:
lie: "npm:~3.3.0"
pako: "npm:~1.0.2"
readable-stream: "npm:~2.3.6"
setimmediate: "npm:^1.0.5"
checksum: 10/bfbfbb9b0a27121330ac46ab9cdb3b4812433faa9ba4a54742c87ca441e31a6194ff70ae12acefa5fe25406c432290e68003900541d948a169b23d30c34dd984
languageName: node
linkType: hard

"just-diff-apply@npm:^5.2.0":
version: 5.5.0
resolution: "just-diff-apply@npm:5.5.0"
Expand Down Expand Up @@ -13520,15 +13467,6 @@ __metadata:
languageName: node
linkType: hard

"lie@npm:~3.3.0":
version: 3.3.0
resolution: "lie@npm:3.3.0"
dependencies:
immediate: "npm:~3.0.5"
checksum: 10/f335ce67fe221af496185d7ce39c8321304adb701e122942c495f4f72dcee8803f9315ee572f5f8e8b08b9e8d7195da91b9fad776e8864746ba8b5e910adf76e
languageName: node
linkType: hard

"lilconfig@npm:3.0.0":
version: 3.0.0
resolution: "lilconfig@npm:3.0.0"
Expand Down Expand Up @@ -13612,6 +13550,13 @@ __metadata:
languageName: node
linkType: hard

"littlezipper@npm:^0.1.4":
version: 0.1.4
resolution: "littlezipper@npm:0.1.4"
checksum: 10/72b14b035a4cd2b5ac9999b955c5f17dcfa67859d83857f1c90d82a70cea4980cdc894915a8f94ed215cb5d207fcd64aa358081581349a167f7e2a261e3c5a04
languageName: node
linkType: hard

"load-json-file@npm:6.2.0":
version: 6.2.0
resolution: "load-json-file@npm:6.2.0"
Expand Down Expand Up @@ -15331,7 +15276,7 @@ __metadata:
"@ourworldindata/grapher": "workspace:^"
"@ourworldindata/utils": "workspace:^"
itty-router: "npm:^5.0.17"
jszip: "npm:^3.10.1"
littlezipper: "npm:^0.1.4"
stripe: "npm:^14.20.0"
svg2png-wasm: "npm:^1.4.1"
languageName: unknown
Expand Down Expand Up @@ -15507,13 +15452,6 @@ __metadata:
languageName: node
linkType: hard

"pako@npm:~1.0.2":
version: 1.0.11
resolution: "pako@npm:1.0.11"
checksum: 10/1ad07210e894472685564c4d39a08717e84c2a68a70d3c1d9e657d32394ef1670e22972a433cbfe48976cb98b154ba06855dcd3fcfba77f60f1777634bec48c0
languageName: node
linkType: hard

"papaparse@npm:^5.3.1":
version: 5.3.1
resolution: "papaparse@npm:5.3.1"
Expand Down Expand Up @@ -18114,13 +18052,6 @@ __metadata:
languageName: node
linkType: hard

"setimmediate@npm:^1.0.5":
version: 1.0.5
resolution: "setimmediate@npm:1.0.5"
checksum: 10/76e3f5d7f4b581b6100ff819761f04a984fa3f3990e72a6554b57188ded53efce2d3d6c0932c10f810b7c59414f85e2ab3c11521877d1dea1ce0b56dc906f485
languageName: node
linkType: hard

"setprototypeof@npm:1.2.0":
version: 1.2.0
resolution: "setprototypeof@npm:1.2.0"
Expand Down

0 comments on commit e08b424

Please sign in to comment.