From 8da3b43ab3950742ec58a6ec32c0cf8c7274ce36 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Mon, 5 Aug 2024 19:19:23 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20use=20grapher=20configs=20from?= =?UTF-8?q?=20R2=20for=20thumbnail=20rendering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dev.vars.example | 2 + adminSiteServer/apiRouter.ts | 2 +- adminSiteServer/chartConfigR2Helpers.ts | 6 +- devTools/syncGraphersToR2/syncGraphersToR2.ts | 2 +- functions/README.md | 10 +- functions/_common/grapherRenderer.ts | 110 ++++++++++++++---- functions/grapher/thumbnail/[slug].ts | 5 + .../types/src/domainTypes/Various.ts | 5 + packages/@ourworldindata/types/src/index.ts | 1 + wrangler.toml | 13 +++ 10 files changed, 121 insertions(+), 35 deletions(-) diff --git a/.dev.vars.example b/.dev.vars.example index 5e3ac2ec579..384f1dc15a7 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -20,3 +20,5 @@ MAILGUN_SENDING_KEY= # optional SLACK_BOT_OAUTH_TOKEN= SLACK_ERROR_CHANNEL_ID=C016H0BNNB1 #bot-testing channel + +GRAPHER_CONFIG_R2_BUCKET_PATH=devs/YOURNAME diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 800f9d0c7b9..5d845c205bb 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -101,6 +101,7 @@ import { FlatTagGraph, DbRawChartConfig, parseChartConfig, + R2GrapherConfigDirectory, } from "@ourworldindata/types" import { uuidv7 } from "uuidv7" import { @@ -174,7 +175,6 @@ import path from "path" import { deleteGrapherConfigFromR2, deleteGrapherConfigFromR2ByUUID, - R2GrapherConfigDirectory, saveGrapherConfigToR2, saveGrapherConfigToR2ByUUID, } from "./chartConfigR2Helpers.js" diff --git a/adminSiteServer/chartConfigR2Helpers.ts b/adminSiteServer/chartConfigR2Helpers.ts index be6277b1f22..7f2623aade9 100644 --- a/adminSiteServer/chartConfigR2Helpers.ts +++ b/adminSiteServer/chartConfigR2Helpers.ts @@ -14,14 +14,10 @@ import { S3Client, } from "@aws-sdk/client-s3" import { JsonError, lazy } from "@ourworldindata/utils" +import { R2GrapherConfigDirectory } from "@ourworldindata/types" import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" import { Base64String } from "../serverUtils/serverUtil.js" -export enum R2GrapherConfigDirectory { - byUUID = "config/by-uuid", - publishedGrapherBySlug = "config/by-slug-published", -} - const getS3Client: () => S3Client = lazy( () => new S3Client({ diff --git a/devTools/syncGraphersToR2/syncGraphersToR2.ts b/devTools/syncGraphersToR2/syncGraphersToR2.ts index 75c71543610..9e546950939 100644 --- a/devTools/syncGraphersToR2/syncGraphersToR2.ts +++ b/devTools/syncGraphersToR2/syncGraphersToR2.ts @@ -22,7 +22,6 @@ import { KnexReadonlyTransaction, knexReadonlyTransaction, } from "../../db/db.js" -import { R2GrapherConfigDirectory } from "../../adminSiteServer/chartConfigR2Helpers.js" import { DbRawChartConfig, excludeUndefined } from "@ourworldindata/utils" import { chunk } from "lodash" import ProgressBar from "progress" @@ -31,6 +30,7 @@ import { HexString, hexToBytes, } from "../../serverUtils/serverUtil.js" +import { R2GrapherConfigDirectory } from "@ourworldindata/types" type HashAndId = Pick diff --git a/functions/README.md b/functions/README.md index caaa61334a0..42219c564b4 100644 --- a/functions/README.md +++ b/functions/README.md @@ -10,6 +10,8 @@ Pages Functions are very similar to Cloudflare Workers; however they will always Pages Functions use file-based routing, which means that the file `grapher/[slug].ts` will serve routes like `/grapher/child-mortality`. In addition, there's a [`_routes.json`](../_routes.json) file that specifies which routes are to be served dynamically. +Inside a file-based route we sometimes use an instance of itty-router to decide on the exact functionality to provide (e.g. png vs svg generation) + ## Development 1. Copy `.dev.vars.example` to `.dev.vars` and fill in the required variables. @@ -26,9 +28,11 @@ Note: compatibility dates between local development, production and preview envi 3. _Refer to each function's "Development" section below for further instructions._ -## Testing on Fondation staging sites vs Cloudfare previews +## Testing on Foundation staging sites vs Cloudflare previews + +We have two cloudflare projects set up that you can deploy previews to. `owid` which is also where our production deployment runs, and `owid-staging`. Currently, `owid` is configured to require authentication while `owid-staging` is accessible from the internet without any kind of auth. -`yarn deployContentPreview` deploys the staging `bakedSite` to a Cloudflare preview at https://[PREVIEW_BRANCH].owid-staging.pages.dev. This is the recommended way to test functions in a production-like environment. See [../ops/buildkite/deploy-content-preview](../ops/buildkite/deploy-content-preview) for more details. +`yarn deployContentPreview` deploys the staging `bakedSite` to a Cloudflare preview at https://[PREVIEW_BRANCH].[PROJECT].pages.dev. This is the recommended way to test functions in a production-like environment. See [../ops/buildkite/deploy-content-preview](../ops/buildkite/deploy-content-preview) for more details. ### Rationale @@ -36,7 +40,7 @@ A custom staging site is available at http://staging-site-[BRANCH] upon pushing When it comes to testing functions in a production-like environment, Cloudflare previews are recommended. -Cloudflare previews are served by Cloudflare (as opposed to `wrangler` on staging sites) and are available at https://[RANDOM_ID].owid-staging.pages.dev. Cloudflare previews do not rely on the `wrangler` CLI and its `.dev.vars` file. Instead, they use the [Cloudflare dashboard to configure environment variables](https://dash.cloudflare.com/078fcdfed9955087315dd86792e71a7e/pages/view/owid/settings/environment-variables), in the same way and place as the production site. +Cloudflare previews are served by Cloudflare (as opposed to `wrangler` on staging sites) and are available at https://[RANDOM_ID].[PROJECT].pages.dev. Cloudflare previews do not rely on the `wrangler` CLI and its `.dev.vars` file, but they do take the `wrangler.toml` file into account for environment variables. For secrets, they use the [values set via the Cloudflare dashboard](https://dash.cloudflare.com/078fcdfed9955087315dd86792e71a7e/pages/view/owid/settings/environment-variables), in the same way and place as the production site. This proximity of configurations in the Cloudflare dashboard makes spotting differences between production and preview environments easier - and is one of the reason of using Cloudflare previews in the same project (owid) over using a new project specific to staging. diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 249488b75bf..f189d2b6329 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -1,5 +1,10 @@ -import { Grapher, GrapherInterface } from "@ourworldindata/grapher" -import { Bounds, deserializeJSONFromHTML } from "@ourworldindata/utils" +import { Grapher } from "@ourworldindata/grapher" +import { + Bounds, + excludeUndefined, + GrapherInterface, + R2GrapherConfigDirectory, +} from "@ourworldindata/utils" import { svg2png, initialize as initializeSvg2Png } from "svg2png-wasm" import { TimeLogger } from "./timeLogger" import { png } from "itty-router" @@ -130,32 +135,82 @@ const extractOptions = (params: URLSearchParams): ImageOptions => { return options as ImageOptions } -async function fetchAndRenderGrapherToSvg({ - slug, - options, - searchParams, - env, -}: { - slug: string - options: ImageOptions - searchParams: URLSearchParams +const WORKER_CACHE_TIME_IN_SECONDS = 60 + +async function fetchFromR2( + url: URL, + etag: string | undefined, + fallbackUrl?: URL +) { + const headers = new Headers() + if (etag) headers.set("If-None-Match", etag) + const init = { + cf: { + cacheEverything: true, + cacheTtl: WORKER_CACHE_TIME_IN_SECONDS, + }, + headers, + } + const primaryResponse = await fetch(url.toString(), init) + if (primaryResponse.status === 404 && fallbackUrl) { + return fetch(fallbackUrl.toString(), init) + } + return primaryResponse +} + +async function fetchAndRenderGrapherToSvg( + slug: string, + options: ImageOptions, + searchParams: URLSearchParams, env: Env -}) { +) { const grapherLogger = new TimeLogger("grapher") - // Fetch grapher config and extract it from the HTML - const grapherConfig: GrapherInterface = await env.ASSETS.fetch( - new URL(`/grapher/${slug}`, env.url) - ) - .then((r) => (r.ok ? r : Promise.reject("Failed to load grapher page"))) - .then((r) => r.text()) - .then((html) => deserializeJSONFromHTML(html)) + // The top level directory is either the bucket path (should be set in dev environments and production) + // or the branch name on preview staging environments + console.log("branch", env.CF_PAGES_BRANCH) + const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_PATH + ? [env.GRAPHER_CONFIG_R2_BUCKET_PATH] + : ["by-branch", env.CF_PAGES_BRANCH] + + const key = excludeUndefined([ + ...topLevelDirectory, + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${slug}.json`, + ]).join("/") + + console.log("fetching grapher config from this key", key) + + const requestUrl = new URL(key, env.GRAPHER_CONFIG_R2_BUCKET_URL) + + let fallbackUrl - if (!grapherConfig) { - throw new Error("Could not find grapher config") + if ( + env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL && + env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH + ) { + const topLevelDirectory = env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH + const fallbackKey = excludeUndefined([ + topLevelDirectory, + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${slug}.json`, + ]).join("/") + fallbackUrl = new URL( + fallbackKey, + env.GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL + ) + } + + // Fetch grapher config + const fetchResponse = await fetchFromR2(requestUrl, undefined, fallbackUrl) + + if (fetchResponse.status !== 200) { + console.log("Failed to fetch grapher config", fetchResponse.status) + return null } - grapherLogger.log("fetchGrapherConfig") + const grapherConfig: GrapherInterface = await fetchResponse.json() + console.log("grapher title", grapherConfig.title) const bounds = new Bounds(0, 0, options.svgWidth, options.svgHeight) const grapher = new Grapher({ @@ -199,12 +254,17 @@ export const fetchAndRenderGrapher = async ( const options = extractOptions(searchParams) console.log("Rendering", slug, outType, options) - const svg = await fetchAndRenderGrapherToSvg({ + const svg = await fetchAndRenderGrapherToSvg( slug, options, searchParams, - env, - }) + env + ) + console.log("fetched svg") + + if (!svg) { + return new Response("Not found", { status: 404 }) + } switch (outType) { case "png": diff --git a/functions/grapher/thumbnail/[slug].ts b/functions/grapher/thumbnail/[slug].ts index b5efae2ac13..e49e32508fc 100644 --- a/functions/grapher/thumbnail/[slug].ts +++ b/functions/grapher/thumbnail/[slug].ts @@ -6,6 +6,11 @@ export interface Env { fetch: typeof fetch } url: URL + GRAPHER_CONFIG_R2_BUCKET_URL: string + GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL: string + GRAPHER_CONFIG_R2_BUCKET_PATH: string + GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH: string + CF_PAGES_BRANCH: string ENV: string } diff --git a/packages/@ourworldindata/types/src/domainTypes/Various.ts b/packages/@ourworldindata/types/src/domainTypes/Various.ts index 5bf4f42f6cd..b9d0daca9c7 100644 --- a/packages/@ourworldindata/types/src/domainTypes/Various.ts +++ b/packages/@ourworldindata/types/src/domainTypes/Various.ts @@ -64,3 +64,8 @@ export class JsonError extends Error { export interface QueryParams { [key: string]: string | undefined } + +export enum R2GrapherConfigDirectory { + byUUID = "config/by-uuid", + publishedGrapherBySlug = "config/by-slug-published", +} diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index a88b15903b5..ba118c9b489 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -17,6 +17,7 @@ export { type RawPageview, type UserCountryInformation, type QueryParams, + R2GrapherConfigDirectory, } from "./domainTypes/Various.js" export { type BreadcrumbItem, type KeyValueProps } from "./domainTypes/Site.js" export { diff --git a/wrangler.toml b/wrangler.toml index 4d88b657784..1d28e99e333 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -6,12 +6,19 @@ pages_build_output_dir = "./localBake" # Vars that should be available in all envs, including local dev [vars] ENV = "development" +GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs-staging.owid.io" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "https://grapher-configs.owid.io" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "v1" + # Overrides for CF preview deployments [env.preview.vars] MAILGUN_DOMAIN = "mg.ourworldindata.org" SLACK_ERROR_CHANNEL_ID = "C016H0BNNB1" ENV = "preview" +GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs-staging.owid.io" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "https://grapher-configs.owid.io" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "v1" # Overrides for CF production deployment [env.production] @@ -21,3 +28,9 @@ compatibility_date = "2024-04-29" ENV = "production" MAILGUN_DOMAIN = "mg.ourworldindata.org" SLACK_ERROR_CHANNEL_ID = "C5JJW19PS" +GRAPHER_CONFIG_R2_BUCKET_URL = "https://grapher-configs.owid.io" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_URL = "" +GRAPHER_CONFIG_R2_BUCKET_FALLBACK_PATH = "" +GRAPHER_CONFIG_R2_BUCKET_PATH = "v1" + +