From 37780f9e9ea54375d47675dace415a2a328db588 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Sat, 16 Mar 2024 14:34:08 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20switch=20almost=20all=20remainin?= =?UTF-8?q?g=20typeorm=20util=20function=20holdouts=20to=20knex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only remaining uses of the typeorm functions are related to the chart revision tool --- adminSiteServer/apiRouter.ts | 169 ++++++++++++++++---------- adminSiteServer/authentication.tsx | 28 +++-- adminSiteServer/gitDataExport.ts | 1 - adminSiteServer/mockSiteRouter.tsx | 8 +- adminSiteServer/publicApiRouter.ts | 7 +- adminSiteServer/testPageRouter.tsx | 38 +++--- baker/GrapherBaker.tsx | 5 +- baker/GrapherBakingUtils.ts | 15 ++- baker/GrapherImageBaker.tsx | 30 +++-- baker/SiteBaker.tsx | 9 +- baker/algolia/indexChartsToAlgolia.ts | 1 - baker/algolia/indexToAlgolia.tsx | 1 - baker/bakeGdocPost.ts | 1 - baker/bakeGdocPosts.ts | 1 - baker/redirects.ts | 13 +- baker/siteRenderers.tsx | 37 ++++-- baker/startDeployQueueServer.ts | 2 - baker/syncRedirectsToGrapher.ts | 1 - db/analyzeWpPosts.ts | 73 +++++------ db/db.ts | 3 - db/exportMetadata.ts | 2 - db/model/Post.ts | 1 - db/syncPostsToGrapher.ts | 1 - db/tests/basic.test.ts | 10 +- devTools/markdownTest/markdown.ts | 3 - devTools/svgTester/dump-data.ts | 11 +- 26 files changed, 277 insertions(+), 194 deletions(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index df427266ebc..22305ccc2a9 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -214,7 +214,8 @@ const getReferencesByChartId = async ( knex ) const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex) - const explorerSlugsPromise = db.queryMysql( + const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>( + knex, `select distinct explorerSlug from explorer_charts where chartId = ?`, [chartId] ) @@ -426,12 +427,13 @@ getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { return { charts } }) -apiRouter.get("/charts.csv", async (req, res) => { +getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 // note: this query is extended from OldChart.listFields. - const charts = await db.queryMysql( - ` + const charts = await db.knexRaw( + trx, + `-- sql SELECT charts.id, charts.config->>"$.version" AS version, @@ -486,25 +488,34 @@ getRouteWithROTransaction( async (req, res, trx) => expectChartById(trx, req.params.chartId) ) -apiRouter.get("/editorData/namespaces.json", async (req, res) => { - const rows = (await db.queryMysql( - `SELECT DISTINCT +getRouteWithROTransaction( + apiRouter, + "/editorData/namespaces.json", + async (req, res, trx) => { + const rows = await db.knexRaw<{ + name: string + description?: string + isArchived: boolean + }>( + trx, + `SELECT DISTINCT namespace AS name, namespaces.description AS description, namespaces.isArchived AS isArchived FROM active_datasets JOIN namespaces ON namespaces.name = active_datasets.namespace` - )) as { name: string; description?: string; isArchived: boolean }[] + ) - return { - namespaces: lodash - .sortBy(rows, (row) => row.description) - .map((namespace) => ({ - ...namespace, - isArchived: !!namespace.isArchived, - })), + return { + namespaces: lodash + .sortBy(rows, (row) => row.description) + .map((namespace) => ({ + ...namespace, + isArchived: !!namespace.isArchived, + })), + } } -}) +) getRouteWithROTransaction( apiRouter, @@ -1375,17 +1386,19 @@ getRouteWithROTransaction( } ) -apiRouter.delete("/users/:userId", async (req, res) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) +deleteRouteWithRWTransaction( + apiRouter, + "/users/:userId", + async (req, res, trx) => { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) - const userId = expectInt(req.params.userId) - await db.transaction(async (t) => { - await t.execute(`DELETE FROM users WHERE id=?`, [userId]) - }) + const userId = expectInt(req.params.userId) + await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) - return { success: true } -}) + return { success: true } + } +) putRouteWithRWTransaction( apiRouter, @@ -1436,10 +1449,13 @@ getRouteWithROTransaction( } ) -apiRouter.get( +getRouteWithROTransaction( + apiRouter, "/chart-bulk-update", async ( - req + req, + res, + trx ): Promise> => { const context: OperationContext = { grapherConfigFieldName: "config", @@ -1458,8 +1474,9 @@ apiRouter.get( // careful there to only allow carefully guarded vocabularies from being used, not // arbitrary user input const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = - await db.queryMysql(`SELECT charts.id as id, + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `SELECT charts.id as id, charts.config as config, charts.createdAt as createdAt, charts.updatedAt as updatedAt, @@ -1473,15 +1490,19 @@ LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId WHERE ${whereClause} ORDER BY charts.id DESC LIMIT 50 -OFFSET ${offset.toString()}`) +OFFSET ${offset.toString()}` + ) const results = resultsWithStringGrapherConfigs.map((row: any) => ({ ...row, config: lodash.isNil(row.config) ? null : JSON.parse(row.config), })) - const resultCount = await db.queryMysql(`SELECT count(*) as count + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `SELECT count(*) as count FROM charts -WHERE ${whereClause}`) +WHERE ${whereClause}` + ) return { rows: results, numTotalRows: resultCount[0].count } } ) @@ -1527,10 +1548,13 @@ patchRouteWithRWTransaction( } ) -apiRouter.get( +getRouteWithROTransaction( + apiRouter, "/variable-annotations", async ( - req + req, + res, + trx ): Promise> => { const context: OperationContext = { grapherConfigFieldName: "grapherConfigAdmin", @@ -1549,8 +1573,9 @@ apiRouter.get( // careful there to only allow carefully guarded vocabularies from being used, not // arbitrary user input const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = - await db.queryMysql(`SELECT variables.id as id, + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `SELECT variables.id as id, variables.name as name, variables.grapherConfigAdmin as config, d.name as datasetname, @@ -1564,30 +1589,37 @@ LEFT JOIN namespaces on d.namespace = namespaces.name WHERE ${whereClause} ORDER BY variables.id DESC LIMIT 50 -OFFSET ${offset.toString()}`) +OFFSET ${offset.toString()}` + ) const results = resultsWithStringGrapherConfigs.map((row: any) => ({ ...row, config: lodash.isNil(row.config) ? null : JSON.parse(row.config), })) - const resultCount = await db.queryMysql(`SELECT count(*) as count + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `SELECT count(*) as count FROM variables LEFT JOIN active_datasets as d on variables.datasetId = d.id LEFT JOIN namespaces on d.namespace = namespaces.name -WHERE ${whereClause}`) +WHERE ${whereClause}` + ) return { rows: results, numTotalRows: resultCount[0].count } } ) -apiRouter.patch("/variable-annotations", async (req) => { - const patchesList = req.body as GrapherConfigPatch[] - const variableIds = new Set(patchesList.map((patch) => patch.id)) +patchRouteWithRWTransaction( + apiRouter, + "/variable-annotations", + async (req, res, trx) => { + const patchesList = req.body as GrapherConfigPatch[] + const variableIds = new Set(patchesList.map((patch) => patch.id)) - await db.transaction(async (manager) => { - const configsAndIds = await manager.query( - `SELECT id, grapherConfigAdmin FROM variables where id IN (?)`, - [[...variableIds.values()]] - ) + const configsAndIds = await db.knexRaw< + Pick + >(trx, `SELECT id, grapherConfigAdmin FROM variables where id IN (?)`, [ + [...variableIds.values()], + ]) const configMap = new Map( configsAndIds.map((item: any) => [ item.id, @@ -1601,26 +1633,31 @@ apiRouter.patch("/variable-annotations", async (req) => { } for (const [variableId, newConfig] of configMap.entries()) { - await manager.execute( + await db.knexRaw( + trx, `UPDATE variables SET grapherConfigAdmin = ? where id = ?`, [JSON.stringify(newConfig), variableId] ) } - }) - return { success: true } -}) + return { success: true } + } +) -apiRouter.get("/variables.usages.json", async (req) => { - const query = `SELECT variableId, COUNT(DISTINCT chartId) AS usageCount +getRouteWithROTransaction( + apiRouter, + "/variables.usages.json", + async (req, res, trx) => { + const query = `SELECT variableId, COUNT(DISTINCT chartId) AS usageCount FROM chart_dimensions GROUP BY variableId ORDER BY usageCount DESC` - const rows = await db.queryMysql(query) + const rows = await db.knexRaw(trx, query) - return rows -}) + return rows + } +) // Used in VariableEditPage getRouteWithROTransaction( @@ -2064,12 +2101,19 @@ postRouteWithRWTransaction( ) // Get a list of redirects that map old slugs to charts -apiRouter.get("/redirects.json", async (req, res) => ({ - redirects: await db.queryMysql(` +getRouteWithROTransaction( + apiRouter, + "/redirects.json", + async (req, res, trx) => ({ + redirects: await db.knexRaw( + trx, + `-- sql SELECT r.id, r.slug, r.chart_id as chartId, JSON_UNQUOTE(JSON_EXTRACT(charts.config, "$.slug")) AS chartSlug FROM chart_slug_redirects AS r JOIN charts ON charts.id = r.chart_id - ORDER BY r.id DESC`), -})) + ORDER BY r.id DESC` + ), + }) +) getRouteWithROTransaction( apiRouter, @@ -2346,8 +2390,9 @@ deleteRouteWithRWTransaction( } ) -apiRouter.get("/posts.json", async (req) => { - const raw_rows = await db.queryMysql( +getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { + const raw_rows = await db.knexRaw( + trx, `-- sql with posts_tags_aggregated as ( select post_id, if(count(tags.id) = 0, json_array(), json_arrayagg(json_object("id", tags.id, "name", tags.name))) as tags diff --git a/adminSiteServer/authentication.tsx b/adminSiteServer/authentication.tsx index 285750176d1..31e6317bf41 100644 --- a/adminSiteServer/authentication.tsx +++ b/adminSiteServer/authentication.tsx @@ -112,9 +112,11 @@ export async function authCloudflareSSOMiddleware( export async function logOut(req: express.Request, res: express.Response) { if (res.locals.user) - await db.queryMysql(`DELETE FROM sessions WHERE session_key = ?`, [ - res.locals.session.id, - ]) + await db.knexReadWriteTransaction((trx) => + db.knexRaw(trx, `DELETE FROM sessions WHERE session_key = ?`, [ + res.locals.session.id, + ]) + ) res.clearCookie("sessionid") res.clearCookie(CLOUDFLARE_COOKIE_NAME) @@ -216,16 +218,18 @@ async function logInAsUser(user: DbPlainUser) { const now = new Date() const expiryDate = new Date(now.getTime() + 1000 * SESSION_COOKIE_AGE) - await db.execute( - `INSERT INTO sessions (session_key, session_data, expire_date) VALUES (?, ?, ?)`, - [sessionId, sessionData, expiryDate] - ) + await db.knexReadWriteTransaction(async (trx) => { + await db.knexRaw( + trx, + `INSERT INTO sessions (session_key, session_data, expire_date) VALUES (?, ?, ?)`, + [sessionId, sessionData, expiryDate] + ) - await db - .knexInstance() - .table("users") - .where({ id: user.id }) - .update({ lastLogin: now }) + await trx + .table("users") + .where({ id: user.id }) + .update({ lastLogin: now }) + }) return { id: sessionId, expiryDate: expiryDate } } diff --git a/adminSiteServer/gitDataExport.ts b/adminSiteServer/gitDataExport.ts index f14fcd6643d..cdb42c387f0 100644 --- a/adminSiteServer/gitDataExport.ts +++ b/adminSiteServer/gitDataExport.ts @@ -64,7 +64,6 @@ export async function syncDatasetToGitRepo( knex: db.KnexReadonlyTransaction, datasetId: number, options: { - transaction?: db.TransactionContext oldDatasetName?: string commitName?: string commitEmail?: string diff --git a/adminSiteServer/mockSiteRouter.tsx b/adminSiteServer/mockSiteRouter.tsx index c8ba6140331..e7302372d79 100644 --- a/adminSiteServer/mockSiteRouter.tsx +++ b/adminSiteServer/mockSiteRouter.tsx @@ -163,7 +163,9 @@ mockSiteRouter.get("/*", async (req, res, next) => { }) mockSiteRouter.get("/collection/top-charts", async (_, res) => { - return res.send(await renderTopChartsCollectionPage()) + await db.knexReadonlyTransaction(async (knex) => { + return res.send(await renderTopChartsCollectionPage(knex)) + }) }) mockSiteRouter.get("/collection/custom", async (_, res) => { @@ -254,7 +256,9 @@ mockSiteRouter.get("/data-insights/:pageNumberOrSlug?", async (req, res) => { mockSiteRouter.get("/charts", async (req, res) => { const explorerAdminServer = new ExplorerAdminServer(GIT_CMS_DIR) - res.send(await renderChartsPage(explorerAdminServer)) + await db.knexReadonlyTransaction(async (trx): Promise => { + res.send(await renderChartsPage(trx, explorerAdminServer)) + }) }) mockSiteRouter.get("/datapage-preview/:id", async (req, res) => { diff --git a/adminSiteServer/publicApiRouter.ts b/adminSiteServer/publicApiRouter.ts index 15a87b65d96..946c8859ced 100644 --- a/adminSiteServer/publicApiRouter.ts +++ b/adminSiteServer/publicApiRouter.ts @@ -9,9 +9,12 @@ function rejectAfterDelay(ms: number) { } publicApiRouter.router.get("/health", async (req: Request, res: Response) => { - const sqlPromise = db.mysqlFirst(`SELECT id FROM charts LIMIT 1`) - const timeoutPromise = rejectAfterDelay(1500) // Wait 1.5 seconds at most try { + const sqlPromise = db.knexRaw( + db.knexInstance() as db.KnexReadonlyTransaction, + `SELECT id FROM charts LIMIT 1` + ) + const timeoutPromise = rejectAfterDelay(1500) // Wait 1.5 seconds at most await Promise.race([sqlPromise, timeoutPromise]) res.status(200).end("OK") } catch (e) { diff --git a/adminSiteServer/testPageRouter.tsx b/adminSiteServer/testPageRouter.tsx index 855d3d55eb6..1e5752e56d7 100644 --- a/adminSiteServer/testPageRouter.tsx +++ b/adminSiteServer/testPageRouter.tsx @@ -619,25 +619,35 @@ function EmbedVariantsTestPage( } testPageRouter.get("/previews", async (req, res) => { - const rows = await db.queryMysql(`SELECT config FROM charts LIMIT 200`) - const charts = rows.map((row: any) => JSON.parse(row.config)) + await db.knexReadonlyTransaction(async (trx): Promise => { + const rows = await db.knexRaw( + trx, + `SELECT config FROM charts LIMIT 200` + ) + const charts = rows.map((row: any) => JSON.parse(row.config)) - res.send(renderToHtmlPage()) + res.send(renderToHtmlPage()) + }) }) testPageRouter.get("/embedVariants", async (req, res) => { - const rows = await db.queryMysql(`SELECT config FROM charts WHERE id=64`) - const charts = rows.map((row: any) => JSON.parse(row.config)) - const viewProps = getViewPropsFromQueryParams(req.query) - - res.send( - renderToHtmlPage( - + await db.knexReadonlyTransaction(async (trx): Promise => { + const rows = await db.knexRaw( + trx, + `SELECT config FROM charts WHERE id=64` ) - ) + const charts = rows.map((row: any) => JSON.parse(row.config)) + const viewProps = getViewPropsFromQueryParams(req.query) + + res.send( + renderToHtmlPage( + + ) + ) + }) }) testPageRouter.get("/:slug.svg", async (req, res) => { diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index e862a6e32cd..16f34127633 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -245,7 +245,7 @@ export async function renderDataPageV2( let slug = "" if (firstTopicTag) { try { - slug = await getSlugForTopicTag(firstTopicTag) + slug = await getSlugForTopicTag(knex, firstTopicTag) } catch (error) { await logErrorAndMaybeSendToBugsnag( `Datapage with variableId "${variableId}" and title "${datapageData.title.title}" is using "${firstTopicTag}" as its primary tag, which we are unable to resolve to a tag in the grapher DB` @@ -293,7 +293,7 @@ export async function renderDataPageV2( datapageData.relatedResearch = await getRelatedResearchAndWritingForVariable(knex, variableId) - const tagToSlugMap = await getTagToSlugMap() + const tagToSlugMap = await getTagToSlugMap(knex) return renderToHtmlPage( 0) { await bakeGraphersToSvgs( + knex, toBake, `${BAKED_SITE_DIR}/exports`, OPTIMIZE_SVG_EXPORTS @@ -141,12 +142,13 @@ export const getGrapherExportsByUrl = async (): Promise => { * "Women's Rights" -> "womens-rights" * 123 -> "womens-rights" */ -export async function getTagToSlugMap(): Promise< - Record -> { - const tags = (await db.queryMysql( +export async function getTagToSlugMap( + knex: db.KnexReadonlyTransaction +): Promise> { + const tags = await db.knexRaw>( + knex, `SELECT slug, name, id FROM tags WHERE slug IS NOT NULL` - )) as Pick[] + ) const tagsByIdAndName: Record = {} for (const tag of tags) { if (tag.slug) { @@ -163,10 +165,11 @@ export async function getTagToSlugMap(): Promise< * Throws an error if no slug is found so we can log it in Bugsnag */ export async function getSlugForTopicTag( + knex: db.KnexReadonlyTransaction, identifier: string | number ): Promise { const propertyToMatch = typeof identifier === "string" ? "slug" : "id" - const tagsByIdAndName = await getTagToSlugMap() + const tagsByIdAndName = await getTagToSlugMap(knex) const slug = tagsByIdAndName[identifier] if (!slug) { diff --git a/baker/GrapherImageBaker.tsx b/baker/GrapherImageBaker.tsx index 03cc579d3c8..a8232510b88 100644 --- a/baker/GrapherImageBaker.tsx +++ b/baker/GrapherImageBaker.tsx @@ -1,4 +1,8 @@ -import { GrapherInterface } from "@ourworldindata/types" +import { + DbPlainChartSlugRedirect, + DbRawChart, + GrapherInterface, +} from "@ourworldindata/types" import { Grapher, GrapherProgrammaticInterface } from "@ourworldindata/grapher" import { MultipleOwidVariableDataDimensionsMap } from "@ourworldindata/utils" import fs from "fs-extra" @@ -48,14 +52,17 @@ export async function bakeGraphersToPngs( ]) } -export async function getGraphersAndRedirectsBySlug() { - const { graphersBySlug, graphersById } = await getPublishedGraphersBySlug() +export async function getGraphersAndRedirectsBySlug( + knex: db.KnexReadonlyTransaction +) { + const { graphersBySlug, graphersById } = + await getPublishedGraphersBySlug(knex) - const redirectQuery = db.queryMysql( - `SELECT slug, chart_id FROM chart_slug_redirects` - ) + const redirectQuery = await db.knexRaw< + Pick + >(knex, `SELECT slug, chart_id FROM chart_slug_redirects`) - for (const row of await redirectQuery) { + for (const row of redirectQuery) { const grapher = graphersById.get(row.chart_id) if (grapher) { graphersBySlug.set(row.slug, grapher) @@ -65,14 +72,16 @@ export async function getGraphersAndRedirectsBySlug() { return graphersBySlug } -export async function getPublishedGraphersBySlug() { +export async function getPublishedGraphersBySlug( + knex: db.KnexReadonlyTransaction +) { const graphersBySlug: Map = new Map() const graphersById: Map = new Map() // Select all graphers that are published const sql = `SELECT id, config FROM charts WHERE config->>"$.isPublished" = "true"` - const query = db.queryMysql(sql) + const query = db.knexRaw>(knex, sql) for (const row of await query) { const grapher = JSON.parse(row.config) @@ -160,12 +169,13 @@ export function buildSvgOutFilepath( } export async function bakeGraphersToSvgs( + knex: db.KnexReadonlyTransaction, grapherUrls: string[], outDir: string, optimizeSvgs = false ) { await fs.mkdirp(outDir) - const graphersBySlug = await getGraphersAndRedirectsBySlug() + const graphersBySlug = await getGraphersAndRedirectsBySlug(knex) return pMap( grapherUrls, diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 87f211b3044..09f9eeebfc8 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -431,8 +431,6 @@ export class SiteBaker { private async removeDeletedPosts(knex: db.KnexReadonlyTransaction) { if (!this.bakeSteps.has("removeDeletedPosts")) return - await db.getConnection() - const postsApi = await getPostsFromSnapshots(knex) const postSlugs = [] @@ -574,7 +572,7 @@ export class SiteBaker { ) await this.stageWrite( `${this.bakedSiteDir}/collection/top-charts.html`, - await renderTopChartsCollectionPage() + await renderTopChartsCollectionPage(knex) ) await this.stageWrite( `${this.bakedSiteDir}/404.html`, @@ -592,7 +590,7 @@ export class SiteBaker { await this.stageWrite( `${this.bakedSiteDir}/charts.html`, - await renderChartsPage(this.explorerAdminServer) + await renderChartsPage(knex, this.explorerAdminServer) ) this.progressBar.tick({ name: "✅ baked special pages" }) } @@ -947,7 +945,7 @@ export class SiteBaker { redirects.join("\n") ) - const grapherRedirects = await getGrapherRedirectsMap("") + const grapherRedirects = await getGrapherRedirectsMap(knex, "") await this.stageWrite( path.join(this.bakedSiteDir, `grapher/_grapherRedirects.json`), JSON.stringify(Object.fromEntries(grapherRedirects), null, 2) @@ -991,7 +989,6 @@ export class SiteBaker { } async bakeNonWordpressPages(knex: db.KnexReadonlyTransaction) { - await db.getConnection() const progressBarTotal = nonWordpressSteps .map((step) => this.bakeSteps.has(step)) .filter((hasStep) => hasStep).length diff --git a/baker/algolia/indexChartsToAlgolia.ts b/baker/algolia/indexChartsToAlgolia.ts index 183421f3c2f..cd96d1e5f76 100644 --- a/baker/algolia/indexChartsToAlgolia.ts +++ b/baker/algolia/indexChartsToAlgolia.ts @@ -131,7 +131,6 @@ const indexChartsToAlgolia = async () => { const index = client.initIndex(SearchIndexName.Charts) - await db.getConnection() const records = await db.knexReadonlyTransaction(getChartsRecords) await index.replaceAllObjects(records) diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index 4acf0a2526a..40108dfe70d 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -233,7 +233,6 @@ const indexToAlgolia = async () => { } const index = client.initIndex(SearchIndexName.Pages) - await db.getConnection() const records = await db.knexReadonlyTransaction(getPagesRecords) await index.replaceAllObjects(records) diff --git a/baker/bakeGdocPost.ts b/baker/bakeGdocPost.ts index 108d116f7c4..3defc7afbf3 100644 --- a/baker/bakeGdocPost.ts +++ b/baker/bakeGdocPost.ts @@ -19,7 +19,6 @@ void yargs(hideBin(process.argv)) async ({ slug }) => { const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) - await db.getConnection() await db.knexReadonlyTransaction((trx) => baker.bakeGDocPosts(trx, [slug]) ) diff --git a/baker/bakeGdocPosts.ts b/baker/bakeGdocPosts.ts index 33e822fa0a9..64de00aa80e 100644 --- a/baker/bakeGdocPosts.ts +++ b/baker/bakeGdocPosts.ts @@ -24,7 +24,6 @@ void yargs(hideBin(process.argv)) async ({ slugs }) => { const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) - await db.getConnection() await db.knexReadonlyTransaction((trx) => baker.bakeGDocPosts(trx, slugs) ) diff --git a/baker/redirects.ts b/baker/redirects.ts index d55be1acc6f..c48f3833ba2 100644 --- a/baker/redirects.ts +++ b/baker/redirects.ts @@ -74,12 +74,19 @@ export const getRedirects = async (knex: db.KnexReadonlyTransaction) => { } export const getGrapherRedirectsMap = async ( + knex: db.KnexReadonlyTransaction, urlPrefix: string = "/grapher/" ) => { - const chartRedirectRows = (await db.queryMysql(`-- sql + const chartRedirectRows = (await db.knexRaw<{ + oldSlug: string + newSlug: string + }>( + knex, + `-- sql SELECT chart_slug_redirects.slug as oldSlug, charts.config ->> "$.slug" as newSlug FROM chart_slug_redirects INNER JOIN charts ON charts.id=chart_id - `)) as Array<{ oldSlug: string; newSlug: string }> + ` + )) as Array<{ oldSlug: string; newSlug: string }> return new Map( chartRedirectRows @@ -104,7 +111,7 @@ export const getGrapherAndWordpressRedirectsMap = memoize( // source: pathnames only (e.g. /transport) // target: pathnames with or without origins (e.g. /transport-new or https://ourworldindata.org/transport-new) - const grapherRedirects = await getGrapherRedirectsMap() + const grapherRedirects = await getGrapherRedirectsMap(knex) const wordpressRedirects = await getWordpressRedirectsMap(knex) // The order the redirects are added to the map is important. Adding the diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index 0d1ee4aa1ea..8b7109ce8e8 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -58,7 +58,7 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { queryMysql, getHomepageId, KnexReadonlyTransaction } from "../db/db.js" +import { getHomepageId, knexRaw, KnexReadonlyTransaction } from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" import { @@ -101,11 +101,14 @@ export const renderToHtmlPage = (element: any) => `${ReactDOMServer.renderToStaticMarkup(element)}` export const renderChartsPage = async ( + knex: KnexReadonlyTransaction, explorerAdminServer: ExplorerAdminServer ) => { const explorers = await explorerAdminServer.getAllPublishedExplorers() - const chartItems = (await queryMysql(` + const chartItems = await knexRaw( + knex, + `-- sql SELECT id, config->>"$.slug" AS slug, @@ -116,13 +119,22 @@ export const renderChartsPage = async ( is_indexable IS TRUE AND publishedAt IS NOT NULL AND config->>"$.isPublished" = "true" - `)) as ChartIndexItem[] + ` + ) - const chartTags = await queryMysql(` + const chartTags = await knexRaw<{ + chartId: number + tagId: number + tagName: string + tagParentId: number + }>( + knex, + `-- sql SELECT ct.chartId, ct.tagId, t.name as tagName, t.parentId as tagParentId FROM chart_tags ct JOIN charts c ON c.id=ct.chartId JOIN tags t ON t.id=ct.tagId - `) + ` + ) for (const c of chartItems) { c.tags = [] @@ -144,9 +156,12 @@ export const renderChartsPage = async ( ) } -export async function renderTopChartsCollectionPage() { - const charts: string[] = await queryMysql( - ` +export async function renderTopChartsCollectionPage( + knex: KnexReadonlyTransaction +) { + const charts: string[] = await knexRaw<{ slug: string }>( + knex, + `-- sql SELECT SUBSTRING_INDEX(url, '/', -1) AS slug FROM analytics_pageviews WHERE url LIKE "%https://ourworldindata.org/grapher/%" @@ -642,7 +657,8 @@ export const renderExplorerPage = async ( type ChartRow = { id: number; config: string } let grapherConfigRows: ChartRow[] = [] if (requiredGrapherIds.length) - grapherConfigRows = await queryMysql( + grapherConfigRows = await knexRaw( + knex, `SELECT id, config FROM charts WHERE id IN (?)`, [requiredGrapherIds] ) @@ -653,7 +669,8 @@ export const renderExplorerPage = async ( grapherConfigETL: string | null }[] = [] if (requiredVariableIds.length) { - partialGrapherConfigRows = await queryMysql( + partialGrapherConfigRows = await knexRaw( + knex, `SELECT id, grapherConfigETL, grapherConfigAdmin FROM variables WHERE id IN (?)`, [requiredVariableIds] ) diff --git a/baker/startDeployQueueServer.ts b/baker/startDeployQueueServer.ts index 6b6a87c902e..775f204ab4b 100644 --- a/baker/startDeployQueueServer.ts +++ b/baker/startDeployQueueServer.ts @@ -26,8 +26,6 @@ const main = async () => { }) } - await db.getConnection() - // Listen for file changes fs.watchFile(DEPLOY_QUEUE_FILE_PATH, () => { // Start deploy after 10 seconds in order to avoid the quick successive diff --git a/baker/syncRedirectsToGrapher.ts b/baker/syncRedirectsToGrapher.ts index 1eea3449816..f27e97bb5ea 100644 --- a/baker/syncRedirectsToGrapher.ts +++ b/baker/syncRedirectsToGrapher.ts @@ -67,7 +67,6 @@ export const syncRedirectsToGrapher = async ( const main = async (): Promise => { try { - await db.getConnection() await db.knexReadWriteTransaction((trx) => syncRedirectsToGrapher(trx)) } finally { await wpdb.singleton.end() diff --git a/db/analyzeWpPosts.ts b/db/analyzeWpPosts.ts index eb0949b3d60..6a33348476b 100644 --- a/db/analyzeWpPosts.ts +++ b/db/analyzeWpPosts.ts @@ -23,46 +23,49 @@ export function traverseNode( } const analyze = async (): Promise => { - await db.getConnection() - - const posts: { id: number; content: string }[] = await db.queryMysql(` + await db.knexReadonlyTransaction(async (trx): Promise => { + const posts: { id: number; content: string }[] = await db.knexRaw( + trx, + ` SELECT id, content from posts where type<>'wp_block' - `) + ` + ) - const tagCounts = new Map() - const decideFilter = (node: CheerioElement): boolean => - node.type === "tag" && node.tagName === "iframe" + const tagCounts = new Map() + const decideFilter = (node: CheerioElement): boolean => + node.type === "tag" && node.tagName === "iframe" - for (const post of posts) { - // temp workaround for load with 3 params not showing up in TS type - const $: CheerioStatic = cheerio.load(post.content) - $("body").each((i, node) => { - traverseNode( - node, - 1, - false, - decideFilter, - (elem, depth, isFilterActive) => { - if (isFilterActive) { - const tagName = - elem.type !== "tag" - ? `${elem.type} - depth ${depth}` - : elem.tagName - const currentCount = tagCounts.get(tagName) ?? 0 - tagCounts.set(tagName, currentCount + 1) + for (const post of posts) { + // temp workaround for load with 3 params not showing up in TS type + const $: CheerioStatic = cheerio.load(post.content) + $("body").each((i, node) => { + traverseNode( + node, + 1, + false, + decideFilter, + (elem, depth, isFilterActive) => { + if (isFilterActive) { + const tagName = + elem.type !== "tag" + ? `${elem.type} - depth ${depth}` + : elem.tagName + const currentCount = tagCounts.get(tagName) ?? 0 + tagCounts.set(tagName, currentCount + 1) + } } - } - ) - }) - } + ) + }) + } - const sortedTagCount = _.sortBy( - Array.from(tagCounts.entries()), - ([tag, _]) => tag - ) - for (const [tag, count] of sortedTagCount) { - console.log(`${tag}: ${count}`) - } + const sortedTagCount = _.sortBy( + Array.from(tagCounts.entries()), + ([tag, _]) => tag + ) + for (const [tag, count] of sortedTagCount) { + console.log(`${tag}: ${count}`) + } + }) await db.closeTypeOrmAndKnexConnections() } diff --git a/db/db.ts b/db/db.ts index cb28560df5e..dc06fdd50a8 100644 --- a/db/db.ts +++ b/db/db.ts @@ -66,9 +66,6 @@ export const queryMysql = async ( return conn.query(params ? mysql.format(queryStr, params) : queryStr) } -// For operations that modify data (TODO: handling to check query isn't used for this) -export const execute = queryMysql - // Return the first match from a mysql query export const mysqlFirst = async ( queryStr: string, diff --git a/db/exportMetadata.ts b/db/exportMetadata.ts index 12ea97d8953..3b7cd2958f2 100644 --- a/db/exportMetadata.ts +++ b/db/exportMetadata.ts @@ -25,8 +25,6 @@ const filePath = const excludeTables = ["sessions", "dataset_files", "analytics_pageviews"] async function dataExport(): Promise { - await db.getConnection() - console.log(`Exporting database structure and metadata to ${filePath}...`) // Expose password to mysqldump diff --git a/db/model/Post.ts b/db/model/Post.ts index bd21c3ff60e..693b888584e 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -264,7 +264,6 @@ const selectHomepagePosts: FilterFnPostRestApi = (post) => export const getBlogIndex = memoize( async (knex: db.KnexReadonlyTransaction): Promise => { - await db.getConnection() // side effect: ensure connection is established const gdocPosts = await GdocPost.getListedGdocPosts(knex) const wpPosts = await Promise.all( await getPostsFromSnapshots( diff --git a/db/syncPostsToGrapher.ts b/db/syncPostsToGrapher.ts index b59b28c8f2c..2ae32e68c6d 100644 --- a/db/syncPostsToGrapher.ts +++ b/db/syncPostsToGrapher.ts @@ -435,7 +435,6 @@ const syncPostsToGrapher = async ( const main = async (): Promise => { try { - await db.getConnection() await db.knexReadWriteTransaction((trx) => syncPostsToGrapher(trx)) } finally { await wpdb.singleton.end() diff --git a/db/tests/basic.test.ts b/db/tests/basic.test.ts index 7639b253bbe..06461be7291 100644 --- a/db/tests/basic.test.ts +++ b/db/tests/basic.test.ts @@ -1,10 +1,8 @@ #! /usr/bin/env jest import sqlFixtures from "sql-fixtures" import { dbTestConfig } from "./dbTestConfig.js" -import { dataSource } from "./dataSource.dbtests.js" import { knex, Knex } from "knex" import { - getConnection, knexRaw, knexReadWriteTransaction, KnexReadonlyTransaction, @@ -12,7 +10,6 @@ import { knexRawFirst, knexReadonlyTransaction, } from "../db.js" -import { DataSource } from "typeorm" import { deleteUser, insertUser, updateUser } from "../model/User.js" import { ChartsTableName, @@ -23,7 +20,6 @@ import { } from "@ourworldindata/types" let knexInstance: Knex | undefined = undefined -let typeOrmConnection: DataSource | undefined = undefined beforeAll(async () => { const dataSpec = { @@ -45,16 +41,12 @@ beforeAll(async () => { // In case you want to see results of fixture creation you can do it like below // console.log(_.users[0].email) }) - typeOrmConnection = await getConnection(dataSource) }) afterAll((done: any) => { // We leave the user in the database for other tests to use // For other cases it is good to drop any rows created in the test - void Promise.allSettled([ - typeOrmConnection?.destroy(), - knexInstance?.destroy(), - ]).then(() => done()) + void Promise.allSettled([knexInstance?.destroy()]).then(() => done()) }) test("it can query a user created in fixture via TypeORM", async () => { diff --git a/devTools/markdownTest/markdown.ts b/devTools/markdownTest/markdown.ts index ba3658f8f13..95f5a1f1ac2 100644 --- a/devTools/markdownTest/markdown.ts +++ b/devTools/markdownTest/markdown.ts @@ -1,12 +1,9 @@ import { closeTypeOrmAndKnexConnections, - getConnection, knexReadonlyTransaction, } from "../../db/db.js" import { getPostRawBySlug } from "../../db/model/Post.js" import { enrichedBlocksToMarkdown } from "../../db/model/Gdoc/enrichedToMarkdown.js" -import { GdocBase } from "../../db/model/Gdoc/GdocBase.js" -import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" import fs from "fs-extra" diff --git a/devTools/svgTester/dump-data.ts b/devTools/svgTester/dump-data.ts index 37375152ce1..069b014d2ba 100644 --- a/devTools/svgTester/dump-data.ts +++ b/devTools/svgTester/dump-data.ts @@ -2,7 +2,10 @@ import { getPublishedGraphersBySlug } from "../../baker/GrapherImageBaker.js" -import { closeTypeOrmAndKnexConnections } from "../../db/db.js" +import { + closeTypeOrmAndKnexConnections, + knexReadonlyTransaction, +} from "../../db/db.js" import fs from "fs-extra" @@ -15,7 +18,11 @@ async function main(parsedArgs: parseArgs.ParsedArgs) { const outDir = parsedArgs["o"] ?? utils.DEFAULT_CONFIGS_DIR if (!fs.existsSync(outDir)) fs.mkdirSync(outDir) - const { graphersBySlug } = await getPublishedGraphersBySlug() + const { graphersBySlug } = await knexReadonlyTransaction( + async (trx) => { + return getPublishedGraphersBySlug(trx) + } + ) const allGraphers = [...graphersBySlug.values()] const saveJobs: utils.SaveGrapherSchemaAndDataJob[] = allGraphers.map( (grapher) => ({ config: grapher, outDir })