Skip to content

Commit

Permalink
🔨 switch almost all remaining typeorm util function holdouts to knex
Browse files Browse the repository at this point in the history 
The only remaining uses of the typeorm functions are related to the chart revision tool
  • Loading branch information
@danyx23
danyx23 committed Mar 16, 2024
1 parent 5a640e2 commit 37780f9
Show file tree
Hide file tree
Showing 26 changed files with 277 additions and 194 deletions.
169 changes: 107 additions & 62 deletions adminSiteServer/apiRouter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,8 @@ const getReferencesByChartId = async (
knex
)
const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex)
const explorerSlugsPromise = db.queryMysql(
const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>(
knex,
`select distinct explorerSlug from explorer_charts where chartId = ?`,
[chartId]
)
Expand Down Expand Up @@ -426,12 +427,13 @@ getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => {
return { charts }
})

apiRouter.get("/charts.csv", async (req, res) => {
getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => {
const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000

// note: this query is extended from OldChart.listFields.
const charts = await db.queryMysql(
`
const charts = await db.knexRaw(
trx,
`-- sql
SELECT
charts.id,
charts.config->>"$.version" AS version,
Expand Down Expand Up @@ -486,25 +488,34 @@ getRouteWithROTransaction(
async (req, res, trx) => expectChartById(trx, req.params.chartId)
)

apiRouter.get("/editorData/namespaces.json", async (req, res) => {
const rows = (await db.queryMysql(
`SELECT DISTINCT
getRouteWithROTransaction(
apiRouter,
"/editorData/namespaces.json",
async (req, res, trx) => {
const rows = await db.knexRaw<{
name: string
description?: string
isArchived: boolean
}>(
trx,
`SELECT DISTINCT
namespace AS name,
namespaces.description AS description,
namespaces.isArchived AS isArchived
FROM active_datasets
JOIN namespaces ON namespaces.name = active_datasets.namespace`
)) as { name: string; description?: string; isArchived: boolean }[]
)

return {
namespaces: lodash
.sortBy(rows, (row) => row.description)
.map((namespace) => ({
...namespace,
isArchived: !!namespace.isArchived,
})),
return {
namespaces: lodash
.sortBy(rows, (row) => row.description)
.map((namespace) => ({
...namespace,
isArchived: !!namespace.isArchived,
})),
}
}
})
)

getRouteWithROTransaction(
apiRouter,
Expand Down Expand Up @@ -1375,17 +1386,19 @@ getRouteWithROTransaction(
}
)

apiRouter.delete("/users/:userId", async (req, res) => {
if (!res.locals.user.isSuperuser)
throw new JsonError("Permission denied", 403)
deleteRouteWithRWTransaction(
apiRouter,
"/users/:userId",
async (req, res, trx) => {
if (!res.locals.user.isSuperuser)
throw new JsonError("Permission denied", 403)

const userId = expectInt(req.params.userId)
await db.transaction(async (t) => {
await t.execute(`DELETE FROM users WHERE id=?`, [userId])
})
const userId = expectInt(req.params.userId)
await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId])

return { success: true }
})
return { success: true }
}
)

putRouteWithRWTransaction(
apiRouter,
Expand Down Expand Up @@ -1436,10 +1449,13 @@ getRouteWithROTransaction(
}
)

apiRouter.get(
getRouteWithROTransaction(
apiRouter,
"/chart-bulk-update",
async (
req
req,
res,
trx
): Promise<BulkGrapherConfigResponse<BulkChartEditResponseRow>> => {
const context: OperationContext = {
grapherConfigFieldName: "config",
Expand All @@ -1458,8 +1474,9 @@ apiRouter.get(
// careful there to only allow carefully guarded vocabularies from being used, not
// arbitrary user input
const whereClause = filterSExpr?.toSql() ?? "true"
const resultsWithStringGrapherConfigs =
await db.queryMysql(`SELECT charts.id as id,
const resultsWithStringGrapherConfigs = await db.knexRaw(
trx,
`SELECT charts.id as id,
charts.config as config,
charts.createdAt as createdAt,
charts.updatedAt as updatedAt,
Expand All @@ -1473,15 +1490,19 @@ LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId
WHERE ${whereClause}
ORDER BY charts.id DESC
LIMIT 50
OFFSET ${offset.toString()}`)
OFFSET ${offset.toString()}`
)

const results = resultsWithStringGrapherConfigs.map((row: any) => ({
...row,
config: lodash.isNil(row.config) ? null : JSON.parse(row.config),
}))
const resultCount = await db.queryMysql(`SELECT count(*) as count
const resultCount = await db.knexRaw<{ count: number }>(
trx,
`SELECT count(*) as count
FROM charts
WHERE ${whereClause}`)
WHERE ${whereClause}`
)
return { rows: results, numTotalRows: resultCount[0].count }
}
)
Expand Down Expand Up @@ -1527,10 +1548,13 @@ patchRouteWithRWTransaction(
}
)

apiRouter.get(
getRouteWithROTransaction(
apiRouter,
"/variable-annotations",
async (
req
req,
res,
trx
): Promise<BulkGrapherConfigResponse<VariableAnnotationsResponseRow>> => {
const context: OperationContext = {
grapherConfigFieldName: "grapherConfigAdmin",
Expand All @@ -1549,8 +1573,9 @@ apiRouter.get(
// careful there to only allow carefully guarded vocabularies from being used, not
// arbitrary user input
const whereClause = filterSExpr?.toSql() ?? "true"
const resultsWithStringGrapherConfigs =
await db.queryMysql(`SELECT variables.id as id,
const resultsWithStringGrapherConfigs = await db.knexRaw(
trx,
`SELECT variables.id as id,
variables.name as name,
variables.grapherConfigAdmin as config,
d.name as datasetname,
Expand All @@ -1564,30 +1589,37 @@ LEFT JOIN namespaces on d.namespace = namespaces.name
WHERE ${whereClause}
ORDER BY variables.id DESC
LIMIT 50
OFFSET ${offset.toString()}`)
OFFSET ${offset.toString()}`
)

const results = resultsWithStringGrapherConfigs.map((row: any) => ({
...row,
config: lodash.isNil(row.config) ? null : JSON.parse(row.config),
}))
const resultCount = await db.queryMysql(`SELECT count(*) as count
const resultCount = await db.knexRaw<{ count: number }>(
trx,
`SELECT count(*) as count
FROM variables
LEFT JOIN active_datasets as d on variables.datasetId = d.id
LEFT JOIN namespaces on d.namespace = namespaces.name
WHERE ${whereClause}`)
WHERE ${whereClause}`
)
return { rows: results, numTotalRows: resultCount[0].count }
}
)

apiRouter.patch("/variable-annotations", async (req) => {
const patchesList = req.body as GrapherConfigPatch[]
const variableIds = new Set(patchesList.map((patch) => patch.id))
patchRouteWithRWTransaction(
apiRouter,
"/variable-annotations",
async (req, res, trx) => {
const patchesList = req.body as GrapherConfigPatch[]
const variableIds = new Set(patchesList.map((patch) => patch.id))

await db.transaction(async (manager) => {
const configsAndIds = await manager.query(
`SELECT id, grapherConfigAdmin FROM variables where id IN (?)`,
[[...variableIds.values()]]
)
const configsAndIds = await db.knexRaw<
Pick<DbRawVariable, "id" | "grapherConfigAdmin">
>(trx, `SELECT id, grapherConfigAdmin FROM variables where id IN (?)`, [
[...variableIds.values()],
])
const configMap = new Map(
configsAndIds.map((item: any) => [
item.id,
Expand All @@ -1601,26 +1633,31 @@ apiRouter.patch("/variable-annotations", async (req) => {
}

for (const [variableId, newConfig] of configMap.entries()) {
await manager.execute(
await db.knexRaw(
trx,
`UPDATE variables SET grapherConfigAdmin = ? where id = ?`,
[JSON.stringify(newConfig), variableId]
)
}
})

return { success: true }
})
return { success: true }
}
)

apiRouter.get("/variables.usages.json", async (req) => {
const query = `SELECT variableId, COUNT(DISTINCT chartId) AS usageCount
getRouteWithROTransaction(
apiRouter,
"/variables.usages.json",
async (req, res, trx) => {
const query = `SELECT variableId, COUNT(DISTINCT chartId) AS usageCount
FROM chart_dimensions
GROUP BY variableId
ORDER BY usageCount DESC`

const rows = await db.queryMysql(query)
const rows = await db.knexRaw(trx, query)

return rows
})
return rows
}
)

// Used in VariableEditPage
getRouteWithROTransaction(
Expand Down Expand Up @@ -2064,12 +2101,19 @@ postRouteWithRWTransaction(
)

// Get a list of redirects that map old slugs to charts
apiRouter.get("/redirects.json", async (req, res) => ({
redirects: await db.queryMysql(`
getRouteWithROTransaction(
apiRouter,
"/redirects.json",
async (req, res, trx) => ({
redirects: await db.knexRaw(
trx,
`-- sql
SELECT r.id, r.slug, r.chart_id as chartId, JSON_UNQUOTE(JSON_EXTRACT(charts.config, "$.slug")) AS chartSlug
FROM chart_slug_redirects AS r JOIN charts ON charts.id = r.chart_id
ORDER BY r.id DESC`),
}))
ORDER BY r.id DESC`
),
})
)

getRouteWithROTransaction(
apiRouter,
Expand Down Expand Up @@ -2346,8 +2390,9 @@ deleteRouteWithRWTransaction(
}
)

apiRouter.get("/posts.json", async (req) => {
const raw_rows = await db.queryMysql(
getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => {
const raw_rows = await db.knexRaw(
trx,
`-- sql
with posts_tags_aggregated as (
select post_id, if(count(tags.id) = 0, json_array(), json_arrayagg(json_object("id", tags.id, "name", tags.name))) as tags
Expand Down
28 changes: 16 additions & 12 deletions adminSiteServer/authentication.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,11 @@ export async function authCloudflareSSOMiddleware(

export async function logOut(req: express.Request, res: express.Response) {
if (res.locals.user)
await db.queryMysql(`DELETE FROM sessions WHERE session_key = ?`, [
res.locals.session.id,
])
await db.knexReadWriteTransaction((trx) =>
db.knexRaw(trx, `DELETE FROM sessions WHERE session_key = ?`, [
res.locals.session.id,
])
)

res.clearCookie("sessionid")
res.clearCookie(CLOUDFLARE_COOKIE_NAME)
Expand Down Expand Up @@ -216,16 +218,18 @@ async function logInAsUser(user: DbPlainUser) {
const now = new Date()
const expiryDate = new Date(now.getTime() + 1000 * SESSION_COOKIE_AGE)

await db.execute(
`INSERT INTO sessions (session_key, session_data, expire_date) VALUES (?, ?, ?)`,
[sessionId, sessionData, expiryDate]
)
await db.knexReadWriteTransaction(async (trx) => {
await db.knexRaw(
trx,
`INSERT INTO sessions (session_key, session_data, expire_date) VALUES (?, ?, ?)`,
[sessionId, sessionData, expiryDate]
)

await db
.knexInstance()
.table("users")
.where({ id: user.id })
.update({ lastLogin: now })
await trx
.table("users")
.where({ id: user.id })
.update({ lastLogin: now })
})

return { id: sessionId, expiryDate: expiryDate }
}
Expand Down
1 change: 0 additions & 1 deletion adminSiteServer/gitDataExport.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@ export async function syncDatasetToGitRepo(
knex: db.KnexReadonlyTransaction,
datasetId: number,
options: {
transaction?: db.TransactionContext
oldDatasetName?: string
commitName?: string
commitEmail?: string
Expand Down
8 changes: 6 additions & 2 deletions adminSiteServer/mockSiteRouter.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,9 @@ mockSiteRouter.get("/*", async (req, res, next) => {
})

mockSiteRouter.get("/collection/top-charts", async (_, res) => {
return res.send(await renderTopChartsCollectionPage())
await db.knexReadonlyTransaction(async (knex) => {
return res.send(await renderTopChartsCollectionPage(knex))
})
})

mockSiteRouter.get("/collection/custom", async (_, res) => {
Expand Down Expand Up @@ -254,7 +256,9 @@ mockSiteRouter.get("/data-insights/:pageNumberOrSlug?", async (req, res) => {

mockSiteRouter.get("/charts", async (req, res) => {
const explorerAdminServer = new ExplorerAdminServer(GIT_CMS_DIR)
res.send(await renderChartsPage(explorerAdminServer))
await db.knexReadonlyTransaction(async (trx): Promise<void> => {
res.send(await renderChartsPage(trx, explorerAdminServer))
})
})

mockSiteRouter.get("/datapage-preview/:id", async (req, res) => {
Expand Down
7 changes: 5 additions & 2 deletions adminSiteServer/publicApiRouter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@ function rejectAfterDelay(ms: number) {
}

publicApiRouter.router.get("/health", async (req: Request, res: Response) => {
const sqlPromise = db.mysqlFirst(`SELECT id FROM charts LIMIT 1`)
const timeoutPromise = rejectAfterDelay(1500) // Wait 1.5 seconds at most
try {
const sqlPromise = db.knexRaw(
db.knexInstance() as db.KnexReadonlyTransaction,
`SELECT id FROM charts LIMIT 1`
)
const timeoutPromise = rejectAfterDelay(1500) // Wait 1.5 seconds at most
await Promise.race([sqlPromise, timeoutPromise])
res.status(200).end("OK")
} catch (e) {
Expand Down
Loading

0 comments on commit 37780f9

Please sign in to comment.