refactor(db): use mysql2 as a db driver

db/DatabaseConnection.ts

@@ -1,8 +1,8 @@
-import mysql from "mysql"
+import mysql2 from "mysql2"
 
 class TransactionContext {
-    conn: mysql.PoolConnection
-    constructor(conn: mysql.PoolConnection) {
+    conn: mysql2.PoolConnection
+    constructor(conn: mysql2.PoolConnection) {
         this.conn = conn
     }
 
@@ -27,19 +27,19 @@ class TransactionContext {
 
 // Promise wrapper for node-mysql with transaction support and some shorthands
 export class DatabaseConnection {
-    config: mysql.PoolConfig
-    pool!: mysql.Pool
+    config: mysql2.PoolOptions
+    pool!: mysql2.Pool
 
-    constructor(config: mysql.PoolConfig) {
+    constructor(config: mysql2.PoolOptions) {
         this.config = config
     }
 
     async connect(): Promise<void> {
-        this.pool = mysql.createPool(this.config)
+        this.pool = mysql2.createPool(this.config)
         await this.getConnection()
     }
 
-    getConnection(): Promise<mysql.PoolConnection> {
+    getConnection(): Promise<mysql2.PoolConnection> {
         return new Promise((resolve, reject) => {
             this.pool.getConnection((poolerr, conn) => {
                 if (poolerr) {

db/dataSource.ts

@@ -17,4 +17,10 @@ export const dataSource = new DataSource({
     entities: ["db/model/**/!(*.test).ts"],
     migrations: ["db/migration/**/!(*.test).ts"],
     charset: "utf8mb4",
+    connectorPackage: "mysql2",
+    extra: {
+        // Instruct the mysql2 driver to not convert json columns to a JS object implicitly; many
+        // of our migrations rely on the raw JSON string
+        jsonStrings: true,
+    },
 })

db/db.ts

@@ -35,7 +35,7 @@ export const knexInstance = (): Knex<any, any[]> => {
     if (_knexInstance) return _knexInstance
 
     _knexInstance = knex({
-        client: "mysql",
+        client: "mysql2",
         connection: {
             host: GRAPHER_DB_HOST,
             user: GRAPHER_DB_USER,
@@ -47,8 +47,14 @@ export const knexInstance = (): Knex<any, any[]> => {
                 if (field.type === "TINY" && field.length === 1) {
                     return field.string() === "1" // 1 = true, 0 = false
                 }
+
                 return next()
             },
+
+            // The mysql2 driver will return JSON objects by default, which is nice, but we have many code paths that
+            // expect JSON strings, so we instead tell it to return JSON strings.
+            //@ts-expect-error This is an option in mysql2 v3.10+, but it's not yet reflected in the knex types
+            jsonStrings: true,
         },
     })
 

db/model/Variable.ts

@@ -8,7 +8,7 @@ import {
 } from "@ourworldindata/grapher"
 import pl from "nodejs-polars"
 import { DATA_API_URL } from "../../settings/serverSettings.js"
-import { escape } from "mysql"
+import { escape } from "mysql2"
 import {
     OwidChartDimensionInterface,
     OwidVariableDisplayConfigInterface,

db/sql-ts/sql-ts-config.json

@@ -1,5 +1,5 @@
 {
-    "client": "mysql",
+    "client": "mysql2",
     "connection": {
         "host": "localhost",
         "user": "grapher",

db/tests/dbTestConfig.ts

@@ -9,7 +9,7 @@ import {
 } from "../../settings/serverSettings.js"
 
 export const dbTestConfig = {
-    client: "mysql",
+    client: "mysql2",
     connection: {
         database: GRAPHER_TEST_DB_NAME,
         user: GRAPHER_TEST_DB_USER,

knexfile.ts

@@ -9,7 +9,7 @@ import {
 } from "../settings/serverSettings.js"
 
 const dbConfig = {
-    client: "mysql",
+    client: "mysql2",
     connection: {
         database: GRAPHER_DB_NAME,
         user: GRAPHER_DB_USER,

package.json

@@ -112,7 +112,7 @@
         "mobx-react": "5",
         "mobx-utils": "5",
         "mousetrap": "^1.6.5",
-        "mysql": "^2.18.1",
+        "mysql2": "^3.10.1",
         "nodejs-polars": "^0.7.2",
         "normalize.css": "^8.0.1",
         "openai": "^4.47.1",
@@ -190,7 +190,6 @@
         "@types/mdast": "^3.0",
         "@types/minimist": "^1.2.2",
         "@types/mousetrap": "^1.6.9",
-        "@types/mysql": "^2.15.21",
         "@types/papaparse": "^5.3.5",
         "@types/progress": "^2.0.5",
         "@types/randomstring": "^1.1.8",

yarn.lock

@@ -5161,15 +5161,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/mysql@npm:^2.15.21":
-  version: 2.15.21
-  resolution: "@types/mysql@npm:2.15.21"
-  dependencies:
-    "@types/node": "npm:*"
-  checksum: 10/79fe91e1242d78b8aae15d883347f5a4f330452855c112fe465aef570e696def33dd8fea163e00416629eddc20b3a075e6146826881f813a0e1b52e47f39159a
-  languageName: node
-  linkType: hard
-
 "@types/node-fetch@npm:^2.6.4":
   version: 2.6.4
   resolution: "@types/node-fetch@npm:2.6.4"
@@ -6656,13 +6647,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"bignumber.js@npm:9.0.0":
-  version: 9.0.0
-  resolution: "bignumber.js@npm:9.0.0"
-  checksum: 10/7406d0d11dfdd2183e19be745f0d5913e3773ded5fbca2a310221e719f15fd8ec6b8d7991031a6081a6276a8e12e27d58ead60f73dcbb9d697ebe9e2dd0ad7e0
-  languageName: node
-  linkType: hard
-
 "bignumber.js@npm:^8.0.1":
   version: 8.1.1
   resolution: "bignumber.js@npm:8.1.1"
@@ -8654,6 +8638,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"denque@npm:^2.1.0":
+  version: 2.1.0
+  resolution: "denque@npm:2.1.0"
+  checksum: 10/8ea05321576624b90acfc1ee9208b8d1d04b425cf7573b9b4fa40a2c3ed4d4b0af5190567858f532f677ed2003d4d2b73c8130b34e3c7b8d5e88cdcfbfaa1fe7
+  languageName: node
+  linkType: hard
+
 "depd@npm:2.0.0":
   version: 2.0.0
   resolution: "depd@npm:2.0.0"
@@ -10502,6 +10493,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"generate-function@npm:^2.3.1":
+  version: 2.3.1
+  resolution: "generate-function@npm:2.3.1"
+  dependencies:
+    is-property: "npm:^1.0.2"
+  checksum: 10/318f85af87c3258d86df4ebbb56b63a2ae52e71bd6cde8d0a79de09450de7422a7047fb1f8d52ccc135564a36cb986d73c63149eed96b7ac57e38acba44f29e2
+  languageName: node
+  linkType: hard
+
 "gensync@npm:^1.0.0-beta.2":
   version: 1.0.0-beta.2
   resolution: "gensync@npm:1.0.0-beta.2"
@@ -10985,7 +10985,6 @@ __metadata:
     "@types/mdast": "npm:^3.0"
     "@types/minimist": "npm:^1.2.2"
     "@types/mousetrap": "npm:^1.6.9"
-    "@types/mysql": "npm:^2.15.21"
     "@types/papaparse": "npm:^5.3.5"
     "@types/progress": "npm:^2.0.5"
     "@types/randomstring": "npm:^1.1.8"
@@ -11071,7 +11070,7 @@ __metadata:
     mobx-react: "npm:5"
     mobx-utils: "npm:5"
     mousetrap: "npm:^1.6.5"
-    mysql: "npm:^2.18.1"
+    mysql2: "npm:^3.10.1"
     nodejs-polars: "npm:^0.7.2"
     normalize.css: "npm:^8.0.1"
     openai: "npm:^4.47.1"
@@ -11612,7 +11611,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2":
+"iconv-lite@npm:0.6.3, iconv-lite@npm:^0.6.2, iconv-lite@npm:^0.6.3":
   version: 0.6.3
   resolution: "iconv-lite@npm:0.6.3"
   dependencies:
@@ -12212,6 +12211,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"is-property@npm:^1.0.2":
+  version: 1.0.2
+  resolution: "is-property@npm:1.0.2"
+  checksum: 10/2f66eacb3d7237ba5c725496672edec656a20b12c80790921988578e6b11c258a062ce1e602f3cd2e3c2e05dd8b6e24e1d59254375207f157424a02ef0abb3d7
+  languageName: node
+  linkType: hard
+
 "is-regex@npm:^1.0.5, is-regex@npm:^1.1.4":
   version: 1.1.4
   resolution: "is-regex@npm:1.1.4"
@@ -13766,6 +13772,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"long@npm:^5.2.1":
+  version: 5.2.3
+  resolution: "long@npm:5.2.3"
+  checksum: 10/9167ec6947a825b827c30da169a7384eec6c0c9ec2f0b9c74da2e93d81159bbe39fb09c3f13dae9721d4b807ccfa09797a7dd1012f5d478e3e33ca3c78b608e6
+  languageName: node
+  linkType: hard
+
 "loose-envify@npm:^1.1.0, loose-envify@npm:^1.2.0, loose-envify@npm:^1.3.1, loose-envify@npm:^1.4.0":
   version: 1.4.0
   resolution: "loose-envify@npm:1.4.0"
@@ -13802,10 +13815,17 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lru-cache@npm:^7.5.1, lru-cache@npm:^7.7.1":
-  version: 7.14.0
-  resolution: "lru-cache@npm:7.14.0"
-  checksum: 10/f8e01009712d19e9da6001a9639188dc9a98f2686ed437a31432792c676e45a3ced8c4d28b117c18fd45eb49c7f8e676e5a5c31bf59c46a8ca0971c6b5280bc2
+"lru-cache@npm:^7.14.1, lru-cache@npm:^7.5.1, lru-cache@npm:^7.7.1":
+  version: 7.18.3
+  resolution: "lru-cache@npm:7.18.3"
+  checksum: 10/6029ca5aba3aacb554e919d7ef804fffd4adfc4c83db00fac8248c7c78811fb6d4b6f70f7fd9d55032b3823446546a007edaa66ad1f2377ae833bd983fac5d98
+  languageName: node
+  linkType: hard
+
+"lru-cache@npm:^8.0.0":
+  version: 8.0.5
+  resolution: "lru-cache@npm:8.0.5"
+  checksum: 10/74153ab136d0c2d735003b8b1c0fa8213c94c2520701dfe8bb31d957f975b3d3665b1ef27ac9a5b9f92c8f581c79008834c0f9bd60c5adf368476f9a95e8fa82
   languageName: node
   linkType: hard
 
@@ -14558,15 +14578,19 @@ __metadata:
   languageName: node
   linkType: hard
 
-"mysql@npm:^2.18.1":
-  version: 2.18.1
-  resolution: "mysql@npm:2.18.1"
+"mysql2@npm:^3.10.1":
+  version: 3.10.1
+  resolution: "mysql2@npm:3.10.1"
   dependencies:
-    bignumber.js: "npm:9.0.0"
-    readable-stream: "npm:2.3.7"
-    safe-buffer: "npm:5.1.2"
-    sqlstring: "npm:2.3.1"
-  checksum: 10/87d80e374717d7767d3e609f7f5e09987fa4dee208ba346ff269fffd2500719dcf2f65ac86c8e77649c3d52b86811a88e33cfd06e7e4a48cec53ecd4ac85c08d
+    denque: "npm:^2.1.0"
+    generate-function: "npm:^2.3.1"
+    iconv-lite: "npm:^0.6.3"
+    long: "npm:^5.2.1"
+    lru-cache: "npm:^8.0.0"
+    named-placeholders: "npm:^1.1.3"
+    seq-queue: "npm:^0.0.5"
+    sqlstring: "npm:^2.3.2"
+  checksum: 10/a22463070d3789c2704bc180c368ae49f62fd32f873e8e26bf6fb56b1ccd40f19e572967c92d7a3eef30c472d0984c37fd86f46627f3ec0ff8166ff2928a908d
   languageName: node
   linkType: hard
 
@@ -14581,6 +14605,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"named-placeholders@npm:^1.1.3":
+  version: 1.1.3
+  resolution: "named-placeholders@npm:1.1.3"
+  dependencies:
+    lru-cache: "npm:^7.14.1"
+  checksum: 10/7834adc91e92ae1b9c4413384e3ccd297de5168bb44017ff0536705ddc4db421723bd964607849265feb3f6ded390f84cf138e5925f22f7c13324f87a803dc73
+  languageName: node
+  linkType: hard
+
 "nanoid@npm:^3.3.3, nanoid@npm:^3.3.6":
   version: 3.3.7
   resolution: "nanoid@npm:3.3.7"
@@ -17430,7 +17463,18 @@ __metadata:
   languageName: node
   linkType: hard
 
-"readable-stream@npm:2.3.7, readable-stream@npm:~2.3.6":
+"readable-stream@npm:^3.0.0, readable-stream@npm:^3.0.2, readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0":
+  version: 3.6.0
+  resolution: "readable-stream@npm:3.6.0"
+  dependencies:
+    inherits: "npm:^2.0.3"
+    string_decoder: "npm:^1.1.1"
+    util-deprecate: "npm:^1.0.1"
+  checksum: 10/b80b3e6a7fafb1c79de7db541de357f4a5ee73bd70c21672f5a7c840d27bb27bdb0151e7ba2fd82c4a888df22ce0c501b0d9f3e4dfe51688876701c437d59536
+  languageName: node
+  linkType: hard
+
+"readable-stream@npm:~2.3.6":
   version: 2.3.7
   resolution: "readable-stream@npm:2.3.7"
   dependencies:
@@ -17445,17 +17489,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"readable-stream@npm:^3.0.0, readable-stream@npm:^3.0.2, readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0":
-  version: 3.6.0
-  resolution: "readable-stream@npm:3.6.0"
-  dependencies:
-    inherits: "npm:^2.0.3"
-    string_decoder: "npm:^1.1.1"
-    util-deprecate: "npm:^1.0.1"
-  checksum: 10/b80b3e6a7fafb1c79de7db541de357f4a5ee73bd70c21672f5a7c840d27bb27bdb0151e7ba2fd82c4a888df22ce0c501b0d9f3e4dfe51688876701c437d59536
-  languageName: node
-  linkType: hard
-
 "readdirp@npm:~3.6.0":
   version: 3.6.0
   resolution: "readdirp@npm:3.6.0"
@@ -18070,6 +18103,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"seq-queue@npm:^0.0.5":
+  version: 0.0.5
+  resolution: "seq-queue@npm:0.0.5"
+  checksum: 10/fa302e3b2aaece644532603ae42d675f9b8750e395a98740dd58dc5e02985ce6f0c2b78715b5984d6f6a807893735a14212a70d6ec591e6fba410397269588a0
+  languageName: node
+  linkType: hard
+
 "serve-static@npm:1.15.0":
   version: 1.15.0
   resolution: "serve-static@npm:1.15.0"
@@ -18554,14 +18594,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"sqlstring@npm:2.3.1":
-  version: 2.3.1
-  resolution: "sqlstring@npm:2.3.1"
-  checksum: 10/bc09237002da7e1172098e7d47401ea0ae45c1e4b224619f7ee2905dc921321f5ccc8c5e076994890df01b4a3363b2b5ea295b7a10d32a35181ef25bad158093
-  languageName: node
-  linkType: hard
-
-"sqlstring@npm:^2.3.3":
+"sqlstring@npm:^2.3.2, sqlstring@npm:^2.3.3":
   version: 2.3.3
   resolution: "sqlstring@npm:2.3.3"
   checksum: 10/4e5a25af2d77a031fe00694034bf9fd822ddc3a483c9383124b120aa6b9ae9ab71e173cd29fba9c653998ebfef9e97be668957839960b9b3dc1afcb45f1ddb64