From a7e064653840c7342e17bb6b60448cbf17282587 Mon Sep 17 00:00:00 2001 From: James Wigger Date: Wed, 5 Jan 2022 14:36:51 +0000 Subject: [PATCH] Feature: adds support for MySQL 8 using mysql2 library --- README.md | 1 + docker-compose.yml | 7 + lib/MySql2DatabaseManager.js | 227 +++++++++++++++++++++++++++++++++ lib/index.js | 5 + package.json | 1 + tests/database-manager.spec.js | 26 +++- tests/dialect-aliases.spec.js | 1 + wait-databases.sh | 23 ++-- 8 files changed, 279 insertions(+), 12 deletions(-) create mode 100644 lib/MySql2DatabaseManager.js diff --git a/README.md b/README.md index bc5ffc4..611e2d0 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ and dropping databases / roles. - PostgreSQL - MySQL +- MySQL 8 - SQLite3 (partial support even though most of the functions won't make sense with this) - ~~Oracle DB Express (TBD)~~ - ~~MSSQL (TBD if we can get integration tests to run automatically)~~ diff --git a/docker-compose.yml b/docker-compose.yml index afaedc7..09c651f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,13 @@ services: environment: - TZ=UTC - MYSQL_ROOT_PASSWORD=mysqlrootpassword + mysql8: + image: mysql:8 + ports: + - '23306:3306' + environment: + - TZ=UTC + - MYSQL_ROOT_PASSWORD=mysqlrootpassword postgresql: image: mdillon/postgis:9.6 ports: diff --git a/lib/MySql2DatabaseManager.js b/lib/MySql2DatabaseManager.js new file mode 100644 index 0000000..55d3b04 --- /dev/null +++ b/lib/MySql2DatabaseManager.js @@ -0,0 +1,227 @@ +var DatabaseManager = require('./DatabaseManager').default, + classUtils = require('./class-utils'), + mysql = require('mysql2'), + Promise = require('bluebird'), + _ = require('lodash'); + +/** + * @constructor + * @extends DatabaseManager + * + * Notes: + * - Even though the method signature implicates that _masterConnectionUrl returns + * an URL string, it actually returns an object because MySQL node lib + * assumes that the database name is defined in the URL format. + * + */ +function MySql2DatabaseManager() { + DatabaseManager.apply(this, arguments); + this._masterClient = null; + this._cachedTableNames = null; +} + +classUtils.inherits(MySql2DatabaseManager, DatabaseManager); + +/** + * @Override + */ +MySql2DatabaseManager.prototype.createDbOwnerIfNotExist = function() { + return this._masterQuery("CREATE USER IF NOT EXISTS ?@'%' IDENTIFIED BY ?", [ + this.config.knex.connection.user, + this.config.knex.connection.password, + ]); +}; + +/** + * @Override + */ +MySql2DatabaseManager.prototype.createDb = function(databaseName) { + databaseName = databaseName || this.config.knex.connection.database; + var collate = this.config.dbManager.collate; + var owner = this.config.knex.connection.user; + var self = this; + var promise = Promise.reject(new Error()); + + if (_.isEmpty(collate)) { + promise = promise.catch(function() { + return self._masterQuery( + 'CREATE DATABASE ?? DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci', + [databaseName] + ); + }); + } else { + // Try to create with each collate. Use the first one that works. This is kind of a hack + // but seems to be the only reliable way to make this work with both windows and unix. + _.each(collate, function(locale) { + promise = promise.catch(function() { + return self._masterQuery( + 'CREATE DATABASE ?? DEFAULT CHARACTER SET utf8 DEFAULT COLLATE ?', + [databaseName, locale] + ); + }); + }); + } + + promise = promise.then(function() { + return self._masterQuery('GRANT ALL PRIVILEGES ON ??.* TO ??', [ + databaseName, + owner, + ]); + }); + + return promise; +}; + +/** + * Drops database with name if db exists. + * + * @Override + */ +MySql2DatabaseManager.prototype.dropDb = function(databaseName) { + databaseName = databaseName || this.config.knex.connection.database; + return this._masterQuery('DROP DATABASE IF EXISTS ??', [databaseName]); +}; + +/** + * @Override + */ +MySql2DatabaseManager.prototype.truncateDb = function(ignoreTables) { + var knex = this.knexInstance(); + var config = this.config; + + if (!this._cachedTableNames) { + this._updateTableNameCache(knex, config); + } + + return this._cachedTableNames.then(function(tableNames) { + if (!_.isEmpty(tableNames)) { + return knex.transaction(function(trx) { + return knex + .raw('SET FOREIGN_KEY_CHECKS = 0') + .transacting(trx) + .then(function() { + // ignore the tables based on `ignoreTables` + var filteredTables = _.differenceWith( + tableNames, + ignoreTables, + _.isEqual + ); + return Promise.map( + filteredTables, + function(tableName) { + return knex + .table(tableName) + .truncate() + .transacting(trx); + }, + { concurrency: 1 } + ); + }); + }); + } + }); +}; + +/** + * @private + */ +MySql2DatabaseManager.prototype._updateTableNameCache = function(knex, config) { + this._cachedTableNames = knex('information_schema.TABLES') + .select('TABLE_NAME') + .where('TABLE_SCHEMA', config.knex.connection.database) + .then(function(tables) { + return _.without( + _.map(tables, 'TABLE_NAME'), + config.knex.migrations.tableName + ); + }); +}; + +/** + * @Override + */ +MySql2DatabaseManager.prototype.close = function() { + var disconnectAll = [this.closeKnex()]; + if (this._masterClient) { + disconnectAll.push( + this._masterClient.then(function(client) { + client.end(); + }) + ); + this._masterClient = null; + } + return Promise.all(disconnectAll); +}; + +/** + * @private + * @returns {Promise} + */ +MySql2DatabaseManager.prototype._masterQuery = function(query, params) { + var self = this; + if (!this._masterClient) { + this._masterClient = this.create_masterClient(); + } + return this._masterClient.then(function(client) { + return self.perform_masterQuery(client, query, params); + }); +}; + +/** + * @private + * @returns {Promise} + */ +MySql2DatabaseManager.prototype.create_masterClient = function() { + var self = this; + return new Promise(function(resolve, reject) { + var client = mysql.createConnection(self._masterConnectionUrl()); + client.connect(function(err) { + if (err) { + reject(err); + } else { + resolve(client); + } + }); + }); +}; + +/** + * @private + * @returns {Promise} + */ +MySql2DatabaseManager.prototype.perform_masterQuery = function( + client, + query, + params +) { + return new Promise(function(resolve, reject) { + if (params) { + query = mysql.format(query, params); + } + client.query(query, function(err, result) { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + }); +}; + +/** + * @private + * @returns {String} + */ +MySql2DatabaseManager.prototype._masterConnectionUrl = function() { + return { + host: this.config.knex.connection.host, + port: this.config.knex.connection.port || 3306, + user: this.config.dbManager.superUser, + password: this.config.dbManager.superPassword, + }; +}; + +module.exports = { + default: MySql2DatabaseManager, + MySql2DatabaseManager: MySql2DatabaseManager, +}; diff --git a/lib/index.js b/lib/index.js index 59a6742..2f047c9 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,3 +1,4 @@ +const {default: MySqlDatabaseManager} = require('./MySqlDatabaseManager'); /** * Configuration is dodo-objection configuration. * @@ -26,6 +27,10 @@ module.exports = { var MySqlDatabaseManager = require('./MySqlDatabaseManager').default; return new MySqlDatabaseManager(config); } + case 'mysql2': { + var MySql2DatabaseManager = require('./MySql2DatabaseManager').default; + return new MySql2DatabaseManager(config); + } case 'sqlite3': case 'sqlite': { var SqliteDatabaseManager = require('./SqliteDatabaseManager').default; diff --git a/package.json b/package.json index ae416d9..e2ebc29 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "istanbul": "^0.4.5", "mocha": "^7.1.1", "mysql": "^2.13.0", + "mysql2": "^2.3.3", "npm-check": "^5.9.2", "nsp": "^3.2.1", "pg": "^8.3.2", diff --git a/tests/database-manager.spec.js b/tests/database-manager.spec.js index 159f818..83bef92 100644 --- a/tests/database-manager.spec.js +++ b/tests/database-manager.spec.js @@ -73,6 +73,22 @@ var mySqlConf = { }, }; +var mySql2Conf = { + knex: { + client: 'mysql2', + connection: _.assign({}, connection, { + port: 23306, + }), + pool: pool, + migrations: migrations, + }, + dbManager: { + collate: ['utf8_swedish_ci'], + superUser: 'root', + superPassword: 'mysqlrootpassword', + }, +}; + var sqliteConf = { knex: { client: 'sqlite', @@ -147,6 +163,10 @@ var availableDatabases = [ { name: 'MySQL 5.7', manager: dbManagerFactory(mySqlConf), + }, + { + name: 'MySQL 8.0', + manager: dbManagerFactory(mySql2Conf), // },{ // name: 'SQLite', // manager: dbManagerFactory(sqliteConf) @@ -268,7 +288,7 @@ _.map(availableDatabases, function(db) { it('#copyDb should copy a database', function() { // CopyDB not implemented on MySqlDatabaseManager yet... - if (dbManager.config.knex.client === 'mysql') { + if (['mysql', 'mysql2'].includes(dbManager.config.knex.client)) { return; } return dbManager @@ -329,7 +349,7 @@ _.map(availableDatabases, function(db) { it('#updateIdSequences should update primary key sequences', function() { // UpdateIdSequences not implemented on MySqlDatabaseManager yet... - if (dbManager.config.knex.client === 'mysql') { + if (['mysql', 'mysql2'].includes(dbManager.config.knex.client)) { return; } @@ -364,7 +384,7 @@ _.map(availableDatabases, function(db) { it('#updateIdSequences should work with empty table and with minimum value other than 1', function() { // UpdateIdSequences not implemented on MySqlDatabaseManager yet... - if (dbManager.config.knex.client === 'mysql') { + if (['mysql', 'mysql2'].includes(dbManager.config.knex.client)) { return; } diff --git a/tests/dialect-aliases.spec.js b/tests/dialect-aliases.spec.js index 57562de..81ef360 100644 --- a/tests/dialect-aliases.spec.js +++ b/tests/dialect-aliases.spec.js @@ -7,6 +7,7 @@ describe('Testing dialect aliases', function() { 'postgres', 'postgresql', 'mysql', + 'mysql2', 'maria', 'mariadb', 'mariasql', diff --git a/wait-databases.sh b/wait-databases.sh index d0d7ed2..00f5d48 100644 --- a/wait-databases.sh +++ b/wait-databases.sh @@ -1,16 +1,21 @@ -for i in $(seq 60); do - docker-compose exec postgresql psql postgres://postgres:postgresrootpassword@localhost -c "SELECT 1" && break; - sleep 1; +for i in $(seq 60); do + docker-compose exec postgresql psql postgres://postgres:postgresrootpassword@localhost -c "SELECT 1" && break; + sleep 1; done -for i in $(seq 60); do - docker-compose exec postgresql10 psql postgres://postgres:postgresrootpassword@localhost -c "SELECT 1" && break; - sleep 1; +for i in $(seq 60); do + docker-compose exec postgresql10 psql postgres://postgres:postgresrootpassword@localhost -c "SELECT 1" && break; + sleep 1; done -for i in $(seq 60); do - docker-compose exec mysql mysql -hmysql -uroot -pmysqlrootpassword -e "SELECT 1" && break; - sleep 1; +for i in $(seq 60); do + docker-compose exec mysql mysql -hmysql -uroot -pmysqlrootpassword -e "SELECT 1" && break; + sleep 1; +done + +for i in $(seq 60); do + docker-compose exec mysql8 mysql -hmysql -uroot -pmysqlrootpassword -e "SELECT 1" && break; + sleep 1; done