diff --git a/src/api.ts b/src/api.ts index c119dbbe..a72416bd 100644 --- a/src/api.ts +++ b/src/api.ts @@ -6,12 +6,15 @@ * are also meant to be the public interface for the developers using it as a package. */ +import { Knex } from 'knex'; + import * as init from './init'; import { log } from './util/logger'; import { existsDir } from './util/fs'; import { withTransaction, mapToConnectionReferences, DatabaseConnections } from './util/db'; import Configuration from './domain/Configuration'; +import { RunScriptParams } from './domain/RunScriptParams'; import SynchronizeParams from './domain/SynchronizeParams'; import ConnectionReference from './domain/ConnectionReference'; import OperationParams from './domain/operation/OperationParams'; @@ -19,8 +22,59 @@ import OperationResult from './domain/operation/OperationResult'; // Service import { executeProcesses } from './service/execution'; -import { runSynchronize, runPrune } from './service/sync'; -import { getMigrationPath, invokeMigrationApi, KnexMigrationAPI } from './migration/service/knexMigrator'; +import { runSynchronize, runPrune, runScript } from './service/sync'; +import { + getManualScriptPath, + getMigrationPath, + invokeMigrationApi, + KnexMigrationAPI +} from './migration/service/knexMigrator'; +import { runScriptWithLog } from './service/sqlRunner'; + +/** + * Run scripts for all the configured database connections. + * + * @param {Configuration} config + * @param {DatabaseConnections} conn + * @param {RunScriptParams} options + * @returns {Promise} + */ +export async function runScriptAPI(config: Configuration, conn: DatabaseConnections, options?: RunScriptParams) { + log('Run Script'); + const scriptPath = getManualScriptPath(config); + const dirExist = await existsDir(scriptPath); + + if (!dirExist) { + log('Script directory does not exist'); + } + + const params: RunScriptParams = { + ...options + }; + + const connections = filterConnections(mapToConnectionReferences(conn), params.only); + + const processes = connections.map(connection => () => + withTransaction( + connection, + trx => + runScript(trx, { + config, + params, + connectionId: connection.id, + migrateFunc: ( + t: Knex.Transaction, + files: string[], + connectionId: string, + runSQLScripts: (t: Knex.Transaction, filteredScript: string[]) => Promise + ) => runScriptWithLog(t, files, connectionId, config.manual.tableName, runSQLScripts) + }), + params['dry-run'] + ) + ); + + return executeProcesses(processes, config); +} /** * Synchronize all the configured database connections. diff --git a/src/commands/run-script.ts b/src/commands/run-script.ts new file mode 100644 index 00000000..f9dbe63b --- /dev/null +++ b/src/commands/run-script.ts @@ -0,0 +1,115 @@ +import { Command, flags } from '@oclif/command'; +import { bold, red, magenta, cyan } from 'chalk'; + +import { runScriptAPI } from '../api'; +import { dbLogger } from '../util/logger'; +import { loadConfig, resolveConnections } from '..'; +import { validateScriptFileName } from '../util/fs'; +import { printLine, printError, printInfo } from '../util/io'; +import OperationResult from '../domain/operation/OperationResult'; + +class RunScript extends Command { + static description = 'Run the provided manual scripts.'; + + static flags = { + 'dry-run': flags.boolean({ description: 'Dry run script.', default: false }), + only: flags.string({ + helpValue: 'CONNECTION_ID(s)', + description: 'Filter provided connection(s). Comma separated ids eg: id1,id2' + }), + file: flags.string({ + required: true, + helpValue: 'Script Name', + parse: validateScriptFileName, + description: 'Name of the manual SQL/JS/TS script' + }), + 'connection-resolver': flags.string({ + helpValue: 'PATH', + description: 'Path to the connection resolver.' + }), + config: flags.string({ + char: 'c', + description: 'Custom configuration file.' + }) + }; + + /** + * Started event handler. + */ + onStarted = async (result: OperationResult) => { + await printLine(bold(` ▸ ${result.connectionId}`)); + + await printInfo(' [✓] Manual script run - started'); + }; + + /** + * Success handler. + */ + onSuccess = async (result: OperationResult) => { + const log = dbLogger(result.connectionId); + const [num, list] = result.data; + const alreadyUpToDate = num && list.length === 0; + + log('Up to date: ', alreadyUpToDate); + + await printInfo(` [✓] Manual script run - completed (${result.timeElapsed}s)`); + + if (alreadyUpToDate) { + await printInfo(' Already up to date.\n'); + + return; + } + + // Completed migrations. + for (const item of list) { + await printLine(cyan(` - ${item}`)); + } + + await printInfo(` Ran ${list.length} scripts.\n`); + }; + + /** + * Failure handler. + */ + onFailed = async (result: OperationResult) => { + await printLine(bold(red(` [✓] Manual script run - Failed\n`))); + }; + + /** + * CLI command execution handler. + * + * @returns {Promise} + */ + async run(): Promise { + const { flags: parsedFlags } = this.parse(RunScript); + const isDryRun = parsedFlags['dry-run']; + const config = await loadConfig(parsedFlags.config); + + const connections = await resolveConnections(config, parsedFlags['connection-resolver']); + + if (isDryRun) await printLine(magenta('\n• DRY RUN STARTED\n')); + + const results = await runScriptAPI(config, connections, { + ...parsedFlags, + onStarted: this.onStarted, + onSuccess: this.onSuccess, + onFailed: this.onFailed + }); + + const failedCount = results.filter(({ success }) => !success).length; + + if (failedCount === 0) { + if (isDryRun) await printLine(magenta('• DRY RUN ENDED\n')); + + return process.exit(0); + } + + printError(`Error: Script failed for ${failedCount} connection(s).`); + + if (isDryRun) await printLine(magenta('\n• DRY RUN ENDED\n')); + + process.exit(-1); + } +} + +export default RunScript; diff --git a/src/config.ts b/src/config.ts index 8ab033ec..6cc27a6b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,6 +39,16 @@ export function getSqlBasePath(config: Configuration): string { return path.join(config.basePath, 'sql'); } +/** + * Get manual scripts path from config. + * + * @param {Configuration} config + * @returns {string} + */ +export function getManualScriptBasePath(config: Configuration): string { + return path.join(config.basePath, config.manual.directory); +} + /** * Load config yaml file. * diff --git a/src/constants.ts b/src/constants.ts index 6b104a19..bdf19956 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -28,6 +28,10 @@ export const DEFAULT_CONFIG: Configuration = { directory: 'migration', tableName: 'knex_migrations', // Note: This is Knex's default value. Just keeping it same. sourceType: 'sql' + }, + manual: { + directory: 'manual', + tableName: 'manual_scripts' } }; diff --git a/src/domain/Configuration.ts b/src/domain/Configuration.ts index b8e55964..4fa488c2 100644 --- a/src/domain/Configuration.ts +++ b/src/domain/Configuration.ts @@ -20,6 +20,10 @@ interface Configuration { tableName: string; sourceType: 'sql' | 'javascript' | 'typescript'; }; + manual: { + directory: string; + tableName: string; + }; } export default Configuration; diff --git a/src/domain/RunScriptContext.ts b/src/domain/RunScriptContext.ts new file mode 100644 index 00000000..a18fdf31 --- /dev/null +++ b/src/domain/RunScriptContext.ts @@ -0,0 +1,14 @@ +import { Knex } from 'knex'; + +import { RunScriptParams } from './RunScriptParams'; +import OperationContext from './operation/OperationContext'; + +export interface RunScriptContext extends OperationContext { + params: RunScriptParams; + migrateFunc: ( + trx: Knex.Transaction, + files: string[], + connectionId: string, + runSQLScripts: (trx: Knex.Transaction, filteredScripts: string[]) => Promise + ) => Promise<(number | string[])[]>; +} diff --git a/src/domain/RunScriptParams.ts b/src/domain/RunScriptParams.ts new file mode 100644 index 00000000..30b63ed2 --- /dev/null +++ b/src/domain/RunScriptParams.ts @@ -0,0 +1,5 @@ +import OperationParams from './operation/OperationParams'; + +export interface RunScriptParams extends OperationParams { + file?: string; +} diff --git a/src/migration/service/knexMigrator.ts b/src/migration/service/knexMigrator.ts index 9903b702..896ae7fc 100644 --- a/src/migration/service/knexMigrator.ts +++ b/src/migration/service/knexMigrator.ts @@ -123,3 +123,17 @@ export function getMigrationPath(config: Configuration): string { return migrationPath; } + +/** + * Get manual scripts directory path. + * + * @param {Configuration} config + * @returns {string} + */ +export function getManualScriptPath(config: Configuration): string { + const { basePath, manual } = config; + + const scriptPath = path.isAbsolute(manual.directory) ? manual.directory : path.join(basePath, manual.directory); + + return scriptPath; +} diff --git a/src/migration/service/migrator.ts b/src/migration/service/migrator.ts index ce56de68..37852a82 100644 --- a/src/migration/service/migrator.ts +++ b/src/migration/service/migrator.ts @@ -1,9 +1,12 @@ +import { Knex } from 'knex'; import * as path from 'path'; import { glob, exists } from '../../util/fs'; +import { getManualScriptPath } from './knexMigrator'; import { resolveFile } from '../../service/sqlRunner'; import FileExtensions from '../../enum/FileExtensions'; import SqlMigrationEntry from '../domain/SqlMigrationEntry'; +import { RunScriptContext } from '../../domain/RunScriptContext'; import JavaScriptMigrationEntry from '../domain/JavaScriptMigrationEntry'; const FILE_PATTERN_JS = /(.+).js$/; @@ -127,3 +130,55 @@ export async function resolveJavaScriptMigrations( return Promise.all(migrationPromises); } + +/** + * Run exposed function from manual JS/TS scripts. + * + * @param {Knex.Transaction} trx + * @param {RunScriptContext} context + * @param {string} connectionId + * @param {string[]} filteredScripts + * @param {string} extension + * @returns {Promise} + */ +export async function runJSTSScripts( + trx: Knex.Transaction, + context: RunScriptContext, + connectionId: string, + filteredScripts: string[], + extension: string +) { + const migrationNames = filteredScripts; + + let mRequire: NodeRequire = require; + + if (extension === FileExtensions.TS) { + // Transpile & execute ts files required on the fly + require('ts-node').register({ + transpileOnly: true + }); + } else { + // On the fly es6 => commonJS + mRequire = require('esm')(module); + } + + const migrationPromises = migrationNames.map(async name => { + const { main } = mRequire(path.resolve(`${getManualScriptPath(context.config)}/${extension}`, name)); + + return { + main, + name + }; + }); + + const fileFuncInfos = await Promise.all(migrationPromises); + + if (!fileFuncInfos.length) { + return; + } + + const func = fileFuncInfos[0].main; + + // Execute the function + await func(trx, connectionId); +} diff --git a/src/service/sqlRunner.ts b/src/service/sqlRunner.ts index 189d2b8f..5e985130 100644 --- a/src/service/sqlRunner.ts +++ b/src/service/sqlRunner.ts @@ -8,8 +8,10 @@ import SqlCode from '../domain/SqlCode'; import { dbLogger } from '../util/logger'; import * as promise from '../util/promise'; import SqlFileInfo from '../domain/SqlFileInfo'; +import { getManualScriptBasePath } from '../config'; import { DROP_ONLY_OBJECT_TERMINATOR } from '../constants'; import DatabaseObjectTypes from '../enum/DatabaseObjectTypes'; +import { RunScriptContext } from '../domain/RunScriptContext'; /** * SQL DROP statements mapping for different object types. @@ -165,3 +167,119 @@ export async function rollbackSequentially(trx: Knex, files: SqlFileInfo[], conn log('Executed: ', sql.dropStatement); } } + +/** + * Create table to log manual script run. + * + * @param {Knex.Transaction} trx + * @param {string} tableName + * @returns {Promise} + */ +async function createScriptLogTable(trx: Knex, tableName: string) { + return trx.schema.createTable(tableName, t => { + t.increments('id').primary().unsigned(); + t.string('name').notNullable(); + t.dateTime('run_time'); + }); +} + +/** + * Filter the runn manual scripts based on log table. + * + * @param {Knex.Transaction} trx + * @param {string[]} files + * @param {string} tableName + * @returns {Promise} + */ +async function getFilteredScriptsToRun(trx: Knex.Transaction, files: string[], tableName: string) { + const data = await trx(tableName).select('name'); + + const totalRunscripts = data.map(sc => sc.name); + + const filteredScripts = files.filter(fname => !totalRunscripts.includes(fname)); + + return filteredScripts; +} + +/** + * Run manual scripts with logging. + * + * @param {Knex.Transaction} trx + * @param {string[]} files + * @param {string} connectionId + * @param {string} tableName + * @returns {Promise} + */ +export async function runScriptWithLog( + trx: Knex.Transaction, + files: string[], + connectionId: string, + tableName: string, + callback: (t: Knex.Transaction, filteredScript: string[]) => Promise +) { + const log = dbLogger(connectionId); + + const logTableExists = await trx.schema.hasTable(tableName); + + if (!logTableExists) { + log(`Manual query record table doesn't exists. Creating one`); + + await createScriptLogTable(trx, tableName); + } + + const filData = await getFilteredScriptsToRun(trx, files, tableName); + + await callback(trx, filData); + + for await (const file of filData) { + await trx(tableName).insert({ + name: file, + run_time: new Date() + }); + } + + return [files.length, filData]; +} + +/** + * Get file details by extension. + * + * @param {string} filename + * @returns {object} + */ +export function getFileDetailsByExtension(filename: string) { + const extension = filename.split('.').slice(-1).pop(); + + return { + extension, + fileNames: [filename], + fileRelativePaths: [`${extension}/${filename}`] + }; +} + +/** + * Run manual scripts in SQL. + * + * @param {Knex.Transaction} trx + * @param {RunScriptContext} context + * @param {string[]} manualSql + * @param {string} connectionId + * @param {string[]} filteredScripts + * @returns {Promise} + */ +export async function runSQLScripts( + trx: Knex.Transaction, + context: RunScriptContext, + manualSql: string[], + connectionId: string, + filteredScripts: string[] +) { + const sqlBasePath = getManualScriptBasePath(context.config); + + const sqlScripts = await resolveFiles(sqlBasePath, manualSql); + + const filteredScriptsToRun = sqlScripts.filter(scd => !filteredScripts.includes(scd.name)); + + // Run the synchronization scripts. + await runSequentially(trx, filteredScriptsToRun, connectionId); +} diff --git a/src/service/sync.ts b/src/service/sync.ts index 56ad2ce7..3e8b3571 100644 --- a/src/service/sync.ts +++ b/src/service/sync.ts @@ -2,13 +2,16 @@ import { Knex } from 'knex'; import * as sqlRunner from './sqlRunner'; import { dbLogger } from '../util/logger'; +import { getSqlBasePath } from '../config'; import { getElapsedTime } from '../util/ts'; -import SynchronizeContext from '../domain/SynchronizeContext'; +import { executeOperation } from './execution'; +import FileExtensions from '../enum/FileExtensions'; import * as configInjection from './configInjection'; +import SynchronizeContext from '../domain/SynchronizeContext'; +import { RunScriptContext } from '../domain/RunScriptContext'; +import { runJSTSScripts } from '../migration/service/migrator'; import OperationResult from '../domain/operation/OperationResult'; import OperationContext from '../domain/operation/OperationContext'; -import { executeOperation } from './execution'; -import { getSqlBasePath } from '../config'; /** * Migrate SQL on a database. @@ -119,6 +122,50 @@ export async function runSynchronize(trx: Knex.Transaction, context: Synchronize }); } +/** + * Run manual scripts on the given database connection (transaction). + * + * @param {Knex.Transaction} trx + * @param {RunScriptContext} context + * @returns {Promise} + */ +export async function runScript(trx: Knex.Transaction, context: RunScriptContext): Promise { + return executeOperation(context, async options => { + const { connectionId, migrateFunc } = context; + const { timeStart } = options; + const log = dbLogger(connectionId); + + const scriptFileName = context.params.file; + + log(`Running manual script for - ${connectionId} - ${timeStart} `); + + let filesToRun: string[] = []; + let manualSqlToRun: string[] = []; + + let ext = FileExtensions.SQL; + + if (!!scriptFileName) { + const { extension, fileNames, fileRelativePaths } = sqlRunner.getFileDetailsByExtension(scriptFileName); + + filesToRun = fileNames; + ext = extension as FileExtensions; + manualSqlToRun = fileRelativePaths; + } + + if (ext === FileExtensions.SQL) { + return migrateFunc(trx, filesToRun, connectionId, (t, filteredScripts) => + sqlRunner.runSQLScripts(t, context, manualSqlToRun, connectionId, filteredScripts) + ); + } + + if ([FileExtensions.JS, FileExtensions.TS].includes(ext)) { + return migrateFunc(trx, filesToRun, connectionId, (t, filteredScripts) => + runJSTSScripts(t, context, connectionId, filteredScripts, ext) + ); + } + }); +} + /** * Rune prune operation (drop all synchronized objects) on the given database connection (transaction). * diff --git a/src/util/fs.ts b/src/util/fs.ts index 965517a6..d12e20ba 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -2,6 +2,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { promisify } from 'util'; +import FileExtensions from '../enum/FileExtensions'; export const mkdir = promisify(fs.mkdir); export const readDir = promisify(fs.readdir); @@ -141,3 +142,19 @@ export function copy(fromPath: string, toPath: string): Promise { }); }); } + +/** + * Validate the script filename provided from CLI. + * + * @param {string} filename + * @returns {string} + */ +export function validateScriptFileName(filename: string): string { + const ext = filename.split('.').pop(); + + if (!ext || ![FileExtensions.TS, FileExtensions.SQL, FileExtensions.JS].includes(ext as FileExtensions)) { + throw new Error('Invalid file name or extension'); + } + + return filename; +} diff --git a/test/cli/commands/run-script.test.ts b/test/cli/commands/run-script.test.ts new file mode 100644 index 00000000..9086f34c --- /dev/null +++ b/test/cli/commands/run-script.test.ts @@ -0,0 +1,18 @@ +import { expect } from 'chai'; +import { it, describe } from 'mocha'; + +import { runCli } from './util'; +import { mkdtempSync } from '../../../src/util/fs'; + +const cwd = mkdtempSync(); + +describe('CLI: Run script', () => { + describe('--help', () => { + it('should print help message.', async () => { + const { stdout } = await runCli(['run-script', '--help'], { cwd }); + + expect(stdout).contains('manual scripts'); + expect(stdout).contains(`USAGE\n $ sync-db run-script`); + }); + }); +}); diff --git a/test/unit/service/sqlRunner.test.ts b/test/unit/service/sqlRunner.test.ts index a505a676..ca9f0a04 100644 --- a/test/unit/service/sqlRunner.test.ts +++ b/test/unit/service/sqlRunner.test.ts @@ -69,4 +69,14 @@ describe('SERVICE: sqlRunner', () => { expect(() => sqlRunner.getDropStatement('views', 'test.hello_world')).to.throw(Error); }); }); + + describe('getFileDetailsByExtension()', () => { + it('should return file details with extention', () => { + expect(sqlRunner.getFileDetailsByExtension('test.sql')).to.deep.equal({ + extension: 'sql', + fileNames: ['test.sql'], + fileRelativePaths: ['sql/test.sql'] + }); + }); + }); }); diff --git a/test/unit/util/config.test.ts b/test/unit/util/config.test.ts index 13fb9025..311770c3 100644 --- a/test/unit/util/config.test.ts +++ b/test/unit/util/config.test.ts @@ -6,7 +6,14 @@ import { it, describe } from 'mocha'; import { mkdtemp, write } from '../../../src/util/fs'; import Configuration from '../../../src/domain/Configuration'; import ConnectionConfig from '../../../src/domain/ConnectionConfig'; -import { validate, getConnectionId, resolveConnectionsFromEnv, isCLI, loadConfig } from '../../../src/config'; +import { + validate, + getConnectionId, + resolveConnectionsFromEnv, + isCLI, + loadConfig, + getManualScriptBasePath +} from '../../../src/config'; describe('CONFIG:', () => { describe('isCLI', () => { @@ -214,4 +221,24 @@ describe('CONFIG:', () => { expect(config).to.have.property('migration'); }); }); + + describe('getManualScriptBasePath', () => { + it('should resolve correct manual script base path.', async () => { + const cwd = await mkdtemp(); + await write( + path.join(cwd, 'sync-db.yml'), + yaml.stringify({ + basePath: 'src', + manual: { + directory: 'manual', + tableName: 'manual_script_logs' + } + } as Configuration) + ); + + process.chdir(cwd); + const config = await loadConfig(); + expect(getManualScriptBasePath(config)).to.eq('src/manual'); + }); + }); }); diff --git a/test/unit/util/fs.test.ts b/test/unit/util/fs.test.ts index 9df42cb5..53dba7e9 100644 --- a/test/unit/util/fs.test.ts +++ b/test/unit/util/fs.test.ts @@ -1,9 +1,9 @@ import * as fs from 'fs'; import * as path from 'path'; -import { expect } from 'chai'; +import { expect, assert } from 'chai'; import { it, describe } from 'mocha'; -import { write, read, remove, exists, mkdtemp, glob, existsDir } from '../../../src/util/fs'; +import { write, read, remove, exists, mkdtemp, glob, existsDir, validateScriptFileName } from '../../../src/util/fs'; describe('UTIL: fs', () => { let filePath: string; @@ -114,4 +114,28 @@ describe('UTIL: fs', () => { expect(result).to.deep.equal([]); }); }); + + describe('validateScriptFileName', () => { + it('should throw error for invalid file name', async () => { + return expect(() => validateScriptFileName('test')).to.throw('Invalid file name or extension'); + }); + + it('should support only JS/TS or SQL script', () => { + const fname0 = validateScriptFileName('test.sql'); + const fname1 = validateScriptFileName('test.js'); + const fname2 = validateScriptFileName('test.ts'); + + assert(fname0 === 'test.sql'); + assert(fname1 === 'test.js'); + assert(fname2 === 'test.ts'); + + expect(() => validateScriptFileName('test.c')).to.throw('Invalid file name or extension'); + }); + + it('should return filename if validation is successfull', () => { + const fname = validateScriptFileName('test.sql'); + + return expect(fname).to.eq('test.sql'); + }); + }); });