Introduce new command to run manual scripts

src/api.ts

@@ -6,21 +6,75 @@
  * are also meant to be the public interface for the developers using it as a package.
  */
 
+import { Knex } from 'knex';
+
 import * as init from './init';
 import { log } from './util/logger';
 import { existsDir } from './util/fs';
 import { withTransaction, mapToConnectionReferences, DatabaseConnections } from './util/db';
 
 import Configuration from './domain/Configuration';
+import { RunScriptParams } from './domain/RunScriptParams';
 import SynchronizeParams from './domain/SynchronizeParams';
 import ConnectionReference from './domain/ConnectionReference';
 import OperationParams from './domain/operation/OperationParams';
 import OperationResult from './domain/operation/OperationResult';
 
 // Service
 import { executeProcesses } from './service/execution';
-import { runSynchronize, runPrune } from './service/sync';
-import { getMigrationPath, invokeMigrationApi, KnexMigrationAPI } from './migration/service/knexMigrator';
+import { runSynchronize, runPrune, runScript } from './service/sync';
+import {
+  getManualScriptPath,
+  getMigrationPath,
+  invokeMigrationApi,
+  KnexMigrationAPI
+} from './migration/service/knexMigrator';
+import { runScriptWithLog } from './service/sqlRunner';
+
+/**
+ * Run scripts for all the configured database connections.
+ *
+ * @param {Configuration} config
+ * @param {DatabaseConnections} conn
+ * @param {RunScriptParams} options
+ * @returns {Promise<OperationResult[]>}
+ */
+export async function runScriptAPI(config: Configuration, conn: DatabaseConnections, options?: RunScriptParams) {
+  log('Run Script');
+  const scriptPath = getManualScriptPath(config);
+  const dirExist = await existsDir(scriptPath);
+
+  if (!dirExist) {
+    log('Script directory does not exist');
+  }
+
+  const params: RunScriptParams = {
+    ...options
+  };
+
+  const connections = filterConnections(mapToConnectionReferences(conn), params.only);
+
+  const processes = connections.map(connection => () =>
+    withTransaction(
+      connection,
+      trx =>
+        runScript(trx, {
+          config,
+          params,
+          connectionId: connection.id,
+          migrateFunc: (
+            t: Knex.Transaction,
+            files: string[],
+            connectionId: string,
+            runSQLScripts: (t: Knex.Transaction, filteredScript: string[]) => Promise<void>
+          ) => runScriptWithLog(t, files, connectionId, config.manual.tableName, runSQLScripts)
+        }),
+      params['dry-run']
+    )
+  );
+
+  return executeProcesses(processes, config);
+}
 
 /**
  * Synchronize all the configured database connections.

src/commands/run-script.ts

@@ -0,0 +1,115 @@
+import { Command, flags } from '@oclif/command';
+import { bold, red, magenta, cyan } from 'chalk';
+
+import { runScriptAPI } from '../api';
+import { dbLogger } from '../util/logger';
+import { loadConfig, resolveConnections } from '..';
+import { validateScriptFileName } from '../util/fs';
+import { printLine, printError, printInfo } from '../util/io';
+import OperationResult from '../domain/operation/OperationResult';
+
+class RunScript extends Command {
+  static description = 'Run the provided manual scripts.';
+
+  static flags = {
+    'dry-run': flags.boolean({ description: 'Dry run script.', default: false }),
+    only: flags.string({
+      helpValue: 'CONNECTION_ID(s)',
+      description: 'Filter provided connection(s). Comma separated ids eg: id1,id2'
+    }),
+    file: flags.string({
+      required: true,
+      helpValue: 'Script Name',
+      parse: validateScriptFileName,
+      description: 'Name of the manual SQL/JS/TS script'
+    }),
+    'connection-resolver': flags.string({
+      helpValue: 'PATH',
+      description: 'Path to the connection resolver.'
+    }),
+    config: flags.string({
+      char: 'c',
+      description: 'Custom configuration file.'
+    })
+  };
+
+  /**
+   * Started event handler.
+   */
+  onStarted = async (result: OperationResult) => {
+    await printLine(bold(` ▸ ${result.connectionId}`));
+
+    await printInfo('   [✓] Manual script run - started');
+  };
+
+  /**
+   * Success handler.
+   */
+  onSuccess = async (result: OperationResult) => {
+    const log = dbLogger(result.connectionId);
+    const [num, list] = result.data;
+    const alreadyUpToDate = num && list.length === 0;
+
+    log('Up to date: ', alreadyUpToDate);
+
+    await printInfo(`   [✓] Manual script run - completed (${result.timeElapsed}s)`);
+
+    if (alreadyUpToDate) {
+      await printInfo('   Already up to date.\n');
+
+      return;
+    }
+
+    // Completed migrations.
+    for (const item of list) {
+      await printLine(cyan(`       - ${item}`));
+    }
+
+    await printInfo(`   Ran ${list.length} scripts.\n`);
+  };
+
+  /**
+   * Failure handler.
+   */
+  onFailed = async (result: OperationResult) => {
+    await printLine(bold(red(`   [✓] Manual script run - Failed\n`)));
+  };
+
+  /**
+   * CLI command execution handler.
+   *
+   * @returns {Promise<void>}
+   */
+  async run(): Promise<void> {
+    const { flags: parsedFlags } = this.parse(RunScript);
+    const isDryRun = parsedFlags['dry-run'];
+    const config = await loadConfig(parsedFlags.config);
+
+    const connections = await resolveConnections(config, parsedFlags['connection-resolver']);
+
+    if (isDryRun) await printLine(magenta('\n• DRY RUN STARTED\n'));
+
+    const results = await runScriptAPI(config, connections, {
+      ...parsedFlags,
+      onStarted: this.onStarted,
+      onSuccess: this.onSuccess,
+      onFailed: this.onFailed
+    });
+
+    const failedCount = results.filter(({ success }) => !success).length;
+
+    if (failedCount === 0) {
+      if (isDryRun) await printLine(magenta('• DRY RUN ENDED\n'));
+
+      return process.exit(0);
+    }
+
+    printError(`Error: Script failed for ${failedCount} connection(s).`);
+
+    if (isDryRun) await printLine(magenta('\n• DRY RUN ENDED\n'));
+
+    process.exit(-1);
+  }
+}
+
+export default RunScript;

src/config.ts

@@ -39,6 +39,16 @@ export function getSqlBasePath(config: Configuration): string {
   return path.join(config.basePath, 'sql');
 }
 
+/**
+ * Get manual scripts path from config.
+ *
+ * @param {Configuration} config
+ * @returns {string}
+ */
+export function getManualScriptBasePath(config: Configuration): string {
+  return path.join(config.basePath, config.manual.directory);
+}
+
 /**
  * Load config yaml file.
  *

src/constants.ts

@@ -28,6 +28,10 @@ export const DEFAULT_CONFIG: Configuration = {
     directory: 'migration',
     tableName: 'knex_migrations', // Note: This is Knex's default value. Just keeping it same.
     sourceType: 'sql'
+  },
+  manual: {
+    directory: 'manual',
+    tableName: 'manual_scripts'
   }
 };
 

src/domain/Configuration.ts

@@ -20,6 +20,10 @@ interface Configuration {
     tableName: string;
     sourceType: 'sql' | 'javascript' | 'typescript';
   };
+  manual: {
+    directory: string;
+    tableName: string;
+  };
 }
 
 export default Configuration;

src/domain/RunScriptContext.ts

@@ -0,0 +1,14 @@
+import { Knex } from 'knex';
+
+import { RunScriptParams } from './RunScriptParams';
+import OperationContext from './operation/OperationContext';
+
+export interface RunScriptContext extends OperationContext {
+  params: RunScriptParams;
+  migrateFunc: (
+    trx: Knex.Transaction,
+    files: string[],
+    connectionId: string,
+    runSQLScripts: (trx: Knex.Transaction, filteredScripts: string[]) => Promise<void>
+  ) => Promise<(number | string[])[]>;
+}

src/domain/RunScriptParams.ts

@@ -0,0 +1,5 @@
+import OperationParams from './operation/OperationParams';
+
+export interface RunScriptParams extends OperationParams {
+  file?: string;
+}

src/migration/service/knexMigrator.ts

@@ -123,3 +123,17 @@ export function getMigrationPath(config: Configuration): string {
 
   return migrationPath;
 }
+
+/**
+ * Get manual scripts directory path.
+ *
+ * @param {Configuration} config
+ * @returns {string}
+ */
+export function getManualScriptPath(config: Configuration): string {
+  const { basePath, manual } = config;
+
+  const scriptPath = path.isAbsolute(manual.directory) ? manual.directory : path.join(basePath, manual.directory);
+
+  return scriptPath;
+}

src/migration/service/migrator.ts

@@ -1,9 +1,12 @@
+import { Knex } from 'knex';
 import * as path from 'path';
 
 import { glob, exists } from '../../util/fs';
+import { getManualScriptPath } from './knexMigrator';
 import { resolveFile } from '../../service/sqlRunner';
 import FileExtensions from '../../enum/FileExtensions';
 import SqlMigrationEntry from '../domain/SqlMigrationEntry';
+import { RunScriptContext } from '../../domain/RunScriptContext';
 import JavaScriptMigrationEntry from '../domain/JavaScriptMigrationEntry';
 
 const FILE_PATTERN_JS = /(.+).js$/;
@@ -127,3 +130,55 @@ export async function resolveJavaScriptMigrations(
 
   return Promise.all(migrationPromises);
 }
+
+/**
+ * Run exposed function from manual JS/TS scripts.
+ *
+ * @param {Knex.Transaction} trx
+ * @param {RunScriptContext} context
+ * @param {string} connectionId
+ * @param {string[]} filteredScripts
+ * @param {string} extension
+ * @returns {Promise<void>}
+ */
+export async function runJSTSScripts(
+  trx: Knex.Transaction,
+  context: RunScriptContext,
+  connectionId: string,
+  filteredScripts: string[],
+  extension: string
+) {
+  const migrationNames = filteredScripts;
+
+  let mRequire: NodeRequire = require;
+
+  if (extension === FileExtensions.TS) {
+    // Transpile & execute ts files required on the fly
+    require('ts-node').register({
+      transpileOnly: true
+    });
+  } else {
+    // On the fly es6 => commonJS
+    mRequire = require('esm')(module);
+  }
+
+  const migrationPromises = migrationNames.map(async name => {
+    const { main } = mRequire(path.resolve(`${getManualScriptPath(context.config)}/${extension}`, name));
+
+    return {
+      main,
+      name
+    };
+  });
+
+  const fileFuncInfos = await Promise.all(migrationPromises);
+
+  if (!fileFuncInfos.length) {
+    return;
+  }
+
+  const func = fileFuncInfos[0].main;
+
+  // Execute the function
+  await func(trx, connectionId);
+}