feat: adds glob support for dir and ignorePattern

bin/node-pg-migrate.ts

@@ -48,6 +48,7 @@ const schemaArg = 'schema';
 const createSchemaArg = 'create-schema';
 const databaseUrlVarArg = 'database-url-var';
 const migrationsDirArg = 'migrations-dir';
+const useGlobArg = 'use-glob';
 const migrationsTableArg = 'migrations-table';
 const migrationsSchemaArg = 'migrations-schema';
 const createMigrationsSchemaArg = 'create-migrations-schema';
@@ -84,9 +85,14 @@ const parser = yargs(process.argv.slice(2))
     [migrationsDirArg]: {
       alias: 'm',
       defaultDescription: '"migrations"',
-      describe: 'The directory containing your migration files',
+      describe: `The directory name or glob pattern containing your migration files (resolved from cwd()). When using glob pattern, '${useGlobArg}' must be used as well`,
       type: 'string',
     },
+    [useGlobArg]: {
+      defaultDescription: 'false',
+      describe: `Use glob to find migration files. This will use '${migrationsDirArg}' _and_ '${ignorePatternArg}' to glob-search for migration files.`,
+      type: 'boolean',
+    },
     [migrationsTableArg]: {
       alias: 't',
       defaultDescription: '"pgmigrations"',
@@ -128,7 +134,7 @@ const parser = yargs(process.argv.slice(2))
     },
     [ignorePatternArg]: {
       defaultDescription: '"\\..*"',
-      describe: 'Regex pattern for file names to ignore',
+      describe: `Regex or glob pattern for migration files to be ignored. When using glob pattern, '${useGlobArg}' must be used as well`,
       type: 'string',
     },
     [decamelizeArg]: {
@@ -253,6 +259,7 @@ if (dotenv) {
 }
 
 let MIGRATIONS_DIR = argv[migrationsDirArg];
+let USE_GLOB = argv[useGlobArg];
 let DB_CONNECTION: string | ConnectionParameters | ClientConfig | undefined =
   process.env[argv[databaseUrlVarArg]];
 let IGNORE_PATTERN = argv[ignorePatternArg];
@@ -469,11 +476,11 @@ const action = argv._.shift();
 
 // defaults
 MIGRATIONS_DIR ??= join(cwd(), 'migrations');
+USE_GLOB ??= false;
 MIGRATIONS_FILE_LANGUAGE ??= 'js';
 MIGRATIONS_FILENAME_FORMAT ??= 'timestamp';
 MIGRATIONS_TABLE ??= 'pgmigrations';
 SCHEMA ??= ['public'];
-IGNORE_PATTERN ??= '\\..*';
 CHECK_ORDER ??= true;
 VERBOSE ??= true;
 
@@ -583,6 +590,7 @@ if (action === 'create') {
       },
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       dir: MIGRATIONS_DIR!,
+      useGlob: USE_GLOB,
       ignorePattern: IGNORE_PATTERN,
       schema: SCHEMA,
       createSchema: CREATE_SCHEMA,

docs/src/api.md

@@ -15,12 +15,13 @@ which takes options argument with the following structure (similar to [command l
 | `migrationsTable`        | `string`                                    | The table storing which migrations have been run                                                                                                                              |
 | `migrationsSchema`       | `string`                                    | The schema storing table which migrations have been run (defaults to same value as `schema`)                                                                                  |
 | `schema`                 | `string or array[string]`                   | The schema on which migration will be run (defaults to `public`)                                                                                                              |
-| `dir`                    | `string`                                    | The directory containing your migration files                                                                                                                                 |
+| `dir`                    | `string or array[string]`                   | The directory containing your migration files. This path is resolved from `cwd()`. Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or an array of glob patterns and set `useGlob = true`. Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns                                                                                                                                      |
+| `useGlob`                | `boolean`                                   | Use [glob](https://www.npmjs.com/package/glob) to find migration files. This will use `dir` _and_ `ignorePattern` to glob-search for migration files. Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns                                                                                                                                                                                                             |
 | `checkOrder`             | `boolean`                                   | Check order of migrations before running them                                                                                                                                 |
 | `direction`              | `enum`                                      | `up` or `down`                                                                                                                                                                |
 | `count`                  | `number`                                    | Amount of migration to run                                                                                                                                                    |
 | `timestamp`              | `boolean`                                   | Treats `count` as timestamp                                                                                                                                                   |
-| `ignorePattern`          | `string`                                    | Regex pattern for file names to ignore (ignores files starting with `.` by default)                                                                                           |
+| `ignorePattern`          | `string or array[string]`                   | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or an array of glob patterns and set `isGlob = true`.Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns                                                                                                                                        |
 | `file`                   | `string`                                    | Run-only migration with this name                                                                                                                                             |
 | `singleTransaction`      | `boolean`                                   | Combines all pending migrations into a single transaction so that if any migration fails, all will be rolled back (defaults to `true`)                                        |
 | `createSchema`           | `boolean`                                   | Creates the configured schema if it doesn't exist                                                                                                                             |

docs/src/cli.md

@@ -87,11 +87,12 @@ You can adjust defaults by passing arguments to `node-pg-migrate`:
 | `schema`                    | `s`     | `public`                        | The schema(s) on which migration will be run, used to set `search_path`                                                                                              |
 | `create-schema`             |         | `false`                         | Create the configured schema if it doesn't exist                                                                                                                     |
 | `database-url-var`          | `d`     | `DATABASE_URL`                  | Name of env variable with database url string                                                                                                                        |
-| `migrations-dir`            | `m`     | `migrations`                    | The directory containing your migration files                                                                                                                        |
+| `migrations-dir`            | `m`     | `migrations`                    | The directory containing your migration files. This path is resolved from `cwd()`. Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns                                                                                                                                                 |
+| `use-glob`                  |         | `false`                         | Use [glob](https://www.npmjs.com/package/glob) to find migration files. This will use `--migrations-dir` _and_ `--ignore-pattern` to glob-search for migration files.|
 | `migrations-schema`         |         | same value as `schema`          | The schema storing table which migrations have been run                                                                                                              |
 | `create-migrations-schema`  |         | `false`                         | Create the configured migrations schema if it doesn't exist                                                                                                          |
 | `migrations-table`          | `t`     | `pgmigrations`                  | The table storing which migrations have been run                                                                                                                     |
-| `ignore-pattern`            |         | `undefined`                     | Regex pattern for file names to ignore                                                                                                                               |
+| `ignore-pattern`            |         | `undefined`                     | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns                                                                                                                                                 |
 | `migration-filename-format` |         | `timestamp`                     | Choose prefix of file, `utc` (`20200605075829074`) or `timestamp` (`1591343909074`)                                                                                  |
 | `migration-file-language`   | `j`     | `js`                            | Language of the migration file to create (`js`, `ts` or `sql`)                                                                                                       |
 | `template-file-name`        |         | `undefined`                     | Utilize a custom migration template file with language inferred from its extension. The file should export the up method, accepting a MigrationBuilder instance.     |

package.json

@@ -97,6 +97,7 @@
     "url": "git+https://github.com/salsita/node-pg-migrate.git"
   },
   "dependencies": {
+    "glob": "11.0.0",
     "yargs": "~17.7.0"
   },
   "devDependencies": {

src/migration.ts

@@ -6,6 +6,7 @@
 
  */
 
+import { glob } from 'glob';
 import { createReadStream, createWriteStream } from 'node:fs';
 import { mkdir, readdir } from 'node:fs/promises';
 import { basename, extname, join, resolve } from 'node:path';
@@ -49,19 +50,67 @@ export type CreateOptions = {
 
 const SEPARATOR = '_';
 
+function compareStringsByValue(a: string, b: string): number {
+  return a.localeCompare(b, 'en', {
+    usage: 'sort',
+    numeric: true,
+    sensitivity: 'variant',
+  });
+}
+
+function compareFileNamesByTimestamp(
+  a: string,
+  b: string,
+  logger?: Logger
+): number {
+  const aTimestamp = getTimestamp(a, logger);
+  const bTimestamp = getTimestamp(b, logger);
+
+  return aTimestamp - bTimestamp;
+}
+
+// TODO should be renamed to make clear that this function doesn't actually load the files - it only reads their names / paths from `dir`
 export async function loadMigrationFiles(
-  dir: string,
-  ignorePattern?: string
+  dir: string | string[],
+  ignorePattern?: string | string[],
+  useGlob: boolean = false,
+  logger?: Logger
 ): Promise<string[]> {
+  if (useGlob) {
+    /**
+     * By default, a `**` in a pattern will follow 1 symbolic link if
+     * it is not the first item in the pattern, or none if it is the
+     * first item in the pattern, following the same behavior as Bash.
+     *
+     * only want files, no dirs.
+     */
+    const globMatches = await glob(dir, { ignore: ignorePattern, nodir: true });
+    return globMatches.sort(compareStringsByValue);
+  }
+
+  if (Array.isArray(dir) || Array.isArray(ignorePattern)) {
+    throw new TypeError(
+      'Options "dir" and "ignorePattern" can only be arrays when "useGlob" is true'
+    );
+  }
+
+  const ignoreRegexp = new RegExp(
+    ignorePattern?.length ? `^${ignorePattern}$` : '^\\..*'
+  );
+
   const dirContent = await readdir(`${dir}/`, { withFileTypes: true });
-  const files = dirContent
-    .map((file) => (file.isFile() || file.isSymbolicLink() ? file.name : null))
-    .filter((file): file is string => Boolean(file))
-    .sort();
-  const filter = new RegExp(`^(${ignorePattern})$`);
-  return ignorePattern === undefined
-    ? files
-    : files.filter((i) => !filter.test(i));
+  return dirContent
+    .filter(
+      (dirent) =>
+        (dirent.isFile() || dirent.isSymbolicLink()) &&
+        !ignoreRegexp.test(dirent.name)
+    )
+    .sort(
+      (a, b) =>
+        compareFileNamesByTimestamp(a.name, b.name, logger) ||
+        compareStringsByValue(a.name, b.name)
+    )
+    .map((dirent) => resolve(dir, dirent.name));
 }
 
 function getSuffixFromFileName(fileName: string): string {
@@ -82,7 +131,10 @@ async function getLastSuffix(
   }
 }
 
-export function getTimestamp(logger: Logger, filename: string): number {
+export function getTimestamp(
+  filename: string,
+  logger: Logger = console
+): number {
   const prefix = filename.split(SEPARATOR)[0];
   if (prefix && /^\d+$/.test(prefix)) {
     if (prefix.length === 13) {
@@ -187,7 +239,7 @@ export class Migration implements RunMigration {
     this.db = db;
     this.path = migrationPath;
     this.name = basename(migrationPath, extname(migrationPath));
-    this.timestamp = getTimestamp(logger, this.name);
+    this.timestamp = getTimestamp(this.name, logger);
     this.up = up;
     this.down = down;
     this.options = options;

src/runner.ts

@@ -32,11 +32,14 @@ async function loadMigrations(
 ): Promise<Migration[]> {
   try {
     let shorthands: ColumnDefinitions = {};
-    const files = await loadMigrationFiles(options.dir, options.ignorePattern);
+    const absoluteFilePaths = await loadMigrationFiles(
+      options.dir,
+      options.ignorePattern,
+      options.useGlob
+    );
 
     const migrations = await Promise.all(
-      files.map(async (file) => {
-        const filePath = resolve(options.dir, file);
+      absoluteFilePaths.map(async (filePath) => {
         const actions: MigrationBuilderActions =
           extname(filePath) === '.sql'
             ? await migrateSqlFile(filePath)
@@ -56,15 +59,7 @@ async function loadMigrations(
       })
     );
 
-    return migrations.sort((m1, m2) => {
-      const compare = m1.timestamp - m2.timestamp;
-
-      if (compare !== 0) {
-        return compare;
-      }
-
-      return m1.name.localeCompare(m2.name);
-    });
+    return migrations;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
   } catch (error: any) {
     throw new Error(`Can't get migration files: ${error.stack}`);

src/types.ts

@@ -783,9 +783,23 @@ export interface RunnerOptionConfig {
   schema?: string | string[];
 
   /**
-   * The directory containing your migration files.
+   * The directory containing your migration files. This path is resolved from `cwd()`.
+   * Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or
+   * an array of glob patterns and set `useGlob = true`
+   *
+   * Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns
    */
-  dir: string;
+  dir: string | string[];
+
+  /**
+   * Use [glob](https://www.npmjs.com/package/glob) to find migration files.
+   * This will use `dir` _and_ `ignorePattern` to glob-search for migration files.
+   *
+   * Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns
+   *
+   * @default: false
+   */
+  useGlob?: boolean;
 
   /**
    * Check order of migrations before running them.
@@ -809,8 +823,12 @@ export interface RunnerOptionConfig {
 
   /**
    * Regex pattern for file names to ignore (ignores files starting with `.` by default).
+   * Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or
+   * an array of glob patterns and set `isGlob = true`
+   *
+   * Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns
    */
-  ignorePattern?: string;
+  ignorePattern?: string | string[];
 
   /**
    * Run only migration with this name.