diff --git a/bin/node-pg-migrate.ts b/bin/node-pg-migrate.ts index 11c243d3..34863bf3 100755 --- a/bin/node-pg-migrate.ts +++ b/bin/node-pg-migrate.ts @@ -48,6 +48,7 @@ const schemaArg = 'schema'; const createSchemaArg = 'create-schema'; const databaseUrlVarArg = 'database-url-var'; const migrationsDirArg = 'migrations-dir'; +const useGlobArg = 'use-glob'; const migrationsTableArg = 'migrations-table'; const migrationsSchemaArg = 'migrations-schema'; const createMigrationsSchemaArg = 'create-migrations-schema'; @@ -84,9 +85,14 @@ const parser = yargs(process.argv.slice(2)) [migrationsDirArg]: { alias: 'm', defaultDescription: '"migrations"', - describe: 'The directory containing your migration files', + describe: `The directory name or glob pattern containing your migration files (resolved from cwd()). When using glob pattern, '${useGlobArg}' must be used as well`, type: 'string', }, + [useGlobArg]: { + defaultDescription: 'false', + describe: `Use glob to find migration files. This will use '${migrationsDirArg}' _and_ '${ignorePatternArg}' to glob-search for migration files.`, + type: 'boolean', + }, [migrationsTableArg]: { alias: 't', defaultDescription: '"pgmigrations"', @@ -128,7 +134,7 @@ const parser = yargs(process.argv.slice(2)) }, [ignorePatternArg]: { defaultDescription: '"\\..*"', - describe: 'Regex pattern for file names to ignore', + describe: `Regex or glob pattern for migration files to be ignored. When using glob pattern, '${useGlobArg}' must be used as well`, type: 'string', }, [decamelizeArg]: { @@ -253,6 +259,7 @@ if (dotenv) { } let MIGRATIONS_DIR = argv[migrationsDirArg]; +let USE_GLOB = argv[useGlobArg]; let DB_CONNECTION: string | ConnectionParameters | ClientConfig | undefined = process.env[argv[databaseUrlVarArg]]; let IGNORE_PATTERN = argv[ignorePatternArg]; @@ -469,11 +476,11 @@ const action = argv._.shift(); // defaults MIGRATIONS_DIR ??= join(cwd(), 'migrations'); +USE_GLOB ??= false; MIGRATIONS_FILE_LANGUAGE ??= 'js'; MIGRATIONS_FILENAME_FORMAT ??= 'timestamp'; MIGRATIONS_TABLE ??= 'pgmigrations'; SCHEMA ??= ['public']; -IGNORE_PATTERN ??= '\\..*'; CHECK_ORDER ??= true; VERBOSE ??= true; @@ -583,6 +590,7 @@ if (action === 'create') { }, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion dir: MIGRATIONS_DIR!, + useGlob: USE_GLOB, ignorePattern: IGNORE_PATTERN, schema: SCHEMA, createSchema: CREATE_SCHEMA, diff --git a/docs/src/api.md b/docs/src/api.md index 90540c4e..dfd24ce5 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -15,12 +15,13 @@ which takes options argument with the following structure (similar to [command l | `migrationsTable` | `string` | The table storing which migrations have been run | | `migrationsSchema` | `string` | The schema storing table which migrations have been run (defaults to same value as `schema`) | | `schema` | `string or array[string]` | The schema on which migration will be run (defaults to `public`) | -| `dir` | `string` | The directory containing your migration files | +| `dir` | `string or array[string]` | The directory containing your migration files. This path is resolved from `cwd()`. Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or an array of glob patterns and set `useGlob = true`. Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns | +| `useGlob` | `boolean` | Use [glob](https://www.npmjs.com/package/glob) to find migration files. This will use `dir` _and_ `ignorePattern` to glob-search for migration files. Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns | | `checkOrder` | `boolean` | Check order of migrations before running them | | `direction` | `enum` | `up` or `down` | | `count` | `number` | Amount of migration to run | | `timestamp` | `boolean` | Treats `count` as timestamp | -| `ignorePattern` | `string` | Regex pattern for file names to ignore (ignores files starting with `.` by default) | +| `ignorePattern` | `string or array[string]` | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or an array of glob patterns and set `isGlob = true`.Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns | | `file` | `string` | Run-only migration with this name | | `singleTransaction` | `boolean` | Combines all pending migrations into a single transaction so that if any migration fails, all will be rolled back (defaults to `true`) | | `createSchema` | `boolean` | Creates the configured schema if it doesn't exist | diff --git a/docs/src/cli.md b/docs/src/cli.md index 4a264bb9..ea5e25a8 100644 --- a/docs/src/cli.md +++ b/docs/src/cli.md @@ -87,11 +87,12 @@ You can adjust defaults by passing arguments to `node-pg-migrate`: | `schema` | `s` | `public` | The schema(s) on which migration will be run, used to set `search_path` | | `create-schema` | | `false` | Create the configured schema if it doesn't exist | | `database-url-var` | `d` | `DATABASE_URL` | Name of env variable with database url string | -| `migrations-dir` | `m` | `migrations` | The directory containing your migration files | +| `migrations-dir` | `m` | `migrations` | The directory containing your migration files. This path is resolved from `cwd()`. Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns | +| `use-glob` | | `false` | Use [glob](https://www.npmjs.com/package/glob) to find migration files. This will use `--migrations-dir` _and_ `--ignore-pattern` to glob-search for migration files.| | `migrations-schema` | | same value as `schema` | The schema storing table which migrations have been run | | `create-migrations-schema` | | `false` | Create the configured migrations schema if it doesn't exist | | `migrations-table` | `t` | `pgmigrations` | The table storing which migrations have been run | -| `ignore-pattern` | | `undefined` | Regex pattern for file names to ignore | +| `ignore-pattern` | | `undefined` | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns | | `migration-filename-format` | | `timestamp` | Choose prefix of file, `utc` (`20200605075829074`) or `timestamp` (`1591343909074`) | | `migration-file-language` | `j` | `js` | Language of the migration file to create (`js`, `ts` or `sql`) | | `template-file-name` | | `undefined` | Utilize a custom migration template file with language inferred from its extension. The file should export the up method, accepting a MigrationBuilder instance. | diff --git a/package.json b/package.json index eff94aa6..bed7f44a 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "url": "git+https://github.com/salsita/node-pg-migrate.git" }, "dependencies": { + "glob": "11.0.0", "yargs": "~17.7.0" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17fa7b25..c206d67c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + glob: + specifier: 11.0.0 + version: 11.0.0 yargs: specifier: ~17.7.0 version: 17.7.2 diff --git a/src/migration.ts b/src/migration.ts index 6900b0f2..ee4e24b8 100644 --- a/src/migration.ts +++ b/src/migration.ts @@ -6,6 +6,7 @@ */ +import { glob } from 'glob'; import { createReadStream, createWriteStream } from 'node:fs'; import { mkdir, readdir } from 'node:fs/promises'; import { basename, extname, join, resolve } from 'node:path'; @@ -49,19 +50,67 @@ export type CreateOptions = { const SEPARATOR = '_'; +function compareStringsByValue(a: string, b: string): number { + return a.localeCompare(b, 'en', { + usage: 'sort', + numeric: true, + sensitivity: 'variant', + }); +} + +function compareFileNamesByTimestamp( + a: string, + b: string, + logger?: Logger +): number { + const aTimestamp = getTimestamp(a, logger); + const bTimestamp = getTimestamp(b, logger); + + return aTimestamp - bTimestamp; +} + +// TODO should be renamed to make clear that this function doesn't actually load the files - it only reads their names / paths from `dir` export async function loadMigrationFiles( - dir: string, - ignorePattern?: string + dir: string | string[], + ignorePattern?: string | string[], + useGlob: boolean = false, + logger?: Logger ): Promise { + if (useGlob) { + /** + * By default, a `**` in a pattern will follow 1 symbolic link if + * it is not the first item in the pattern, or none if it is the + * first item in the pattern, following the same behavior as Bash. + * + * only want files, no dirs. + */ + const globMatches = await glob(dir, { ignore: ignorePattern, nodir: true }); + return globMatches.sort(compareStringsByValue); + } + + if (Array.isArray(dir) || Array.isArray(ignorePattern)) { + throw new TypeError( + 'Options "dir" and "ignorePattern" can only be arrays when "useGlob" is true' + ); + } + + const ignoreRegexp = new RegExp( + ignorePattern?.length ? `^${ignorePattern}$` : '^\\..*' + ); + const dirContent = await readdir(`${dir}/`, { withFileTypes: true }); - const files = dirContent - .map((file) => (file.isFile() || file.isSymbolicLink() ? file.name : null)) - .filter((file): file is string => Boolean(file)) - .sort(); - const filter = new RegExp(`^(${ignorePattern})$`); - return ignorePattern === undefined - ? files - : files.filter((i) => !filter.test(i)); + return dirContent + .filter( + (dirent) => + (dirent.isFile() || dirent.isSymbolicLink()) && + !ignoreRegexp.test(dirent.name) + ) + .sort( + (a, b) => + compareFileNamesByTimestamp(a.name, b.name, logger) || + compareStringsByValue(a.name, b.name) + ) + .map((dirent) => resolve(dir, dirent.name)); } function getSuffixFromFileName(fileName: string): string { @@ -82,7 +131,10 @@ async function getLastSuffix( } } -export function getTimestamp(logger: Logger, filename: string): number { +export function getTimestamp( + filename: string, + logger: Logger = console +): number { const prefix = filename.split(SEPARATOR)[0]; if (prefix && /^\d+$/.test(prefix)) { if (prefix.length === 13) { @@ -187,7 +239,7 @@ export class Migration implements RunMigration { this.db = db; this.path = migrationPath; this.name = basename(migrationPath, extname(migrationPath)); - this.timestamp = getTimestamp(logger, this.name); + this.timestamp = getTimestamp(this.name, logger); this.up = up; this.down = down; this.options = options; diff --git a/src/runner.ts b/src/runner.ts index 729da38d..a6b382b8 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -32,11 +32,14 @@ async function loadMigrations( ): Promise { try { let shorthands: ColumnDefinitions = {}; - const files = await loadMigrationFiles(options.dir, options.ignorePattern); + const absoluteFilePaths = await loadMigrationFiles( + options.dir, + options.ignorePattern, + options.useGlob + ); const migrations = await Promise.all( - files.map(async (file) => { - const filePath = resolve(options.dir, file); + absoluteFilePaths.map(async (filePath) => { const actions: MigrationBuilderActions = extname(filePath) === '.sql' ? await migrateSqlFile(filePath) @@ -56,15 +59,7 @@ async function loadMigrations( }) ); - return migrations.sort((m1, m2) => { - const compare = m1.timestamp - m2.timestamp; - - if (compare !== 0) { - return compare; - } - - return m1.name.localeCompare(m2.name); - }); + return migrations; // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (error: any) { throw new Error(`Can't get migration files: ${error.stack}`); diff --git a/src/types.ts b/src/types.ts index d0b59684..3ed4ee52 100644 --- a/src/types.ts +++ b/src/types.ts @@ -783,9 +783,23 @@ export interface RunnerOptionConfig { schema?: string | string[]; /** - * The directory containing your migration files. + * The directory containing your migration files. This path is resolved from `cwd()`. + * Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or + * an array of glob patterns and set `useGlob = true` + * + * Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns */ - dir: string; + dir: string | string[]; + + /** + * Use [glob](https://www.npmjs.com/package/glob) to find migration files. + * This will use `dir` _and_ `ignorePattern` to glob-search for migration files. + * + * Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns + * + * @default: false + */ + useGlob?: boolean; /** * Check order of migrations before running them. @@ -809,8 +823,12 @@ export interface RunnerOptionConfig { /** * Regex pattern for file names to ignore (ignores files starting with `.` by default). + * Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or + * an array of glob patterns and set `isGlob = true` + * + * Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns */ - ignorePattern?: string; + ignorePattern?: string | string[]; /** * Run only migration with this name. diff --git a/test/migration.spec.ts b/test/migration.spec.ts index a2e2ab59..ec484f04 100644 --- a/test/migration.spec.ts +++ b/test/migration.spec.ts @@ -1,7 +1,8 @@ +import { resolve } from 'node:path'; import type { Mock } from 'vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { DBConnection } from '../src/db'; -import { getTimestamp, Migration } from '../src/migration'; +import { getTimestamp, loadMigrationFiles, Migration } from '../src/migration'; import type { Logger, RunnerOption } from '../src/types'; const callbackMigration = '1414549381268_names.js'; @@ -33,18 +34,83 @@ describe('migration', () => { it('should get timestamp for normal timestamp', () => { const now = Date.now(); - expect(getTimestamp(logger, String(now))).toBe(now); + expect(getTimestamp(String(now), logger)).toBe(now); }); it('should get timestamp for shortened iso format', () => { const now = new Date(); - expect(getTimestamp(logger, now.toISOString().replace(/\D/g, ''))).toBe( + expect(getTimestamp(now.toISOString().replace(/\D/g, ''), logger)).toBe( now.valueOf() ); }); }); + describe('loadMigrationFiles', () => { + it('should resolve files directly in `dir`', async () => { + const dir = 'test/migrations'; + const resolvedDir = resolve(dir); + const filePaths = await loadMigrationFiles( + dir, + undefined, + undefined, + logger + ); + + expect(Array.isArray(filePaths)).toBeTruthy(); + expect(filePaths).toHaveLength(91); + expect(filePaths).not.toContainEqual(expect.stringContaining('nested')); + + for (const filePath of filePaths) { + expect(filePath).toMatch(resolvedDir); + expect(filePath).toMatch(/\.js$/); + } + }); + + it('should resolve files directly in `dir` and ignore matching ignorePattern', async () => { + const dir = 'test/migrations'; + // ignores those files that have `test` in their name (not in the path, just filename) + const ignorePattern = '.+test.+'; + + const filePaths = await loadMigrationFiles( + dir, + ignorePattern, + undefined, + logger + ); + + expect(Array.isArray(filePaths)).toBeTruthy(); + expect(filePaths).toHaveLength(66); + }); + + it('should resolve files matching `dir` glob (starting from cwd())', async () => { + const dir = 'test/{cockroach,migrations}/**'; + + const filePaths = await loadMigrationFiles(dir, undefined, true, logger); + + expect(Array.isArray(filePaths)).toBeTruthy(); + expect(filePaths).toHaveLength(104); + expect(filePaths).toContainEqual(expect.stringContaining('nested')); + }); + + it('should resolve files matching `dir` glob (starting from cwd()) and ignore matching ignorePattern', async () => { + const dir = 'test/{cockroach,migrations}/**'; + // ignores those files that have `test` in their name (not in the path, just filename) + const ignorePattern = '*/cockroach/*test*'; + + const filePaths = await loadMigrationFiles( + dir, + ignorePattern, + true, + logger + ); + + expect(Array.isArray(filePaths)).toBeTruthy(); + expect(filePaths).toHaveLength(103); + expect(filePaths).toContainEqual(expect.stringContaining('nested')); + }); + }); + describe('self.applyUp', () => { it('should call db.query on normal operations', async () => { const migration = new Migration( diff --git a/test/migrations/nested/001_nested_noop.js b/test/migrations/nested/001_nested_noop.js new file mode 100644 index 00000000..fcb02f0c --- /dev/null +++ b/test/migrations/nested/001_nested_noop.js @@ -0,0 +1 @@ +exports.up = () => {};