Skip to content

Commit

Permalink
feat: adds glob support for dir and ignorePattern
Browse files Browse the repository at this point in the history 
Signed-off-by: Benjamin Kroeger <[email protected]>
  • Loading branch information
@benkroeger
benkroeger committed Sep 18, 2024
1 parent f47e927 commit 42f98e6
Show file tree
Hide file tree
Showing 10 changed files with 183 additions and 37 deletions.
14 changes: 11 additions & 3 deletions bin/node-pg-migrate.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ const schemaArg = 'schema';
const createSchemaArg = 'create-schema';
const databaseUrlVarArg = 'database-url-var';
const migrationsDirArg = 'migrations-dir';
const useGlobArg = 'use-glob';
const migrationsTableArg = 'migrations-table';
const migrationsSchemaArg = 'migrations-schema';
const createMigrationsSchemaArg = 'create-migrations-schema';
Expand Down Expand Up @@ -84,9 +85,14 @@ const parser = yargs(process.argv.slice(2))
[migrationsDirArg]: {
alias: 'm',
defaultDescription: '"migrations"',
describe: 'The directory containing your migration files',
describe: `The directory name or glob pattern containing your migration files (resolved from cwd()). When using glob pattern, '${useGlobArg}' must be used as well`,
type: 'string',
},
[useGlobArg]: {
defaultDescription: 'false',
describe: `Use glob to find migration files. This will use '${migrationsDirArg}' _and_ '${ignorePatternArg}' to glob-search for migration files.`,
type: 'boolean',
},
[migrationsTableArg]: {
alias: 't',
defaultDescription: '"pgmigrations"',
Expand Down Expand Up @@ -128,7 +134,7 @@ const parser = yargs(process.argv.slice(2))
},
[ignorePatternArg]: {
defaultDescription: '"\\..*"',
describe: 'Regex pattern for file names to ignore',
describe: `Regex or glob pattern for migration files to be ignored. When using glob pattern, '${useGlobArg}' must be used as well`,
type: 'string',
},
[decamelizeArg]: {
Expand Down Expand Up @@ -253,6 +259,7 @@ if (dotenv) {
}

let MIGRATIONS_DIR = argv[migrationsDirArg];
let USE_GLOB = argv[useGlobArg];
let DB_CONNECTION: string | ConnectionParameters | ClientConfig | undefined =
process.env[argv[databaseUrlVarArg]];
let IGNORE_PATTERN = argv[ignorePatternArg];
Expand Down Expand Up @@ -469,11 +476,11 @@ const action = argv._.shift();

// defaults
MIGRATIONS_DIR ??= join(cwd(), 'migrations');
USE_GLOB ??= false;
MIGRATIONS_FILE_LANGUAGE ??= 'js';
MIGRATIONS_FILENAME_FORMAT ??= 'timestamp';
MIGRATIONS_TABLE ??= 'pgmigrations';
SCHEMA ??= ['public'];
IGNORE_PATTERN ??= '\\..*';
CHECK_ORDER ??= true;
VERBOSE ??= true;

Expand Down Expand Up @@ -583,6 +590,7 @@ if (action === 'create') {
},
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
dir: MIGRATIONS_DIR!,
useGlob: USE_GLOB,
ignorePattern: IGNORE_PATTERN,
schema: SCHEMA,
createSchema: CREATE_SCHEMA,
Expand Down
5 changes: 3 additions & 2 deletions docs/src/api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@ which takes options argument with the following structure (similar to [command l
| `migrationsTable` | `string` | The table storing which migrations have been run |
| `migrationsSchema` | `string` | The schema storing table which migrations have been run (defaults to same value as `schema`) |
| `schema` | `string or array[string]` | The schema on which migration will be run (defaults to `public`) |
| `dir` | `string` | The directory containing your migration files |
| `dir` | `string or array[string]` | The directory containing your migration files. This path is resolved from `cwd()`. Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or an array of glob patterns and set `useGlob = true`. Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns |
| `useGlob` | `boolean` | Use [glob](https://www.npmjs.com/package/glob) to find migration files. This will use `dir` _and_ `ignorePattern` to glob-search for migration files. Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns |
| `checkOrder` | `boolean` | Check order of migrations before running them |
| `direction` | `enum` | `up` or `down` |
| `count` | `number` | Amount of migration to run |
| `timestamp` | `boolean` | Treats `count` as timestamp |
| `ignorePattern` | `string` | Regex pattern for file names to ignore (ignores files starting with `.` by default) |
| `ignorePattern` | `string or array[string]` | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or an array of glob patterns and set `isGlob = true`.Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns |
| `file` | `string` | Run-only migration with this name |
| `singleTransaction` | `boolean` | Combines all pending migrations into a single transaction so that if any migration fails, all will be rolled back (defaults to `true`) |
| `createSchema` | `boolean` | Creates the configured schema if it doesn't exist |
Expand Down
5 changes: 3 additions & 2 deletions docs/src/cli.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,12 @@ You can adjust defaults by passing arguments to `node-pg-migrate`:
| `schema` | `s` | `public` | The schema(s) on which migration will be run, used to set `search_path` |
| `create-schema` | | `false` | Create the configured schema if it doesn't exist |
| `database-url-var` | `d` | `DATABASE_URL` | Name of env variable with database url string |
| `migrations-dir` | `m` | `migrations` | The directory containing your migration files |
| `migrations-dir` | `m` | `migrations` | The directory containing your migration files. This path is resolved from `cwd()`. Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns |
| `use-glob` | | `false` | Use [glob](https://www.npmjs.com/package/glob) to find migration files. This will use `--migrations-dir` _and_ `--ignore-pattern` to glob-search for migration files.|
| `migrations-schema` | | same value as `schema` | The schema storing table which migrations have been run |
| `create-migrations-schema` | | `false` | Create the configured migrations schema if it doesn't exist |
| `migrations-table` | `t` | `pgmigrations` | The table storing which migrations have been run |
| `ignore-pattern` | | `undefined` | Regex pattern for file names to ignore |
| `ignore-pattern` | | `undefined` | Regex pattern for file names to ignore (ignores files starting with `.` by default). Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern and set `--use-glob`. Note: enabling glob will read both, `--migrations-dir` _and_ `--ignore-pattern` as glob patterns |
| `migration-filename-format` | | `timestamp` | Choose prefix of file, `utc` (`20200605075829074`) or `timestamp` (`1591343909074`) |
| `migration-file-language` | `j` | `js` | Language of the migration file to create (`js`, `ts` or `sql`) |
| `template-file-name` | | `undefined` | Utilize a custom migration template file with language inferred from its extension. The file should export the up method, accepting a MigrationBuilder instance. |
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@
"url": "git+https://github.com/salsita/node-pg-migrate.git"
},
"dependencies": {
"glob": "11.0.0",
"yargs": "~17.7.0"
},
"devDependencies": {
Expand Down
3 changes: 3 additions & 0 deletions pnpm-lock.yaml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

76 changes: 64 additions & 12 deletions src/migration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/

import { glob } from 'glob';
import { createReadStream, createWriteStream } from 'node:fs';
import { mkdir, readdir } from 'node:fs/promises';
import { basename, extname, join, resolve } from 'node:path';
Expand Down Expand Up @@ -49,19 +50,67 @@ export type CreateOptions = {

const SEPARATOR = '_';

function compareStringsByValue(a: string, b: string): number {
return a.localeCompare(b, 'en', {
usage: 'sort',
numeric: true,
sensitivity: 'variant',
});
}

function compareFileNamesByTimestamp(
a: string,
b: string,
logger?: Logger
): number {
const aTimestamp = getTimestamp(a, logger);
const bTimestamp = getTimestamp(b, logger);

return aTimestamp - bTimestamp;
}

// TODO should be renamed to make clear that this function doesn't actually load the files - it only reads their names / paths from `dir`
export async function loadMigrationFiles(
dir: string,
ignorePattern?: string
dir: string | string[],
ignorePattern?: string | string[],
useGlob: boolean = false,
logger?: Logger
): Promise<string[]> {
if (useGlob) {
/**
* By default, a `**` in a pattern will follow 1 symbolic link if
* it is not the first item in the pattern, or none if it is the
* first item in the pattern, following the same behavior as Bash.
*
* only want files, no dirs.
*/
const globMatches = await glob(dir, { ignore: ignorePattern, nodir: true });
return globMatches.sort(compareStringsByValue);
}

if (Array.isArray(dir) || Array.isArray(ignorePattern)) {
throw new TypeError(
'Options "dir" and "ignorePattern" can only be arrays when "useGlob" is true'
);
}

const ignoreRegexp = new RegExp(
ignorePattern?.length ? `^${ignorePattern}$` : '^\\..*'
);

const dirContent = await readdir(`${dir}/`, { withFileTypes: true });
const files = dirContent
.map((file) => (file.isFile() || file.isSymbolicLink() ? file.name : null))
.filter((file): file is string => Boolean(file))
.sort();
const filter = new RegExp(`^(${ignorePattern})$`);
return ignorePattern === undefined
? files
: files.filter((i) => !filter.test(i));
return dirContent
.filter(
(dirent) =>
(dirent.isFile() || dirent.isSymbolicLink()) &&
!ignoreRegexp.test(dirent.name)
)
.sort(
(a, b) =>
compareFileNamesByTimestamp(a.name, b.name, logger) ||
compareStringsByValue(a.name, b.name)
)
.map((dirent) => resolve(dir, dirent.name));
}

function getSuffixFromFileName(fileName: string): string {
Expand All @@ -82,7 +131,10 @@ async function getLastSuffix(
}
}

export function getTimestamp(logger: Logger, filename: string): number {
export function getTimestamp(
filename: string,
logger: Logger = console
): number {
const prefix = filename.split(SEPARATOR)[0];
if (prefix && /^\d+$/.test(prefix)) {
if (prefix.length === 13) {
Expand Down Expand Up @@ -187,7 +239,7 @@ export class Migration implements RunMigration {
this.db = db;
this.path = migrationPath;
this.name = basename(migrationPath, extname(migrationPath));
this.timestamp = getTimestamp(logger, this.name);
this.timestamp = getTimestamp(this.name, logger);
this.up = up;
this.down = down;
this.options = options;
Expand Down
19 changes: 7 additions & 12 deletions src/runner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,14 @@ async function loadMigrations(
): Promise<Migration[]> {
try {
let shorthands: ColumnDefinitions = {};
const files = await loadMigrationFiles(options.dir, options.ignorePattern);
const absoluteFilePaths = await loadMigrationFiles(
options.dir,
options.ignorePattern,
options.useGlob
);

const migrations = await Promise.all(
files.map(async (file) => {
const filePath = resolve(options.dir, file);
absoluteFilePaths.map(async (filePath) => {
const actions: MigrationBuilderActions =
extname(filePath) === '.sql'
? await migrateSqlFile(filePath)
Expand All @@ -56,15 +59,7 @@ async function loadMigrations(
})
);

return migrations.sort((m1, m2) => {
const compare = m1.timestamp - m2.timestamp;

if (compare !== 0) {
return compare;
}

return m1.name.localeCompare(m2.name);
});
return migrations;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (error: any) {
throw new Error(`Can't get migration files: ${error.stack}`);
Expand Down
24 changes: 21 additions & 3 deletions src/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -783,9 +783,23 @@ export interface RunnerOptionConfig {
schema?: string | string[];

/**
* The directory containing your migration files.
* The directory containing your migration files. This path is resolved from `cwd()`.
* Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or
* an array of glob patterns and set `useGlob = true`
*
* Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns
*/
dir: string;
dir: string | string[];

/**
* Use [glob](https://www.npmjs.com/package/glob) to find migration files.
* This will use `dir` _and_ `ignorePattern` to glob-search for migration files.
*
* Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns
*
* @default: false
*/
useGlob?: boolean;

/**
* Check order of migrations before running them.
Expand All @@ -809,8 +823,12 @@ export interface RunnerOptionConfig {

/**
* Regex pattern for file names to ignore (ignores files starting with `.` by default).
* Alternatively, provide a [glob](https://www.npmjs.com/package/glob) pattern or
* an array of glob patterns and set `isGlob = true`
*
* Note: enabling glob will read both, `dir` _and_ `ignorePattern` as glob patterns
*/
ignorePattern?: string;
ignorePattern?: string | string[];

/**
* Run only migration with this name.
Expand Down
Loading

0 comments on commit 42f98e6

Please sign in to comment.