feat: use kysely for handling migrations & add scripts

apps/scripts/README.md

@@ -0,0 +1,112 @@
+# grants-stack-indexer: scripts
+
+This package contains scripts for managing the database schema and migrations.
+
+## Available Scripts
+
+| Script              | Description                             |
+| ------------------- | --------------------------------------- |
+| `script:db:migrate` | Runs all pending database migrations    |
+| `script:db:reset`   | Drops and recreates the database schema |
+
+## Environment Setup
+
+1. Create a `.env` file in the `apps/scripts` directory:
+
+```env
+# Database connection URL
+DATABASE_URL=postgresql://user:password@localhost:5432/mydb
+
+# Schema name to manage
+DATABASE_SCHEMA=grants_stack
+```
+
+### Environment Variables
+
+| Variable          | Description               | Example                                          |
+| ----------------- | ------------------------- | ------------------------------------------------ |
+| `DATABASE_URL`    | PostgreSQL connection URL | `postgresql://user:password@localhost:5432/mydb` |
+| `DATABASE_SCHEMA` | Database schema name      | `grants_stack`                                   |
+
+## Usage
+
+First, install dependencies:
+
+```bash
+pnpm install
+```
+
+### Running Migrations
+
+To apply all pending migrations:
+
+```bash
+pnpm script:db:migrate
+```
+
+This will:
+
+1. Load environment variables
+2. Connect to the database
+3. Create the schema if it doesn't exist
+4. Run any pending migrations
+5. Log the results
+
+### Resetting the Database
+
+To completely reset the database schema:
+
+```bash
+pnpm script:db:reset
+```
+
+**Warning**: This will:
+
+1. Drop the existing schema and all its data
+2. Recreate an empty schema
+3. You'll need to run migrations again after reset
+
+## Development
+
+### Adding New Migrations
+
+1. Create a new migration file in [`packages/repository/src/migrations`](../../packages//repository//migrations)
+2. Name it using the format: `YYYYMMDDTHHmmss_description.ts`
+3. Implement the `up` and `down` functions
+4. Run `pnpm script:db:migrate` to apply the new migration
+
+Example migration file:
+
+```typescript
+import { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+    // Your migration code here
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+    // Code to reverse the migration
+}
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Connection Error**
+
+    - Check if PostgreSQL is running
+    - Verify DATABASE_URL is correct
+    - Ensure the database exists
+
+2. **Permission Error**
+
+    - Verify user has necessary permissions
+    - Check schema ownership
+
+3. **Migration Failed**
+    - Check migration logs
+    - Ensure no conflicting changes
+    - Verify schema consistency
+
+TODO: add E2E tests for the scripts

apps/scripts/package.json

@@ -0,0 +1,36 @@
+{
+    "name": "@grants-stack-indexer/scripts",
+    "version": "0.0.1",
+    "private": true,
+    "description": "",
+    "license": "MIT",
+    "author": "Wonderland",
+    "type": "module",
+    "directories": {
+        "src": "src"
+    },
+    "files": [
+        "package.json"
+    ],
+    "scripts": {
+        "build": "tsc -p tsconfig.build.json",
+        "check-types": "tsc --noEmit -p ./tsconfig.json",
+        "clean": "rm -rf dist/",
+        "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"",
+        "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"",
+        "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"",
+        "lint:fix": "pnpm lint --fix",
+        "script:db:migrate": "tsx src/migrateDb.script.ts",
+        "script:db:reset": "tsx src/resetDb.script.ts",
+        "test": "vitest run --config vitest.config.ts --passWithNoTests",
+        "test:cov": "vitest run --config vitest.config.ts --coverage --passWithNoTests"
+    },
+    "dependencies": {
+        "@grants-stack-indexer/repository": "workspace:*",
+        "dotenv": "16.4.5",
+        "zod": "3.23.8"
+    },
+    "devDependencies": {
+        "tsx": "4.19.2"
+    }
+}

apps/scripts/src/migrateDb.script.ts

@@ -0,0 +1,69 @@
+import { configDotenv } from "dotenv";
+
+import { createKyselyDatabase, migrateToLatest } from "@grants-stack-indexer/repository";
+
+import { getDatabaseConfigFromEnv } from "./schemas/index.js";
+
+configDotenv();
+
+/**
+ * This script handles database migrations for the grants-stack-indexer project.
+ *
+ * It performs the following steps:
+ * 1. Loads environment variables from .env file
+ * 2. Gets database configuration (URL and schema name) from environment
+ * 3. Creates a Kysely database connection with the specified schema
+ * 4. Runs any pending migrations from packages/repository/migrations
+ * 5. Reports success/failure of migrations
+ * 6. Closes database connection and exits
+ *
+ * Environment variables required:
+ * - DATABASE_URL: PostgreSQL connection string
+ * - DATABASE_SCHEMA: Schema name to migrate (e.g. "grants_stack")
+ *
+ * The script will:
+ * - Create the schema if it doesn't exist
+ * - Run all pending migrations in order
+ * - Log results of each migration
+ * - Exit with code 0 on success, 1 on failure
+ */
+
+export const main = async (): Promise<void> => {
+    const { DATABASE_URL, DATABASE_SCHEMA } = getDatabaseConfigFromEnv();
+
+    const db = createKyselyDatabase({
+        connectionString: DATABASE_URL,
+        withSchema: DATABASE_SCHEMA,
+    });
+
+    console.log(`Migrating database schema '${DATABASE_SCHEMA}'...`);
+
+    const migrationResults = await migrateToLatest({
+        db,
+        schema: DATABASE_SCHEMA,
+    });
+
+    if (migrationResults && migrationResults?.length > 0) {
+        const failedMigrations = migrationResults.filter(
+            (migrationResult) => migrationResult.status === "Error",
+        );
+
+        if (failedMigrations.length > 0) {
+            console.error("❌ Failed migrations:", failedMigrations);
+            throw new Error("Failed migrations");
+        }
+
+        console.log(`✅ Migrations applied successfully`);
+    } else {
+        console.log("No migrations to apply");
+    }
+
+    await db.destroy();
+
+    process.exit(0);
+};
+
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

apps/scripts/src/resetDb.script.ts

@@ -0,0 +1,70 @@
+import { configDotenv } from "dotenv";
+
+import { createKyselyDatabase, resetDatabase } from "@grants-stack-indexer/repository";
+
+import { getDatabaseConfigFromEnv } from "./schemas/index.js";
+
+configDotenv();
+
+/**
+ * This script handles database reset for the grants-stack-indexer project.
+ *
+ * It performs the following steps:
+ * 1. Loads environment variables from .env file
+ * 2. Gets database configuration (URL and schema name) from environment
+ * 3. Creates a Kysely database connection with the specified schema
+ * 4. Drops and recreates the database schema
+ * 5. Reports success/failure of reset operation
+ * 6. Closes database connection and exits
+ *
+ * Environment variables required:
+ * - DATABASE_URL: PostgreSQL connection string
+ * - DATABASE_SCHEMA: Schema name to reset (e.g. "grants_stack")
+ *
+ * The script will:
+ * - Drop the schema if it exists
+ * - Recreate an empty schema
+ * - Log results of the reset operation
+ * - Exit with code 0 on success, 1 on failure
+ *
+ * WARNING: This is a destructive operation that will delete all data in the schema.
+ * Make sure you have backups if needed before running this script.
+ */
+
+const main = async (): Promise<void> => {
+    const { DATABASE_URL, DATABASE_SCHEMA } = getDatabaseConfigFromEnv();
+
+    const db = createKyselyDatabase({
+        connectionString: DATABASE_URL,
+        withSchema: DATABASE_SCHEMA,
+    });
+
+    console.log(`Resetting database schema '${DATABASE_SCHEMA}'...`);
+
+    const resetResults = await resetDatabase({
+        db,
+        schema: DATABASE_SCHEMA,
+    });
+
+    if (resetResults && resetResults?.length > 0) {
+        const failedResets = resetResults.filter((resetResult) => resetResult.status === "Error");
+
+        if (failedResets.length > 0) {
+            console.error("❌ Failed resets:", failedResets);
+            throw new Error("Failed resets");
+        }
+
+        console.log(`✅ Reset applied successfully`);
+    } else {
+        console.log("No resets to apply");
+    }
+
+    await db.destroy();
+
+    process.exit(0);
+};
+
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

apps/scripts/src/schemas/index.ts

@@ -0,0 +1,19 @@
+import { z } from "zod";
+
+const dbEnvSchema = z.object({
+    DATABASE_URL: z.string().url(),
+    DATABASE_SCHEMA: z.string().min(1),
+});
+
+export type DbEnvConfig = z.infer<typeof dbEnvSchema>;
+
+export function getDatabaseConfigFromEnv(): DbEnvConfig {
+    const result = dbEnvSchema.safeParse(process.env);
+
+    if (!result.success) {
+        console.error("❌ Invalid environment variables:", result.error.format());
+        throw new Error("Invalid environment variables");
+    }
+
+    return result.data;
+}

apps/scripts/tsconfig.build.json

@@ -0,0 +1,11 @@
+{
+    "extends": "../../tsconfig.build.json",
+    "compilerOptions": {
+        "composite": true,
+        "declarationMap": true,
+        "declaration": true,
+        "outDir": "dist"
+    },
+    "include": ["src/**/*"],
+    "exclude": ["node_modules", "dist", "test"]
+}

apps/scripts/tsconfig.json

@@ -0,0 +1,4 @@
+{
+    "extends": "../../tsconfig.json",
+    "include": ["src/**/*", "test/**/*"]
+}

apps/scripts/vitest.config.ts

@@ -0,0 +1,22 @@
+import path from "path";
+import { configDefaults, defineConfig } from "vitest/config";
+
+export default defineConfig({
+    test: {
+        globals: true, // Use Vitest's global API without importing it in each file
+        environment: "node", // Use the Node.js environment
+        include: ["test/**/*.spec.ts"], // Include test files
+        exclude: ["node_modules", "dist"], // Exclude certain directories
+        coverage: {
+            provider: "v8",
+            reporter: ["text", "json", "html"], // Coverage reporters
+            exclude: ["node_modules", "dist", ...configDefaults.exclude], // Files to exclude from coverage
+        },
+    },
+    resolve: {
+        alias: {
+            // Setup path alias based on tsconfig paths
+            "@": path.resolve(__dirname, "src"),
+        },
+    },
+});

package.json

@@ -17,6 +17,8 @@
         "lint": "turbo run lint",
         "lint:fix": "turbo run lint:fix",
         "prepare": "husky",
+        "script:db:migrate": "pnpm run --filter @grants-stack-indexer/scripts script:db:migrate",
+        "script:db:reset": "pnpm run --filter @grants-stack-indexer/scripts script:db:reset",
         "start": "turbo run start",
         "test": "turbo run test",
         "test:cov": "turbo run test:cov",