chore (CLI): introspect PG directly (#534)

This PR modifies the CLI such that the `yarn client:generate` script directly introspects the PG database (via the proxy) as described in #437. (Up till now, we would fetch migrations from Electric endpoint, recreate a local SQLite DB, and introspect that DB). Testing this PR is a bit tricky because the necessary packages are not published. You can test it as follows: - build Electric (`cd components/electric && make docker-build`) - build and pack the ts-client (`cd clients/typescript && pnpm install && pnpm build && pnpm pack`) - build the starter (`cd examples/starter && pnpm install && pnpm build && npm link`) - create an app `create-electric-app my-app` - modify your app to use the local Electric image (because it has not yet been published to Docker hub) i.e. in `my-app/backend/compose/.envrc` change the `ELECTRIC_IMAGE` env var to be `electric:local-build`. - modify your app to use our packed ts-client: - change the electric-sql dependency in `my-app/package.json` to be `"electric-sql": "file:<path to electric mono repo>/clients/typescript/electric-sql-0.6.4.tgz",` - now run the app: - `yarn` - `yarn backend:start` - `yarn db:migrate` - `yarn client:generate` - `yarn start` --------- Co-authored-by: Oleksii Sholik <[email protected]> Co-authored-by: Garry Hill <[email protected]>
electric-sql · Oct 25, 2023 · cfded69 · cfded69
1 parent e5fb598
commit cfded69
Show file tree

Hide file tree

Showing 6 changed files with 36 additions and 141 deletions.
diff --git a/.changeset/silver-dogs-rest.md b/.changeset/silver-dogs-rest.md
@@ -0,0 +1,6 @@
+---
+"electric-sql": patch
+"create-electric-app": patch
+---
+
+Modify CLI to introspect Postgres database through Electric's proxy.
diff --git a/clients/typescript/src/cli/migrations/handler.ts b/clients/typescript/src/cli/migrations/handler.ts
@@ -11,6 +11,13 @@ type GeneratorArgs = Partial<GeneratorOptions>
  *     If not provided, it uses the url set in the `ELECTRIC_URL`
  *     environment variable. If that variable is not set, it
  *     resorts to the default url which is `http://localhost:5133`.
+ * - `--proxy <url>`
+ *    Optional argument providing the url to connect to the PG database via the proxy.
+ *    If not provided, it uses the url set in the `PG_PROXY_URL` environment variable.
+ *    If that variable is not set, it resorts to the default url which is
+ *    'postgresql://prisma:password@localhost:65432/electric'.
+ *    NOTE: the generator introspects the PG database via the proxy,
+ *          the URL must therefore connect using the "prisma" user.
  *  - `--out <path>`
  *     Optional argument to specify where to write the generated client.
  *     If this argument is not provided the generated client is written
@@ -78,18 +85,24 @@ export function parseGenerateArgs(args: string[]): GeneratorArgs {
       process.exit(9)
     }
   }
-  const service = genArgs.service?.trim()
 
   // prepend protocol if not provided in service url
+  const service = genArgs.service?.trim()
   if (service && !/^https?:\/\//.test(service)) {
     genArgs.service = 'http://' + service
   }
 
+  // prepend protocol if not provided in proxy url
+  const proxy = genArgs.proxy?.trim()
+  if (proxy && !/^postgresql?:\/\//.test(proxy)) {
+    genArgs.proxy = 'postgresql://' + proxy
+  }
+
   return genArgs
 }
 
 function checkFlag(flag: string): keyof GeneratorArgs {
-  const supportedFlags = ['--service', '--out', '--watch']
+  const supportedFlags = ['--service', '--out', '--watch', '--proxy']
   if (supportedFlags.includes(flag))
     return flag.substring(2) as keyof GeneratorArgs
   // substring removes the double dash --

diff --git a/clients/typescript/src/cli/migrations/migrate.ts b/clients/typescript/src/cli/migrations/migrate.ts
@@ -5,7 +5,6 @@ import { createWriteStream } from 'fs'
 import http from 'node:http'
 import https from 'node:https'
 import decompress from 'decompress'
-import Database from 'better-sqlite3'
 import { buildMigrations, getMigrationNames } from './builder'
 import { exec } from 'child_process'
 import { dedent } from 'ts-dedent'
@@ -14,30 +13,16 @@ const appRoot = path.resolve() // path where the user ran `npx electric migrate`
 
 export const defaultOptions = {
   service: process.env.ELECTRIC_URL ?? 'http://127.0.0.1:5133',
+  proxy:
+    process.env.ELECTRIC_PROXY_URL ??
+    'postgresql://prisma:proxy_password@localhost:65432/electric', // use "prisma" user because we will introspect the DB via the proxy
   out: path.join(appRoot, 'src/generated/client'),
   watch: false,
   pollingInterval: 1000, // in ms
 }
 
 export type GeneratorOptions = typeof defaultOptions
 
-/**
- * A `DataSourceDescription` object describes on which line the Prisma schema
- * data source is defined and on which line its `provider` and `url` are defined
- * and what their values are.
- */
-type DataSourceDescription = {
-  dataSourceLineIdx: number
-  provider: {
-    lineIdx: number
-    value: string
-  }
-  url: {
-    lineIdx: number
-    value: string
-  }
-}
-
 export async function generate(opts: GeneratorOptions) {
   if (opts.watch) {
     watchMigrations(opts)
@@ -182,37 +167,11 @@ async function _generate(opts: Omit<GeneratorOptions, 'watch'>) {
     // Fetch the migrations from Electric endpoint and write them into `tmpFolder`
     await fetchMigrations(migrationEndpoint, migrationsFolder, tmpFolder)
 
-    const dbFile = path.resolve(path.join(tmpFolder, 'electric.db'))
-    const db = new Database(dbFile)
-
-    // Load migrations and apply them on our fresh `db` SQLite DB
-    const migrations = await loadMigrations(migrationsFolder)
-    await applyMigrations(migrations, db)
-
-    // Close the DB
-    db.close()
-
-    // Create a fresh Prisma schema that will be used
-    // to introspect the SQLite DB after migrating it
     const prismaSchema = await createPrismaSchema(tmpFolder, opts)
 
-    // Replace the data source in the Prisma schema to be SQLite
-    // Remember the original data source such that we can restore it later
-    const originalDataSource = await getDataSource(prismaSchema)
-    await changeDataSourceToSQLite(prismaSchema, dbFile)
-
     // Introspect the created DB to update the Prisma schema
     await introspectDB(prismaSchema)
 
-    // Modify the data source back to Postgres
-    // because Prisma won't generate createMany/updateMany/... schemas
-    // if the data source is a SQLite DB.
-    await setDataSource(
-      prismaSchema,
-      originalDataSource.provider.value,
-      originalDataSource.url.value
-    )
-
     // Modify snake_case table names to PascalCase
     await pascalCaseTableNames(prismaSchema)
 
@@ -241,7 +200,7 @@ async function _generate(opts: Omit<GeneratorOptions, 'watch'>) {
  */
 async function createPrismaSchema(
   folder: string,
-  { out }: Omit<GeneratorOptions, 'watch'>
+  { out, proxy }: Omit<GeneratorOptions, 'watch'>
 ) {
   const prismaDir = path.join(folder, 'prisma')
   const prismaSchemaFile = path.join(prismaDir, 'schema.prisma')
@@ -264,23 +223,12 @@ async function createPrismaSchema(
 
     datasource db {
       provider = "postgresql"
-      url      = env("PRISMA_DB_URL")
+      url      = "${proxy}"
     }`
   await fs.writeFile(prismaSchemaFile, schema)
   return prismaSchemaFile
 }
 
-async function loadMigrations(migrationsFolder: string): Promise<string[]> {
-  const migrationDirNames = await getMigrationNames(migrationsFolder)
-  const migrationFiles = migrationDirNames.map((dirName) =>
-    path.join(migrationsFolder, dirName, 'migration.sql')
-  )
-  const migrations = await Promise.all(
-    migrationFiles.map((migration) => fs.readFile(migration, 'utf8'))
-  )
-  return migrations
-}
-
 async function getFileLines(prismaSchema: string): Promise<Array<string>> {
   const contents = await fs.readFile(prismaSchema, 'utf8')
   return contents.split(/\r?\n/)
@@ -371,76 +319,6 @@ export function doPascalCaseTableNames(lines: string[]): string[] {
   return lines
 }
 
-async function getDataSource(
-  prismaSchema: string
-): Promise<DataSourceDescription> {
-  const lines = await getFileLines(prismaSchema)
-  const dataSourceStartIdx = lines.findIndex((ln) =>
-    ln.trim().startsWith('datasource ')
-  )
-  if (dataSourceStartIdx === -1) {
-    throw new Error('Prisma schema does not define a datasource.')
-  }
-
-  const linesStartingAtDataSource = lines.slice(dataSourceStartIdx)
-  const providerIdx =
-    dataSourceStartIdx +
-    linesStartingAtDataSource.findIndex((ln) =>
-      ln.trim().startsWith('provider ')
-    )
-  const urlIdx =
-    dataSourceStartIdx +
-    linesStartingAtDataSource.findIndex((ln) => ln.trim().startsWith('url '))
-
-  const providerLine = lines[providerIdx]
-  const urlLine = lines[urlIdx]
-
-  return {
-    dataSourceLineIdx: dataSourceStartIdx,
-    provider: {
-      lineIdx: providerIdx,
-      value: providerLine,
-    },
-    url: {
-      lineIdx: urlIdx,
-      value: urlLine,
-    },
-  }
-}
-
-async function changeDataSourceToSQLite(prismaSchema: string, dbFile: string) {
-  await setDataSource(
-    prismaSchema,
-    '  provider = "sqlite"',
-    `  url = "file:${dbFile}"`
-  )
-}
-
-/**
- * Changes the data source in the Prisma schema to the provided data source values.
- * @param prismaSchema Path to the schema whose datasource must be modified.
- * @param provider The new provider
- * @param url The new url
- */
-async function setDataSource(
-  prismaSchema: string,
-  provider: string,
-  url: string
-) {
-  const ogDataSource = await getDataSource(prismaSchema)
-  const providerLineIdx = ogDataSource.provider.lineIdx
-  const urlLineIdx = ogDataSource.url.lineIdx
-
-  const lines = (await getFileLines(prismaSchema)).map((ln) => ln)
-  lines[providerLineIdx] = provider
-  lines[urlLineIdx] = url
-
-  const data = lines.join('\n')
-
-  // Write the modified schema to the file
-  await fs.writeFile(prismaSchema, data)
-}
-
 async function introspectDB(prismaSchema: string): Promise<void> {
   await executeShellCommand(
     `npx prisma db pull --schema="${prismaSchema}"`,
@@ -477,15 +355,6 @@ async function executeShellCommand(
   })
 }
 
-/**
- * Opens the provided DB (or creates it if it doesn't exist) and applies the given migrations on it.
- * @migrations Migrations to apply
- * @db The DB on which to apply the migrations
- */
-async function applyMigrations(migrations: string[], db: Database.Database) {
-  migrations.forEach((migration) => db.exec(migration))
-}
-
 /**
  * Fetches the migrations from the provided endpoint,
  * unzips them and writes them to the `writeTo` location.

diff --git a/examples/starter/src/index.ts b/examples/starter/src/index.ts
@@ -169,7 +169,10 @@ projectPackageJson.name = projectName
 
 await fs.writeFile(
   packageJsonFile,
-  JSON.stringify(projectPackageJson, null, 2).replace('http://localhost:5133', `http://localhost:${electricPort}`)
+  JSON
+    .stringify(projectPackageJson, null, 2)
+    .replace('http://localhost:5133', `http://localhost:${electricPort}`)
+    .replace('postgresql://prisma:proxy_password@localhost:65432/electric', `postgresql://prisma:proxy_password@localhost:${electricProxyPort}/electric`)
 )
 
 // Update the project's title in the index.html file

diff --git a/examples/starter/template/change-ports.js b/examples/starter/template/change-ports.js
@@ -60,6 +60,10 @@ async function init() {
 
   // Update port in package.json file
   await findAndReplaceInFile(`http://localhost:${oldElectricPort}`, `http://localhost:${electricPort}`, packageJsonFile)
+  await findAndReplaceInFile(
+    `postgresql://prisma:proxy_password@localhost:${oldElectricProxyPort}/electric`,
+    `postgresql://prisma:proxy_password@localhost:${electricProxyPort}/electric`, packageJsonFile
+  )
 
   // Update the port on which Electric runs in the builder.js file
   await findAndReplaceInFile(`ws://localhost:${oldElectricPort}`, `ws://localhost:${electricPort}`, builderFile)

diff --git a/examples/starter/template/package.json b/examples/starter/template/package.json
@@ -7,7 +7,7 @@
     "backend:stop": "docker compose --env-file backend/compose/.envrc -f backend/compose/docker-compose.yaml stop",
     "backend:up": "yarn backend:start --detach",
     "backend:down": "docker compose --env-file backend/compose/.envrc -f backend/compose/docker-compose.yaml down --volumes",
-    "client:generate": "yarn electric:check && npx electric-sql generate --service http://localhost:5133",
+    "client:generate": "yarn electric:check && npx electric-sql generate --service http://localhost:5133 --proxy postgresql://prisma:proxy_password@localhost:65432/electric",
     "client:watch": "yarn client:generate --watch",
     "db:migrate": "node ./db/migrate.js",
     "db:psql": "node ./db/connect.js",
@@ -25,7 +25,7 @@
     "electric-sql": "latest",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "wa-sqlite": "rhashimoto/wa-sqlite#master"
+    "wa-sqlite": "rhashimoto/wa-sqlite"
   },
   "devDependencies": {
     "@databases/pg-migrations": "^5.0.2",