diff --git a/.changeset/silver-dogs-rest.md b/.changeset/silver-dogs-rest.md new file mode 100644 index 0000000000..2e3e7ff8b3 --- /dev/null +++ b/.changeset/silver-dogs-rest.md @@ -0,0 +1,6 @@ +--- +"electric-sql": patch +"create-electric-app": patch +--- + +Modify CLI to introspect Postgres database through Electric's proxy. diff --git a/clients/typescript/src/cli/migrations/handler.ts b/clients/typescript/src/cli/migrations/handler.ts index a24176a7cf..8890f1f786 100644 --- a/clients/typescript/src/cli/migrations/handler.ts +++ b/clients/typescript/src/cli/migrations/handler.ts @@ -11,6 +11,13 @@ type GeneratorArgs = Partial * If not provided, it uses the url set in the `ELECTRIC_URL` * environment variable. If that variable is not set, it * resorts to the default url which is `http://localhost:5133`. + * - `--proxy ` + * Optional argument providing the url to connect to the PG database via the proxy. + * If not provided, it uses the url set in the `PG_PROXY_URL` environment variable. + * If that variable is not set, it resorts to the default url which is + * 'postgresql://prisma:password@localhost:65432/electric'. + * NOTE: the generator introspects the PG database via the proxy, + * the URL must therefore connect using the "prisma" user. * - `--out ` * Optional argument to specify where to write the generated client. * If this argument is not provided the generated client is written @@ -78,18 +85,24 @@ export function parseGenerateArgs(args: string[]): GeneratorArgs { process.exit(9) } } - const service = genArgs.service?.trim() // prepend protocol if not provided in service url + const service = genArgs.service?.trim() if (service && !/^https?:\/\//.test(service)) { genArgs.service = 'http://' + service } + // prepend protocol if not provided in proxy url + const proxy = genArgs.proxy?.trim() + if (proxy && !/^postgresql?:\/\//.test(proxy)) { + genArgs.proxy = 'postgresql://' + proxy + } + return genArgs } function checkFlag(flag: string): keyof GeneratorArgs { - const supportedFlags = ['--service', '--out', '--watch'] + const supportedFlags = ['--service', '--out', '--watch', '--proxy'] if (supportedFlags.includes(flag)) return flag.substring(2) as keyof GeneratorArgs // substring removes the double dash -- diff --git a/clients/typescript/src/cli/migrations/migrate.ts b/clients/typescript/src/cli/migrations/migrate.ts index 1b2c83b770..96714b2fc5 100644 --- a/clients/typescript/src/cli/migrations/migrate.ts +++ b/clients/typescript/src/cli/migrations/migrate.ts @@ -5,7 +5,6 @@ import { createWriteStream } from 'fs' import http from 'node:http' import https from 'node:https' import decompress from 'decompress' -import Database from 'better-sqlite3' import { buildMigrations, getMigrationNames } from './builder' import { exec } from 'child_process' import { dedent } from 'ts-dedent' @@ -14,6 +13,9 @@ const appRoot = path.resolve() // path where the user ran `npx electric migrate` export const defaultOptions = { service: process.env.ELECTRIC_URL ?? 'http://127.0.0.1:5133', + proxy: + process.env.ELECTRIC_PROXY_URL ?? + 'postgresql://prisma:proxy_password@localhost:65432/electric', // use "prisma" user because we will introspect the DB via the proxy out: path.join(appRoot, 'src/generated/client'), watch: false, pollingInterval: 1000, // in ms @@ -21,23 +23,6 @@ export const defaultOptions = { export type GeneratorOptions = typeof defaultOptions -/** - * A `DataSourceDescription` object describes on which line the Prisma schema - * data source is defined and on which line its `provider` and `url` are defined - * and what their values are. - */ -type DataSourceDescription = { - dataSourceLineIdx: number - provider: { - lineIdx: number - value: string - } - url: { - lineIdx: number - value: string - } -} - export async function generate(opts: GeneratorOptions) { if (opts.watch) { watchMigrations(opts) @@ -182,37 +167,11 @@ async function _generate(opts: Omit) { // Fetch the migrations from Electric endpoint and write them into `tmpFolder` await fetchMigrations(migrationEndpoint, migrationsFolder, tmpFolder) - const dbFile = path.resolve(path.join(tmpFolder, 'electric.db')) - const db = new Database(dbFile) - - // Load migrations and apply them on our fresh `db` SQLite DB - const migrations = await loadMigrations(migrationsFolder) - await applyMigrations(migrations, db) - - // Close the DB - db.close() - - // Create a fresh Prisma schema that will be used - // to introspect the SQLite DB after migrating it const prismaSchema = await createPrismaSchema(tmpFolder, opts) - // Replace the data source in the Prisma schema to be SQLite - // Remember the original data source such that we can restore it later - const originalDataSource = await getDataSource(prismaSchema) - await changeDataSourceToSQLite(prismaSchema, dbFile) - // Introspect the created DB to update the Prisma schema await introspectDB(prismaSchema) - // Modify the data source back to Postgres - // because Prisma won't generate createMany/updateMany/... schemas - // if the data source is a SQLite DB. - await setDataSource( - prismaSchema, - originalDataSource.provider.value, - originalDataSource.url.value - ) - // Modify snake_case table names to PascalCase await pascalCaseTableNames(prismaSchema) @@ -241,7 +200,7 @@ async function _generate(opts: Omit) { */ async function createPrismaSchema( folder: string, - { out }: Omit + { out, proxy }: Omit ) { const prismaDir = path.join(folder, 'prisma') const prismaSchemaFile = path.join(prismaDir, 'schema.prisma') @@ -264,23 +223,12 @@ async function createPrismaSchema( datasource db { provider = "postgresql" - url = env("PRISMA_DB_URL") + url = "${proxy}" }` await fs.writeFile(prismaSchemaFile, schema) return prismaSchemaFile } -async function loadMigrations(migrationsFolder: string): Promise { - const migrationDirNames = await getMigrationNames(migrationsFolder) - const migrationFiles = migrationDirNames.map((dirName) => - path.join(migrationsFolder, dirName, 'migration.sql') - ) - const migrations = await Promise.all( - migrationFiles.map((migration) => fs.readFile(migration, 'utf8')) - ) - return migrations -} - async function getFileLines(prismaSchema: string): Promise> { const contents = await fs.readFile(prismaSchema, 'utf8') return contents.split(/\r?\n/) @@ -371,76 +319,6 @@ export function doPascalCaseTableNames(lines: string[]): string[] { return lines } -async function getDataSource( - prismaSchema: string -): Promise { - const lines = await getFileLines(prismaSchema) - const dataSourceStartIdx = lines.findIndex((ln) => - ln.trim().startsWith('datasource ') - ) - if (dataSourceStartIdx === -1) { - throw new Error('Prisma schema does not define a datasource.') - } - - const linesStartingAtDataSource = lines.slice(dataSourceStartIdx) - const providerIdx = - dataSourceStartIdx + - linesStartingAtDataSource.findIndex((ln) => - ln.trim().startsWith('provider ') - ) - const urlIdx = - dataSourceStartIdx + - linesStartingAtDataSource.findIndex((ln) => ln.trim().startsWith('url ')) - - const providerLine = lines[providerIdx] - const urlLine = lines[urlIdx] - - return { - dataSourceLineIdx: dataSourceStartIdx, - provider: { - lineIdx: providerIdx, - value: providerLine, - }, - url: { - lineIdx: urlIdx, - value: urlLine, - }, - } -} - -async function changeDataSourceToSQLite(prismaSchema: string, dbFile: string) { - await setDataSource( - prismaSchema, - ' provider = "sqlite"', - ` url = "file:${dbFile}"` - ) -} - -/** - * Changes the data source in the Prisma schema to the provided data source values. - * @param prismaSchema Path to the schema whose datasource must be modified. - * @param provider The new provider - * @param url The new url - */ -async function setDataSource( - prismaSchema: string, - provider: string, - url: string -) { - const ogDataSource = await getDataSource(prismaSchema) - const providerLineIdx = ogDataSource.provider.lineIdx - const urlLineIdx = ogDataSource.url.lineIdx - - const lines = (await getFileLines(prismaSchema)).map((ln) => ln) - lines[providerLineIdx] = provider - lines[urlLineIdx] = url - - const data = lines.join('\n') - - // Write the modified schema to the file - await fs.writeFile(prismaSchema, data) -} - async function introspectDB(prismaSchema: string): Promise { await executeShellCommand( `npx prisma db pull --schema="${prismaSchema}"`, @@ -477,15 +355,6 @@ async function executeShellCommand( }) } -/** - * Opens the provided DB (or creates it if it doesn't exist) and applies the given migrations on it. - * @migrations Migrations to apply - * @db The DB on which to apply the migrations - */ -async function applyMigrations(migrations: string[], db: Database.Database) { - migrations.forEach((migration) => db.exec(migration)) -} - /** * Fetches the migrations from the provided endpoint, * unzips them and writes them to the `writeTo` location. diff --git a/examples/starter/src/index.ts b/examples/starter/src/index.ts index 23892d0bdf..4df486e2fe 100644 --- a/examples/starter/src/index.ts +++ b/examples/starter/src/index.ts @@ -169,7 +169,10 @@ projectPackageJson.name = projectName await fs.writeFile( packageJsonFile, - JSON.stringify(projectPackageJson, null, 2).replace('http://localhost:5133', `http://localhost:${electricPort}`) + JSON + .stringify(projectPackageJson, null, 2) + .replace('http://localhost:5133', `http://localhost:${electricPort}`) + .replace('postgresql://prisma:proxy_password@localhost:65432/electric', `postgresql://prisma:proxy_password@localhost:${electricProxyPort}/electric`) ) // Update the project's title in the index.html file diff --git a/examples/starter/template/change-ports.js b/examples/starter/template/change-ports.js index ec51b14a32..fb9572455d 100644 --- a/examples/starter/template/change-ports.js +++ b/examples/starter/template/change-ports.js @@ -60,6 +60,10 @@ async function init() { // Update port in package.json file await findAndReplaceInFile(`http://localhost:${oldElectricPort}`, `http://localhost:${electricPort}`, packageJsonFile) + await findAndReplaceInFile( + `postgresql://prisma:proxy_password@localhost:${oldElectricProxyPort}/electric`, + `postgresql://prisma:proxy_password@localhost:${electricProxyPort}/electric`, packageJsonFile + ) // Update the port on which Electric runs in the builder.js file await findAndReplaceInFile(`ws://localhost:${oldElectricPort}`, `ws://localhost:${electricPort}`, builderFile) diff --git a/examples/starter/template/package.json b/examples/starter/template/package.json index a6a1cfbd96..cb677983b3 100644 --- a/examples/starter/template/package.json +++ b/examples/starter/template/package.json @@ -7,7 +7,7 @@ "backend:stop": "docker compose --env-file backend/compose/.envrc -f backend/compose/docker-compose.yaml stop", "backend:up": "yarn backend:start --detach", "backend:down": "docker compose --env-file backend/compose/.envrc -f backend/compose/docker-compose.yaml down --volumes", - "client:generate": "yarn electric:check && npx electric-sql generate --service http://localhost:5133", + "client:generate": "yarn electric:check && npx electric-sql generate --service http://localhost:5133 --proxy postgresql://prisma:proxy_password@localhost:65432/electric", "client:watch": "yarn client:generate --watch", "db:migrate": "node ./db/migrate.js", "db:psql": "node ./db/connect.js", @@ -25,7 +25,7 @@ "electric-sql": "latest", "react": "^18.2.0", "react-dom": "^18.2.0", - "wa-sqlite": "rhashimoto/wa-sqlite#master" + "wa-sqlite": "rhashimoto/wa-sqlite" }, "devDependencies": { "@databases/pg-migrations": "^5.0.2",