feat(starter): starter improvements to avoid port clashes (#505)

This PR improves the starter to enable several (independent) Electric applications to run on the same machine and thus fixes #497 and #489. Also addressed VAX-1114. The changes are as follows: - Avoid volume clashes by putting project name on the app name - Avoid web server port clashes by supporting a custom port to be used and fix it everywhere in the template - Avoid Electric port clashes by supporting a custom port to be used and fix it everywhere in the template - Introduced 2 modes for the starter: fast mode and interactive mode - Fast mode: app name and ports (optional) are provided as command line arguments - Interactive mode: no arguments to the script, app name and ports are asked via prompts - Changed start to check that the chosen port for Electric is free. If not, prompt user if they want to change the port - Script to change the project’s configuration with other Electric and web server ports. Exposed as a new `ports:configure` command. - Changed `backend:start` to check for port clashes and if there is a port clash suggest to run `ports:configure`. - Changed `yarn start` and `client:generate` to check that the Electric port the app is configured to use is the app's own Electric service (e.g. catches problem where project 1 is running Electric on port 5133 and another project is accidentally configured to also connect to port 5133). - Modified build script to use forward requests to right esbuild server. This fixes a bug where if an esbuild server was already running, the new app would forward its requests to the esbuild server of the other app. tldr: it should now be possible to run several Electric applications. Port clashes should always be detected and reported with a suggestion to reconfigure the app with other ports. --------- Co-authored-by: Oleksii Sholik <[email protected]>
electric-sql · Oct 3, 2023 · 0d879a8 · 0d879a8
1 parent b5ba482
commit 0d879a8
Show file tree

Hide file tree

Showing 16 changed files with 591 additions and 43 deletions.
diff --git a/.changeset/twelve-poems-exercise.md b/.changeset/twelve-poems-exercise.md
@@ -0,0 +1,11 @@
+---
+"create-electric-app": patch
+---
+
+Improved starter such that several (independent) Electric projects can run concurrently.
+The starter now has 2 modes: fast mode and interactive mode.
+In fast mode, you can provide the app name and optional ports for Electric and the webserver as arguments.
+In interactive mode, the script will prompt for an app name and ports (suggesting defaults).
+Port clashes are now detected and reported to the user.
+The user can change the ports the app uses by invoking 'yarn ports:configure'.
+Also fixes the bug where all Electric applications would forward requests to the esbuild server that is running on port 8000 instead of their own esbuild server.
diff --git a/examples/starter/package.json b/examples/starter/package.json
@@ -20,11 +20,14 @@
   "devDependencies": {
     "@tsmodule/tsmodule": "42",
     "@types/node": "^18.8.4",
-    "typescript": "^5.1.3",
-    "@types/shelljs": "^0.8.12"
+    "@types/shelljs": "^0.8.12",
+    "@types/tcp-port-used": "^1.0.2",
+    "typescript": "^5.1.3"
   },
   "dependencies": {
     "ora": "^7.0.1",
-    "shelljs": "^0.8.5"
+    "prompt": "^1.3.0",
+    "shelljs": "^0.8.5",
+    "tcp-port-used": "^1.0.2"
   }
 }
diff --git a/examples/starter/src/index.ts b/examples/starter/src/index.ts
@@ -8,11 +8,110 @@ import { fileURLToPath } from 'url'
 import path from 'path'
 import ora from 'ora'
 import shell from 'shelljs'
+import portUsed from 'tcp-port-used'
+import prompt from 'prompt'
 
-const spinner = ora('Creating project structure').start()
+// Regex to check that a number is between 0 and 65535
+const portRegex = /^([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/
+const spinner = ora('Validating arguments').start()
 
-// The first argument will be the project name
-const projectName = process.argv[2]
+const error = (err: string) => {
+  spinner.stop()
+  console.error('\x1b[31m', err + '\nnpx create-electric-app [<app-name>] [--electric-port <port>] [--webserver-port <port>]')
+  process.exit(1)
+}
+
+let projectName = process.argv[2]
+let args = process.argv.slice(3)
+let electricPort = 5133 // default port for Electric
+let webserverPort = 3001 // default port for the webserver
+
+// Validate the provided command line arguments
+while (args.length > 0) {
+  // There are arguments to parse
+  const flag = args[0]
+  const value = args[1]
+
+  args = args.slice(2)
+
+  const checkValue = () => {
+    if (typeof value === 'undefined') {
+      error(`Missing value for option '${flag}'.`)
+    }
+  }
+
+  switch (flag) {
+    case '--electric-port':
+      checkValue()
+      electricPort = parsePort(value)
+      break
+    case '--webserver-port':
+      checkValue()
+      webserverPort = parsePort(value)
+      break
+    default:
+      error(`Unrecognized option: '${flag}'.`)
+  }
+}
+
+spinner.text = 'Validating app name'
+const appNameRegex = /^[a-z0-9]+[a-z0-9-_]*$/
+const invalidAppNameMessage = `Invalid app name. ` +
+  'App names must contain only lowercase letters, decimal digits, dashes, and underscores, ' +
+  'and must begin with a lowercase letter or decimal digit.'
+
+if (typeof projectName === 'undefined') {
+  // no project name is provided -> enter prompt mode
+  spinner.stop()
+  prompt.start()
+  const userInput = (await prompt.get({
+    properties: {
+      appName: {
+        description: 'App name',
+        type: 'string',
+        // Validate the project name to follow
+        // the restrictions for Docker compose project names.
+        // cf. https://docs.docker.com/compose/environment-variables/envvars/
+        // Because we will use the app name
+        // as the Docker compose project name.
+        pattern: appNameRegex,
+        message: invalidAppNameMessage,
+        required: true,
+      },
+      electricPort: {
+        description: 'Port on which to run Electric',
+        type: 'number',
+        pattern: portRegex,
+        message: 'Port should be between 0 and 65535.',
+        default: electricPort
+      },
+      webserverPort: {
+        description: 'Port on which to run the webserver',
+        type: 'number',
+        pattern: portRegex,
+        message: 'Port should be between 0 and 65535.',
+        default: webserverPort
+      },
+    }
+  })) as { appName: string, electricPort: number, webserverPort: number }
+
+  spinner.start()
+  projectName = userInput.appName
+  electricPort = userInput.electricPort
+  webserverPort = userInput.webserverPort
+}
+
+spinner.text = 'Ensuring the necessary ports are free'
+
+if (!appNameRegex.test(projectName)) {
+  error(invalidAppNameMessage)
+}
+
+electricPort = await checkPort(electricPort, 'Electric', 5133)
+webserverPort = await checkPort(webserverPort, 'the web server', 3001)
+
+spinner.text = 'Creating project structure'
+spinner.start()
 
 // Create a project directory with the project name
 const currentDir = process.cwd()
@@ -55,15 +154,26 @@ const projectPackageJson = JSON.parse(await fs.readFile(packageJsonFile, 'utf8')
 projectPackageJson.name = projectName
 
 await fs.writeFile(
-  path.join(projectDir, 'package.json'),
-  JSON.stringify(projectPackageJson, null, 2)
+  packageJsonFile,
+  JSON.stringify(projectPackageJson, null, 2).replace('http://localhost:5133', `http://localhost:${electricPort}`)
 )
 
 // Update the project's title in the index.html file
 const indexFile = path.join(projectDir, 'public', 'index.html')
-const index = await fs.readFile(indexFile, 'utf8')
-const newIndex = index.replace('ElectricSQL starter template', projectName)
-await fs.writeFile(indexFile, newIndex)
+await findAndReplaceInFile('ElectricSQL starter template', projectName, indexFile)
+
+// Update the port on which Electric runs in the builder.js file
+const builderFile = path.join(projectDir, 'builder.js')
+await findAndReplaceInFile('ws://localhost:5133', `ws://localhost:${electricPort}`, builderFile)
+
+// Update the port on which Electric runs in startElectric.js file
+const startElectricFile = path.join(projectDir, 'backend', 'startElectric.js')
+await findAndReplaceInFile('5133', `${electricPort}`, startElectricFile)
+
+// Update the port of the web server of the example in the builder.js file
+await findAndReplaceInFile("listen(3001)", `listen(${webserverPort})`, builderFile)
+await findAndReplaceInFile("http://localhost:3001", `http://localhost:${webserverPort}`, builderFile)
+
 
 // Store the app's name in .envrc
 // db name must start with a letter
@@ -73,9 +183,12 @@ await fs.writeFile(indexFile, newIndex)
 const name = projectName.match(/[a-zA-Z].*/)?.[0] // strips prefix of non-alphanumeric characters
 if (name) {
   const dbName = name.replace(/[\W_]+/g, '_')
-  await fs.appendFile(envrcFile, `export APP_NAME=${dbName}`)
+  await fs.appendFile(envrcFile, `export APP_NAME=${dbName}\n`)
 }
 
+// Also write the port for Electric to .envrc
+await fs.appendFile(envrcFile, `export ELECTRIC_PORT=${electricPort}\n`)
+
 // Run `yarn install` in the project directory to install the dependencies
 // Also run `yarn upgrade` to replace `electric-sql: latest` by `electric-sql: x.y.z`
 // where `x.y.z` corresponds to the latest version.
@@ -96,6 +209,7 @@ proc.on('close', async (code) => {
   if (code === 0) {
     // Pull latest electric image from docker hub
     // such that we are sure that it is compatible with the latest client
+    spinner.text = 'Pulling latest Electric image'
     shell.exec('docker pull electricsql/electric:latest', { silent: true })
 
     const { stdout } = shell.exec("docker image inspect --format '{{.RepoDigests}}' electricsql/electric:latest", { silent: true })
@@ -111,7 +225,7 @@ proc.on('close', async (code) => {
     }
 
     // write the electric image to use to .envrc file
-    await fs.appendFile(envrcFile, `\nexport ELECTRIC_IMAGE=${electricImage}\n`)
+    await fs.appendFile(envrcFile, `export ELECTRIC_IMAGE=${electricImage}\n`)
   }
 
   spinner.stop()
@@ -125,3 +239,61 @@ proc.on('close', async (code) => {
 
   console.log(`Navigate to your app folder \`cd ${projectName}\` and follow the instructions in the README.md.`)
 })
+
+/*
+ * Replaces the first occurence of `find` by `replace` in the file `file`.
+ * If `find` is a regular expression that sets the `g` flag, then it replaces all occurences.
+ */ 
+async function findAndReplaceInFile(find: string | RegExp, replace: string, file: string) {
+  const content = await fs.readFile(file, 'utf8')
+  const replacedContent = content.replace(find, replace)
+  await fs.writeFile(file, replacedContent)
+}
+
+/**
+ * Checks if the given port is open.
+ * If not, it will ask the user if
+ * they want to choose another port.
+ * @returns The chosen port.
+ */
+async function checkPort(port: number, process: string, defaultPort: number): Promise<number> {
+  const portOccupied = await portUsed.check(port)
+  if (!portOccupied) {
+    return port
+  }
+
+  spinner.stop()
+
+  // Warn the user that the chosen port is occupied
+  console.warn(`Port ${port} for ${process} is already in use.`)
+  // Propose user to change port
+  prompt.start()
+
+  const { port: newPort } = (await prompt.get({
+    properties: {
+      port: {
+        description: 'Hit Enter to keep it or enter a different port number',
+        type: 'number',
+        pattern: portRegex,
+        message: 'Please choose a port between 0 and 65535',
+        default: port,
+      }
+    }
+  }))
+
+  if (newPort === port) {
+    // user chose not to change port
+    return newPort
+  }
+  else {
+    // user changed port, check that it is free
+    return checkPort(newPort, process, defaultPort)
+  }
+}
+
+function parsePort(port: string): number {
+  if (!portRegex.test(port)) {
+    error(`Invalid port '${port}. Port should be between 0 and 65535.'`)
+  }
+  return Number.parseInt(port)
+}
diff --git a/examples/starter/src/prompt.d.ts b/examples/starter/src/prompt.d.ts
@@ -0,0 +1 @@
+declare module 'prompt'
diff --git a/examples/starter/template/backend/compose/docker-compose.yaml b/examples/starter/template/backend/compose/docker-compose.yaml
@@ -1,4 +1,5 @@
 version: "3.8"
+name: "${APP_NAME:-electric}"
 
 configs:
   postgres_config:
@@ -38,7 +39,7 @@ services:
       LOGICAL_PUBLISHER_HOST: electric
       AUTH_MODE: insecure
     ports:
-      - 5133:5133
+      - ${ELECTRIC_PORT:-5133}:5133
     volumes:
       - electric_data:/app/data
     depends_on:

diff --git a/examples/starter/template/backend/startCompose.js b/examples/starter/template/backend/startCompose.js
@@ -0,0 +1,18 @@
+const exec = require('shelljs-live')
+const path = require('path')
+
+const envrcFile = path.join(__dirname, 'compose', '.envrc')
+const composeFile = path.join(__dirname, 'compose', 'docker-compose.yaml')
+
+const cliArguments = process.argv.slice(2).join(' ')
+
+const res = exec(`docker compose --env-file ${envrcFile} -f ${composeFile} up ${cliArguments}`)
+
+if (res.code !== 0 && res.stderr.includes('port is already allocated')) {
+  // inform the user that they should change ports
+  console.error(
+    '\x1b[31m',
+    'Could not start Electric because the port seems to be taken.\n' +
+    'To run Electric on another port execute `yarn ports:configure`'
+  )
+}
diff --git a/examples/starter/template/backend/startElectric.js b/examples/starter/template/backend/startElectric.js
@@ -1,35 +1,74 @@
-const shell = require('shelljs')
+const exec = require('shelljs-live')
 
 let db = process.env.DATABASE_URL
+let electricPort = process.env.ELECTRIC_PORT ?? 5133
 
-if (process.argv.length === 4) {
-  const command = process.argv[2]
+let args = process.argv.slice(2)
 
-  if (command !== '-db') {
-    console.error(`Unsupported option ${command}. Only '-db' option is supported.`)
+while (args.length > 0) {
+  // There are arguments to parse
+  const flag = args[0]
+  const value = args[1]
 
-    process.exit(1)
+  args = args.slice(2)
+
+  const checkValue = () => {
+    if (typeof value === 'undefined') {
+      error(`Missing value for option '${flag}'.`)
+    }
   }
 
-  db = process.argv[3]
+  switch (flag) {
+    case '-db':
+      checkValue()
+      db = value
+      break
+    case '--electric-port':
+      checkValue()
+      parseElectricPort(value)
+      break
+    default:
+      error(`Unrecognized option: '${flag}'.`)
+  }
 }
-else if (process.argv.length !== 2) {
-  console.log('Wrong number of arguments provided. Only one optional argument `-db <Postgres connection url>` is supported.')
+
+function parseElectricPort(port) {
+  // checks that the number is between 0 and 65535
+  const portRegex = /^([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/
+  if (!portRegex.test(port)) {
+    error(`Invalid port '${port}. Port should be between 0 and 65535.'`)
+  }
+  electricPort = port
 }
 
 if (db === undefined) {
   console.error(`Database URL is not provided. Please provide one using the DATABASE_URL environment variable.`)
-
   process.exit(1)
 }
 
 const electric = process.env.ELECTRIC_IMAGE ?? "electricsql/electric:latest"
 
-shell.exec(
+// 5433 is the logical publisher port
+// it is exposed because PG must be able to connect to Electric
+const res = exec(
   `docker run \
       -e "DATABASE_URL=${db}" \
       -e "LOGICAL_PUBLISHER_HOST=localhost" \
       -e "AUTH_MODE=insecure" \
-      -p 5133:5133 \
+      -p ${electricPort}:5133 \
       -p 5433:5433 ${electric}`
 )
+
+if (res.code !== 0 && res.stderr.includes('port is already allocated')) {
+  // inform the user that they should change ports
+  console.error(
+    '\x1b[31m',
+    'Could not start Electric because the port seems to be taken.\n' +
+    'To run Electric on another port execute `yarn ports:configure`'
+  )
+}
+
+function error(err) {
+  console.error('\x1b[31m', err + '\nyarn electric:start [-db <Postgres connection url>] [-p <Electric port>]')
+  process.exit(1)
+}