fix(satellite): clients overload the server on fatal errors (#518)

Make better use of the back off strategy for reconnecting and expose back off configuration to the app

Stop trying to reconnect to electric on fatal errors.

.changeset/famous-cars-agree.md

@@ -0,0 +1,5 @@
+---
+"electric-sql": patch
+---
+
+prevent reconnect loop of doom on fatal errors

clients/typescript/src/config/index.ts

@@ -1,4 +1,8 @@
 import { AuthConfig } from '../auth/index'
+import {
+  ConnectionBackoffOptions as ConnectionBackOffOptions,
+  satelliteDefaults,
+} from '../satellite/config'
 
 export interface ElectricConfig {
   auth: AuthConfig
@@ -24,6 +28,10 @@ export interface ElectricConfig {
    * Defaults to `false`.
    */
   debug?: boolean
+  /**
+   * Optional backoff options for connecting with Electric
+   */
+  connectionBackOffOptions?: ConnectionBackOffOptions
 }
 
 export type HydratedConfig = {
@@ -34,6 +42,7 @@ export type HydratedConfig = {
     ssl: boolean
   }
   debug: boolean
+  connectionBackOffOptions: ConnectionBackOffOptions
 }
 
 export type InternalElectricConfig = {
@@ -44,6 +53,7 @@ export type InternalElectricConfig = {
     ssl: boolean
   }
   debug?: boolean
+  connectionBackOffOptions?: ConnectionBackOffOptions
 }
 
 export const hydrateConfig = (config: ElectricConfig): HydratedConfig => {
@@ -68,9 +78,30 @@ export const hydrateConfig = (config: ElectricConfig): HydratedConfig => {
     ssl: sslEnabled,
   }
 
+  const {
+    delayFirstAttempt,
+    jitter,
+    maxDelay,
+    numOfAttempts,
+    startingDelay,
+    timeMultiple,
+  } =
+    config.connectionBackOffOptions ??
+    satelliteDefaults.connectionBackOffOptions
+
+  const connectionBackOffOptions = {
+    delayFirstAttempt,
+    jitter,
+    maxDelay,
+    numOfAttempts,
+    startingDelay,
+    timeMultiple,
+  }
+
   return {
     auth,
     replication,
     debug,
+    connectionBackOffOptions,
   }
 }

clients/typescript/src/satellite/config.ts

@@ -1,5 +1,7 @@
+import { IBackOffOptions } from 'exponential-backoff'
 import { QualifiedTablename } from '../util/tablename'
 
+export type ConnectionBackoffOptions = Omit<IBackOffOptions, 'retry'>
 export interface SatelliteOpts {
   /** The database table where Satellite keeps its processing metadata. */
   metaTable: QualifiedTablename
@@ -18,6 +20,8 @@ export interface SatelliteOpts {
   minSnapshotWindow: number
   /** On reconnect, clear client's state if cannot catch up with Electric buffered WAL*/
   clearOnBehindWindow: boolean
+  /** Backoff options for connecting with Electric*/
+  connectionBackOffOptions: ConnectionBackoffOptions
 }
 
 export interface SatelliteOverrides {
@@ -37,6 +41,14 @@ export const satelliteDefaults: SatelliteOpts = {
   pollingInterval: 2000,
   minSnapshotWindow: 40,
   clearOnBehindWindow: true,
+  connectionBackOffOptions: {
+    delayFirstAttempt: false,
+    startingDelay: 1000,
+    jitter: 'full',
+    maxDelay: 10000,
+    numOfAttempts: 50,
+    timeMultiple: 2,
+  },
 }
 
 export const satelliteClientDefaults = {

clients/typescript/src/satellite/error.ts

@@ -0,0 +1,46 @@
+import { SatelliteError, SatelliteErrorCode } from '../util'
+import Log from 'loglevel'
+
+const fatalErrorDescription =
+  "Client can't connect with the server after a fatal error. This can happen due to divergence between local client and server. Use developer tools to clear the local database, or delete the database file. We're working on tools to allow recovering the state of the local database."
+
+const throwErrors = [
+  SatelliteErrorCode.INTERNAL,
+  SatelliteErrorCode.FATAL_ERROR,
+]
+
+const fatalErrors = [
+  SatelliteErrorCode.INVALID_REQUEST,
+  SatelliteErrorCode.UNKNOWN_SCHEMA_VSN,
+  SatelliteErrorCode.AUTH_REQUIRED,
+]
+
+const outOfSyncErrors = [
+  SatelliteErrorCode.INVALID_POSITION,
+  SatelliteErrorCode.BEHIND_WINDOW,
+  SatelliteErrorCode.SUBSCRIPTION_NOT_FOUND,
+]
+
+export function isThrowable(error: SatelliteError) {
+  return throwErrors.includes(error.code)
+}
+
+export function isFatal(error: SatelliteError) {
+  return fatalErrors.includes(error.code)
+}
+
+export function isOutOfSyncError(error: SatelliteError) {
+  return outOfSyncErrors.includes(error.code)
+}
+
+function logFatalErrorDescription() {
+  Log.error(fatalErrorDescription)
+}
+
+export function wrapFatalError(error: SatelliteError) {
+  logFatalErrorDescription()
+  return new SatelliteError(
+    SatelliteErrorCode.FATAL_ERROR,
+    `Fatal error: ${error.message}. Check log for more information`
+  )
+}