routing: Add automatic failover

lib/readyset.rb

@@ -146,9 +146,7 @@ class << self
 
   def self.healthchecker
     @healthchecker ||= Readyset::Health::Healthchecker.new(
-      healthcheck_interval: config.failover_healthcheck_interval,
-      error_window_period: config.failover_error_window_period,
-      error_window_size: config.failover_error_window_size,
+      config.failover,
       shard: shard,
     )
   end

lib/readyset/configuration.rb

@@ -3,18 +3,26 @@
 
 module Readyset
   class Configuration
-    attr_accessor :enable_failover, :failover_error_window_period, :failover_error_window_size,
-      :failover_healthcheck_interval, :migration_path, :shard
+    attr_accessor :migration_path, :shard
 
     def initialize
-      @enable_failover = false
-      @failover_healthcheck_interval = 5.seconds
-      @failover_error_window_period = 1.minute
-      @failover_error_window_size = 10
       @migration_path = File.join(Rails.root, 'db/readyset_caches.rb')
       @shard = :readyset
     end
 
+    def failover
+      if @failover
+        @failover
+      else
+        inner = ActiveSupport::OrderedOptions.new
+        inner.enabled = false
+        inner.healthcheck_interval = 5.seconds
+        inner.error_window_period = 1.minute
+        inner.error_window_size = 10
+        @failover = inner
+      end
+    end
+
     def hostname
       ActiveRecord::Base.configurations.configs_for(name: shard.to_s).configuration_hash[:host]
     end

lib/readyset/health/healthchecker.rb

@@ -12,14 +12,16 @@ module Health
     class Healthchecker
       UNHEALTHY_ERRORS = [::PG::UnableToSend, ::PG::ConnectionBad].freeze
 
-      def initialize(healthcheck_interval:, error_window_period:, error_window_size:, shard:)
+      def initialize(config, shard:)
         @healthy = Concurrent::AtomicBoolean.new(true)
-        @healthcheck_interval = healthcheck_interval
+        @healthcheck_interval = config.healthcheck_interval!
         @healthchecks = Health::Healthchecks.new(shard: shard)
         @lock = Mutex.new
         @shard = shard
-        @window_counter = Readyset::Utils::WindowCounter.
-          new(window_size: error_window_size, time_period: error_window_period)
+        @window_counter = Readyset::Utils::WindowCounter.new(
+          window_size: config.error_window_size!,
+          time_period: config.error_window_period!,
+        )
       end
 
       # Returns true only if the connection to ReadySet is healthy. ReadySet's health is gauged by

spec/configuration_spec.rb

@@ -17,24 +17,24 @@
       expect(config.migration_path).to eq(expected)
     end
 
-    it 'initializes enable_failover with false' do
+    it 'initializes failover.enabled with false' do
       config = Readyset::Configuration.new
-      expect(config.enable_failover).to eq(false)
+      expect(config.failover.enabled).to eq(false)
     end
 
-    it 'initializes failover_healthcheck_interval to be 5 seconds' do
+    it 'initializes failover.healthcheck_interval to be 5 seconds' do
       config = Readyset::Configuration.new
-      expect(config.failover_healthcheck_interval).to eq(5.seconds)
+      expect(config.failover.healthcheck_interval).to eq(5.seconds)
     end
 
-    it 'initializes failover_error_window_period to be 1 minute' do
+    it 'initializes failover.error_window_period to be 1 minute' do
       config = Readyset::Configuration.new
-      expect(config.failover_error_window_period).to eq(1.minute)
+      expect(config.failover.error_window_period).to eq(1.minute)
     end
 
-    it 'initializes failover_error_window_size to be 10' do
+    it 'initializes failover.error_window_size to be 10' do
       config = Readyset::Configuration.new
-      expect(config.failover_error_window_size).to eq(10)
+      expect(config.failover.error_window_size).to eq(10)
     end
   end
 end

spec/health/healthchecker_spec.rb

@@ -129,12 +129,12 @@ def setup
   private
 
   def build_healthchecker(healthcheck_interval: 5.seconds)
-    Readyset::Health::Healthchecker.new(
-      healthcheck_interval: healthcheck_interval,
-      error_window_period: error_window_period,
-      error_window_size: error_window_size,
-      shard: :readyset
-    )
+    config = ActiveSupport::OrderedOptions.new
+    config.healthcheck_interval = healthcheck_interval
+    config.error_window_period = error_window_period
+    config.error_window_size = error_window_size
+
+    Readyset::Health::Healthchecker.new(config, shard: :readyset)
   end
 
   def error_window_period