Retry to NATS client connections in integration

cloudfoundry · Nov 21, 2024 · bd57b05 · bd57b05
1 parent c76e625
commit bd57b05
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 10 deletions.
diff --git a/src/spec/integration_support/director.rb b/src/spec/integration_support/director.rb
@@ -123,8 +123,7 @@ def instances_ps_vitals_failing(options = {})
 
     def start_recording_nats
       Thread.new do
-        @nats_client = NATS::IO::Client.new
-        @nats_client.connect(@director_nats_config)
+        @nats_client = connect_to_nats
         @nats_client.subscribe('>') do |msg, reply, sub|
           @nats_recording << [sub, msg]
         end
@@ -206,6 +205,15 @@ def insert_task(state, type, deployment)
 
     private
 
+    def connect_to_nats
+      # Explicitly retry on AuthErrors to allow time for the NATS sync process to allow the client
+      Bosh::Common.retryable(sleep: 0.5, tries: 20, on: [Errno::ECONNREFUSED, NATS::IO::AuthError]) do
+        nats = NATS::IO::Client.new
+        nats.connect(@director_nats_config)
+        nats
+      end
+    end
+
     def add_defaults(options)
       options[:json] = true
       options[:deployment_name] ||= SharedSupport::DeploymentManifestHelper::DEFAULT_DEPLOYMENT_NAME

diff --git a/src/spec/integration_support/instance.rb b/src/spec/integration_support/instance.rb
@@ -67,8 +67,7 @@ def write_agent_log(file_path, file_contents)
 
     def fail_job
       @logger.info("Failing job #{@vm_cid}")
-      nats = NATS::IO::Client.new
-      nats.connect(@nats_config)
+      nats = connect_to_nats
       msg = JSON.dump(
         method: 'set_dummy_status',
         status: 'failing',
@@ -79,8 +78,7 @@ def fail_job
 
     def fail_start_task
       @logger.info("Failing task #{@vm_cid}")
-      nats = NATS::IO::Client.new
-      nats.connect(@nats_config)
+      nats = connect_to_nats
       msg = JSON.dump(
         method: 'set_task_fail',
         status: 'fail_task',
@@ -136,6 +134,15 @@ def dns_records
 
     private
 
+    def connect_to_nats
+      # Explicitly retry on AuthErrors to allow time for the NATS sync process to allow the client
+      Bosh::Common.retryable(sleep: 0.5, tries: 20, on: [Errno::ECONNREFUSED, NATS::IO::AuthError]) do
+        nats = NATS::IO::Client.new
+        nats.connect(@nats_config)
+        nats
+      end
+    end
+
     def jobs_logs_path
       File.join(@agent_base_dir, 'sys', 'log')
     end

diff --git a/src/spec/integration_support/vm.rb b/src/spec/integration_support/vm.rb
@@ -52,8 +52,7 @@ def write_agent_log(file_path, file_contents)
 
     def fail_job
       @logger.info("Failing job #{@cid}")
-      nats = NATS::IO::Client.new
-      nats.connect(@nats_config)
+      nats = connect_to_nats
       msg = JSON.dump(
           method: 'set_dummy_status',
           status: 'failing',
@@ -64,8 +63,7 @@ def fail_job
 
     def fail_start_task
       @logger.info("Failing task #{@cid}")
-      nats = NATS::IO::Client.new
-      nats.connect(@nats_config)
+      nats = connect_to_nats
       msg = JSON.dump(
           method: 'set_task_fail',
           status: 'fail_task',
@@ -116,6 +114,15 @@ def read_etc_hosts
 
     private
 
+    def connect_to_nats
+      # Explicitly retry on AuthErrors to allow time for the NATS sync process to allow the client
+      Bosh::Common.retryable(sleep: 0.5, tries: 20, on: [Errno::ECONNREFUSED, NATS::IO::AuthError]) do
+        nats = NATS::IO::Client.new
+        nats.connect(@nats_config)
+        nats
+      end
+    end
+
     def jobs_logs_path
       File.join(@agent_base_dir, 'sys', 'log')
     end