AWS::LoadBalancer: modernize target group lookup; MommaCat#notify: gu…

…ard recursive deadlocks; some better messaging generally

modules/mu/cloud/resource_base.rb

@@ -738,7 +738,7 @@ def dependencies(use_cache: false, debug: false)
                 MU.log "Couldn't find existing resource #{ext_deploy["cloud_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
                 @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: found.mu_name, triggering_node: @mu_name)
               elsif ext_deploy["mu_name"] && ext_deploy["deploy_id"]
-                MU.log "#{ext_deploy["mu_name"]} / #{ext_deploy["deploy_id"]}"
+                MU.log "#{self}: Importing metadata for #{ext_deploy["cloud_type"]} #{ext_deploy["mu_name"]} from #{ext_deploy["deploy_id"]}"
                 found = MU::MommaCat.findStray(
                   @config['cloud'],
                   ext_deploy["cloud_type"],
@@ -748,8 +748,11 @@ def dependencies(use_cache: false, debug: false)
                   dummy_ok: false
                 ).first
 
-                MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR if found.nil?
-                @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
+                if found.nil?
+                  MU.log "Couldn't find existing resource #{ext_deploy["mu_name"]}/#{ext_deploy["deploy_id"]}, #{ext_deploy["cloud_type"]}", MU::ERR
+                else
+                  @deploy.notify(ext_deploy["cloud_type"], found.config["name"], found.deploydata, mu_name: ext_deploy["mu_name"], triggering_node: @mu_name)
+                end
               else
                 MU.log "Trying to find existing deploy, but either the cloud_id is not valid or no mu_name and deploy_id where provided", MU::ERR
               end

modules/mu/deploy.rb

@@ -634,7 +634,7 @@ def createResources(services, mode="create")
             begin
               if myservice['#MUOBJECT'].nil?
                 if @mommacat
-                  ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false)
+                  ext_obj = @mommacat.findLitterMate(type: myservice["#MU_CLOUDCLASS"].cfg_plural, name: myservice['name'], credentials: myservice['credentials'], created_only: true, return_all: false, ignore_missing: !@updating)
                   if @updating and ext_obj
                     ext_obj.config!(myservice)
                   end

modules/mu/mommacat.rb

@@ -547,9 +547,14 @@ def SSHKey
     # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
     # @return [void]
     def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
+      no_write = (@no_artifacts or caller.grep(/\/mommacat\.rb:\d+:in `notify'/))
 
       begin
-        MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
+        if !no_write
+          if !MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id, retries: 10)
+            raise MuError, "Failed to get deployment-notifcation lock for #{@deploy_id}"
+          end
+        end
 
         if !@need_deploy_flush or @deployment.nil? or @deployment.empty?
           loadDeploy(true) # make sure we're saving the latest and greatest
@@ -588,7 +593,9 @@ def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, d
             @deployment[type][key] = data
             MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data
           end
-          save!(key) if !delayed_save and !@no_artifacts
+          if !delayed_save and !no_write
+            save!(key)
+          end
         else
           have_deploy = true
           if @deployment[type].nil? or @deployment[type][key].nil?
@@ -613,10 +620,10 @@ def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, d
               end
             }
           end
-          save! if !delayed_save and !@no_artifacts
+          save! if !delayed_save and !no_write
         end
       ensure
-        MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
+        MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !no_write
       end
     end
 

modules/mu/mommacat/search.rb

@@ -138,7 +138,7 @@ def self.findStray(cloud, type,
     # @param created_only [Boolean]: Only return the littermate if its cloud_id method returns a value
     # @param return_all [Boolean]: Return a Hash of matching objects indexed by their mu_name, instead of a single match. Only valid for resource types where has_multiples is true.
     # @return [MU::Cloud]
-    def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, debug: false, **flags)
+    def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, ignore_missing: false, debug: false, **flags)
       _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type)
 
       # If we specified a habitat, which we may also have done by its shorthand
@@ -173,8 +173,10 @@ def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_on
             end
           end
           if @object_load_fails or !@kittens[type]
-            MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
-            @object_load_fails = true
+            if !ignore_missing
+              MU.log "#{@deploy_id}'s original config has #{@original_config[type].size == 1 ? "a" : @original_config[type].size.to_s} #{type}, but loadObjects could not populate anything from deployment metadata", MU::ERR if !@object_load_fails
+              @object_load_fails = true
+            end
             return nil
           end
         end

modules/mu/mommacat/storage.rb

@@ -123,11 +123,12 @@ def updateBasketofKittens(new_conf, skip_validation: false, new_metadata: nil, s
       MU::Cloud.resource_types.each_pair { |res_type, attrs|
         next if !@deployment.has_key?(attrs[:cfg_plural])
         deletia = []
+# existing_deploys
         @deployment[attrs[:cfg_plural]].each_pair { |res_name, data|
           orig_cfg = findResourceConfig(attrs[:cfg_plural], res_name, (scrub_with || @original_config))
 
-          if orig_cfg.nil?
-            MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE
+          if orig_cfg.nil? and (!data['mu_name'] or data['mu_name'] =~ /^#{Regexp.quote(@deploy_id)}/)
+            MU.log "#{res_type} #{res_name} no longer configured, will remove deployment metadata", MU::NOTICE, details: data
             deletia << res_name
           end
         }
@@ -176,7 +177,7 @@ def self.unlockAll
     # @param id [String]: The lock identifier to release.
     # @param nonblock [Boolean]: Whether to block while waiting for the lock. In non-blocking mode, we simply return false if the lock is not available.
     # return [false, nil]
-    def self.lock(id, nonblock = false, global = false, deploy_id: MU.deploy_id)
+    def self.lock(id, nonblock = false, global = false, retries: 0, deploy_id: MU.deploy_id)
       raise MuError, "Can't pass a nil id to MU::MommaCat.lock" if id.nil?
 
       if !global
@@ -189,6 +190,7 @@ def self.lock(id, nonblock = false, global = false, deploy_id: MU.deploy_id)
         MU.log "Creating #{lockdir}", MU::DEBUG
         Dir.mkdir(lockdir, 0700)
       end
+      nonblock = true if retries > 0
 
       @lock_semaphore.synchronize {
         if @locks[Thread.current.object_id].nil?
@@ -197,11 +199,34 @@ def self.lock(id, nonblock = false, global = false, deploy_id: MU.deploy_id)
 
         @locks[Thread.current.object_id][id] = File.open("#{lockdir}/#{id}.lock", File::CREAT|File::RDWR, 0600)
       }
-      MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG
+
+      MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG, details: caller
+      show_relevant = Proc.new {
+        @locks.each_pair { |thread_id, lock|
+          lock.each_pair { |lockid, lockpath|
+            if lockid == id
+              thread = Thread.list.select { |t| t.object_id == thread_id }.first
+              if thread.object_id != Thread.current.object_id
+                MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
+              end
+            end
+          }
+        }
+      }
       begin
         if nonblock
           if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
-            return false
+            if retries > 0
+              success = false
+              MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) {
+                success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
+              show_relevant.call() if !success
+              }
+              show_relevant.call() if !success
+              return success
+            else
+              return false
+            end
           end
         else
           @locks[Thread.current.object_id][id].flock(File::LOCK_EX)

modules/mu/providers/aws/loadbalancer.rb

@@ -593,13 +593,34 @@ def cloud_desc(use_cache: true)
             @cloud_desc_cache = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
               names: [@cloud_id]
             ).load_balancers.first
-            if @targetgroups.nil? and !@deploy.nil? and
-                @deploy.deployment['loadbalancers'].has_key?(@config['name']) and
-                @deploy.deployment['loadbalancers'][@config['name']].has_key?("targetgroups")
+            if @targetgroups.nil? 
               @targetgroups = {}
-              @deploy.deployment['loadbalancers'][@config['name']]["targetgroups"].each_pair { |tg_name, tg_arn|
-                @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
-              }
+              if !@deploy.nil? and
+                 @deploy.deployment['loadbalancers'] and
+                 @deploy.deployment['loadbalancers'][@config['name']] and
+                 @deploy.deployment['loadbalancers'][@config['name']]["targetgroups"]
+                @deploy.deployment['loadbalancers'][@config['name']]["targetgroups"].each_pair { |tg_name, tg_arn|
+                  @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
+                }
+              else
+                pp @config['targetgroups']
+                MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(load_balancer_arn: @cloud_desc_cache.load_balancer_arn).target_groups.each { |tg|
+                  tg_name = tg.target_group_name
+                  if @config['targetgroups']
+                    @config['targetgroups'].each { |tg_cfg|
+                      if tg_name = @deploy.getResourceName(tg_cfg["name"], max_length: 32, disallowed_chars: /[^A-Za-z0-9-]/)
+                        tg_name = tg_cfg['name']
+                        break
+                      end
+                    }
+                  end
+                  @targetgroups[tg_name] = tg
+                }
+#                @config['targetgroups'].each { |tg|
+#                  tg_name = @deploy.getResourceName(tg["name"], max_length: 32, disallowed_chars: /[^A-Za-z0-9-]/)
+#                  @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
+#                }
+              end
             end
 
             return @cloud_desc_cache

modules/mu/providers/aws/server_pool.rb

@@ -1416,20 +1416,12 @@ def buildOptionsHash
                     # XXX probably have to query API to get the DNS name of this one
                 }
               elsif lb["concurrent_load_balancer"]
-                raise MuError, "No loadbalancers exist! I need one named #{lb['concurrent_load_balancer']}" if !@deploy.deployment["loadbalancers"]
-                found = false
-                @deploy.deployment["loadbalancers"].each_pair { |lb_name, deployed_lb|
-                  if lb_name == lb['concurrent_load_balancer']
-                    lbs << deployed_lb["awsname"] # XXX check for classic
-                    if deployed_lb.has_key?("targetgroups")
-                      deployed_lb["targetgroups"].values.each { |tg_arn|
-                        tg_arns << tg_arn
-                      }
-                    end
-                    found = true
-                  end
-                }
-                raise MuError, "I need a loadbalancer named #{lb['concurrent_load_balancer']}, but none seems to have been created!" if !found
+                lb = @deploy.findLitterMate(name: lb['concurrent_load_balancer'], type: "loadbalancers")
+                raise MuError, "No loadbalancers exist! I need one named #{lb['concurrent_load_balancer']}" if !lb
+                lbs << lb.mu_name
+                if lb.targetgroups
+                  tg_arns = lb.targetgroups.values.map { |tg| tg.target_group_arn }
+                end
               end
             }
             if tg_arns.size > 0