Merge pull request #197 from cloudamatic/development

AWS SDK v3

cloud-mu.gemspec

@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.3.2'
-  s.date        = '2020-10-04'
+  s.version     = '3.4.0'
+  s.date        = '2020-10-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"
@@ -36,7 +36,7 @@ EOF
     'https://github.com/cloudamatic/mu'
   s.license       = 'BSD-3-Clause-Attribution'
   s.add_runtime_dependency 'addressable', '~> 2.5'
-  s.add_runtime_dependency "aws-sdk-core", "< 3"
+  s.add_runtime_dependency "aws-sdk", "~> 3.0"
   s.add_runtime_dependency 'azure_sdk', "~> 0.52"
   s.add_runtime_dependency 'bundler', "~> 1.17"
   s.add_runtime_dependency 'chronic_duration', "~> 0.10"

cookbooks/mu-tools/attributes/default.rb

@@ -21,6 +21,13 @@
   disk_name_str = node['hostname']
 end rescue NoMethodError
 
+diskdevs = :xvd
+if !platform_family?("windows")
+  if default['kernel']['modules'].keys.include?("nvme")
+    diskdevs = :nvme
+  end
+end
+
 default['os_updates_using_chef'] = false
 
 default['application_attributes']['application_volume']['mount_directory'] = '/apps'

cookbooks/mu-tools/libraries/helper.rb

@@ -45,6 +45,70 @@ def get_aws_metadata(param)
       nil
     end
 
+    # Just list our block devices
+    # @return [Array<String>]
+    def list_disk_devices
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n | egrep ' disk( |$)'}).stdout.each_line.map { |l|
+          l.chomp.sub(/ .*/, '')
+        }
+      else
+        # XXX something dumber
+        nil
+      end
+    end
+
+    # If we're in AWS and NVME-aware, return a mapping of AWS-side device names
+    # to actual NVME devices.
+    # @return [Hash]
+    def attached_nvme_disks
+      if get_aws_metadata("meta-data/instance-id").nil? or
+         !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
+        return {}
+      end
+      map = {}
+      devices = list_disk_devices
+      return {} if !devices
+      devices.each { |d|
+        if d =~ /^\/dev\/nvme/
+          shell_out(%Q{/sbin/nvme id-ctrl -v #{d}}).stdout.each_line { |desc|
+            if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
+              virt_dev = Regexp.last_match[1]
+              map[virt_dev] = d
+              if !File.exists?(virt_dev)
+                begin
+                  File.symlink(d, virt_dev)
+                rescue Errno::EEXIST # XXX whyyyyy is this needed
+                end
+              end
+              break
+            end
+          }
+        end
+      }
+      map
+    end
+
+    def real_devicepath(dev)
+      map = attached_nvme_disks
+      if map[dev]
+        map[dev]
+      else
+        dev # be nice to actually handle this too
+      end
+    end
+
+    def nvme?
+      if File.executable?("/bin/lsblk")
+        shell_out(%Q{/bin/lsblk -i -p -r -n}).stdout.each_line { |l|
+          return true if l =~ /^\/dev\/nvme\d/
+        }
+      else
+        return true if File.exists?("/dev/nvme0n1")
+      end
+      false
+    end
+
     @project = nil
     @authorizer = nil
     def set_gcp_cfg_params
@@ -186,12 +250,12 @@ def get_deploy_secret
       if cloud == "AWS"
         resp = nil
         begin
+          Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
           resp = s3.get_object(bucket: bucket, key: filename)
         rescue ::Aws::S3::Errors::PermanentRedirect => e
           tmps3 = Aws::S3::Client.new(region: "us-east-1")
           resp = tmps3.get_object(bucket: bucket, key: filename)
         end
-        Chef::Log.info("Fetch deploy secret from s3://#{bucket}/#{filename}")
         secret = resp.body.read
       elsif cloud == "Google"
         include_recipe "mu-tools::gcloud"
@@ -230,6 +294,7 @@ def get_deploy_secret
     end
 
     def mommacat_request(action, arg)
+      params = Base64.urlsafe_encode64(JSON.generate(arg)) if arg
       uri = URI("https://#{get_mu_master_ips.first}:2260/")
       req = Net::HTTP::Post.new(uri)
       res_type = (node['deployment'].has_key?(:server_pools) and node['deployment']['server_pools'].has_key?(node['service_name'])) ? "server_pool" : "server"
@@ -241,21 +306,40 @@ def mommacat_request(action, arg)
         end
 
         Chef::Log.info("Sending Momma Cat #{action} request to #{uri} from #{get_aws_metadata("meta-data/instance-id")}")
+        disks_before = list_disk_devices if action == "add_volume"
+
         req.set_form_data(
           "mu_id" => mu_get_tag_value("MU-ID"),
           "mu_resource_name" => node['service_name'],
           "mu_instance_id" => get_aws_metadata("meta-data/instance-id") || get_google_metadata("name"),
           "mu_resource_type" => res_type,
           "mu_user" => node['deployment']['mu_user'] || node['deployment']['chef_user'],
           "mu_deploy_secret" => secret,
-          action => arg
+          action => params
         )
         http = Net::HTTP.new(uri.hostname, uri.port)
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE # XXX this sucks
         response = http.request(req)
         if response.code != "200"
           Chef::Log.error("Got #{response.code} back from #{uri} on #{action} => #{arg}")
+        else
+          if action == "add_volume" and arg and arg.is_a?(Hash) and arg[:dev]
+            seen_requested = false
+            retries = 0
+            begin
+              list_disk_devices.each { |d|
+                if d == arg[:dev] or
+                   (nvme? and d == attached_nvme_disks[arg[:dev]])
+                  seen_requested = true
+                end
+              }
+              if !seen_requested
+                sleep 6
+                retries += 1
+              end
+            end while retries < 5 and !seen_requested
+          end
         end
       rescue EOFError => e
         # Sometimes deployment metadata is incomplete and missing a

cookbooks/mu-tools/recipes/apply_security.rb

@@ -23,13 +23,37 @@
       include_recipe "mu-tools::aws_api"
       include_recipe "mu-tools::google_api"
 
+      if node['platform_version'].to_i < 6
+        package "policycoreutils"
+      elsif node['platform_version'].to_i < 8
+        package "policycoreutils-python"
+      else
+        package "xfsprogs"
+        package "xfsprogs-devel"
+        package "policycoreutils-python-utils"
+      end
 
-      %w{ policycoreutils-python authconfig ntp aide }.each do |pkg|
+      %w{ authconfig aide }.each do |pkg|
         package "apply_security package #{pkg}" do
           package_name pkg
         end
       end
 
+      if node['platform_version'].to_i < 8
+        package "ntp"
+        bash "NTP" do
+          user "root"
+          code <<-EOH
+    				chkconfig ntpd on
+  	  			ntpdate pool.ntp.org
+  		  		service ntpd start
+          EOH
+        end
+      else
+        package "chrony"
+        service "chronyd"
+      end
+
       execute "enable manual auditd restarts" do
         command "sed -i s/RefuseManualStop=yes/#RefuseManualStop=yes/ /usr/lib/systemd/system/auditd.service ; pkill auditd"
         ignore_failure true
@@ -60,14 +84,6 @@
         content "set -r autologout 15\n"
       end
 
-      bash "NTP" do
-        user "root"
-        code <<-EOH
-  				chkconfig ntpd on
-  				ntpdate pool.ntp.org
-  				service ntpd start
-        EOH
-      end
 
       #File integrity checking. Default configuration
       bash "AIDE" do

cookbooks/mu-tools/recipes/aws_api.rb

@@ -30,3 +30,7 @@
     end
   end
 end
+
+package "nvme-cli" do
+  ignore_failure true
+end

cookbooks/mu-tools/recipes/google_api.rb

@@ -23,3 +23,7 @@
 		only_if { !get_google_metadata("instance/name").nil? }
   end
 }
+
+package "nvme-cli" do
+  ignore_failure true
+end

cookbooks/mu-tools/recipes/rsyslog.rb

@@ -33,7 +33,14 @@
     if platform_family?("rhel") or platform_family?("amazon")
       $rsyslog_ssl_ca_path = "/etc/pki/Mu_CA.pem"
       if !platform?("amazon")
-        package node['platform_version'].to_i < 6 ? "policycoreutils" : "policycoreutils-python"
+        semanage_pkg = if node['platform_version'].to_i < 6
+          "policycoreutils"
+        elsif node['platform_version'].to_i < 8
+          "policycoreutils-python"
+        else
+          "policycoreutils-python-utils"
+        end
+        package semanage_pkg
         execute "allow rsyslog to meddle with port 10514" do
           command "/usr/sbin/semanage port -a -t syslogd_port_t -p tcp 10514"
           not_if "/usr/sbin/semanage port -l | grep '^syslog.*10514'"

cookbooks/mu-tools/resources/disk.rb

@@ -10,25 +10,43 @@
 default_action :create
 
 action :create do
-  device = new_resource.device
+  devicepath = new_resource.device
   path = new_resource.mountpoint
-  devicename = device
+  devicename = devicepath.dup
 
   if set_gcp_cfg_params
     devicename= devicename.gsub(/.*?\//, "")
-    device = "/dev/disk/by-id/google-"+devicename
+    devicepath = "/dev/disk/by-id/google-"+devicename
   end
 
-  mu_tools_mommacat_request "create #{path}" do
+#  if devicename =~ /^\/dev\/(?:sd|xvd)([a-z])/
+#    if nvme?
+#      map = attached_nvme_disks
+#      if map[devicename]
+#        devicepath = map[devicename]
+#      end
+#    end
+#  end
+
+  mu_tools_mommacat_request "create #{devicepath} for #{path}" do
     request "add_volume"
     passparams(
       :dev => devicename,
       :size => new_resource.size,
       :delete_on_termination => new_resource.delete_on_termination
     )
-    not_if { ::File.exist?(device) }
+    not_if { ::File.exist?(real_devicepath(devicepath)) }
   end
 
+#  if nvme? and device.nil?
+#    map = attached_nvme_disks
+#    if map[devicename]
+#      devicepath = map[devicename]
+#    else
+#      Chef::Application.fatal!("In NVME mode and attempted to allocate disk #{devicename}, but didn't find it in metadata of any of our NVME block devices (#{map.values.join(", ")})")
+#    end
+#  end
+
   reboot "Rebooting after adding #{path}" do
     action :nothing
   end
@@ -38,7 +56,7 @@
     action :nothing
   end
   mount "/mnt#{backupname}" do
-    device device
+    device real_devicepath(devicepath)
     options "nodev"
     action :nothing
     notifies :create, "directory[/mnt#{backupname}]", :before
@@ -51,10 +69,11 @@
     action :nothing
   end
 
-  mkfs_cmd = node['platform_version'].to_i == 6 ? "mkfs.ext4 -F #{device}" : "mkfs.xfs -i size=512 #{device}"
-  guard_cmd = node['platform_version'].to_i == 6 ? "tune2fs -l #{device} > /dev/null" : "xfs_admin -l #{device} > /dev/null"
+#  mkfs_cmd = node['platform_version'].to_i == 6 ? "mkfs.ext4 -F #{real_devicepath(devicepath)}" : "mkfs.xfs -i size=512 #{real_devicepath(devicepath)}"
+#  guard_cmd = node['platform_version'].to_i == 6 ? "tune2fs -l #{real_devicepath(devicepath)} > /dev/null" : "xfs_admin -l #{real_devicepath(devicepath)} > /dev/null"
 
-  execute mkfs_cmd do
+  execute "format #{devicename}" do
+    command (node['platform_version'].to_i == 6 ? "mkfs.ext4 -F #{real_devicepath(devicepath)}" : "mkfs.xfs -i size=512 #{real_devicepath(devicepath)}")
     if new_resource.preserve_data
       notifies :mount, "mount[/mnt#{backupname}]", :immediately
       notifies :run, "execute[back up #{backupname}]", :immediately
@@ -63,11 +82,13 @@
     if new_resource.reboot_after_create
       notifies :request_reboot, "reboot[Rebooting after adding #{path}]", :delayed
     end
-    not_if guard_cmd
+    retries 5 # sometimes there's a bit of lag
+    retry_delay 6
+    not_if (node['platform_version'].to_i == 6 ? "tune2fs -l #{real_devicepath(devicepath)} > /dev/null" : "xfs_admin -l #{real_devicepath(devicepath)} > /dev/null")
   end
 
   if !new_resource.reboot_after_create
-    directory "Ensure existence of #{path} for #{device}" do
+    directory "Ensure existence of #{path} for #{real_devicepath(devicepath)}" do
       recursive true
       path path
     end
@@ -78,7 +99,7 @@
     end
 
     mount path do
-      device device
+      device real_devicepath(devicepath)
       options "nodev"
       action [:mount, :enable]
       notifies :run, "execute[/sbin/restorecon -R #{path}]", :immediately

cookbooks/mu-tools/resources/mommacat_request.rb

@@ -6,6 +6,5 @@
 default_action :run
 
 action :run do
-  params = Base64.urlsafe_encode64(JSON.generate(new_resource.passparams))
-  mommacat_request(new_resource.request, params)
+  mommacat_request(new_resource.request, new_resource.passparams)
 end