diff --git a/lib/kitchen/busser.rb b/lib/kitchen/busser.rb index dae2d16e6..54374c1bf 100644 --- a/lib/kitchen/busser.rb +++ b/lib/kitchen/busser.rb @@ -89,6 +89,13 @@ def diagnose result end + # Returns an array of all files to be copied to the instance + # + # @return [Array] array of local payload files + def local_payload + local_suite_files.concat(helper_files) + end + # Returns a command string which installs Busser, and installs all # required Busser plugins for the suite. # @@ -132,6 +139,26 @@ def setup_cmd Util.wrap_command(cmd, shell) end + # Returns a command string which removes all suite test files on the + # instance. + # + # If no work needs to be performed, for example if there are no tests for + # the given suite, then `nil` will be returned. + # + # @return [String] a command string to remove all suite test files, or + # nil if no work needs to be performed. + def cleanup_cmd + return if local_suite_files.empty? + + cmd = <<-CMD.gsub(/^ {8}/, "") + #{busser_setup_env} + + #{sudo(config[:busser_bin])} suite cleanup + + CMD + Util.wrap_command(cmd, shell) + end + # Returns a command string which transfers all suite test files to the # instance. # diff --git a/lib/kitchen/driver/base.rb b/lib/kitchen/driver/base.rb index a59d8245d..8a01413ba 100644 --- a/lib/kitchen/driver/base.rb +++ b/lib/kitchen/driver/base.rb @@ -92,7 +92,9 @@ def setup(state) # @raise [ActionFailed] if the action could not be completed def verify(state) transport.connection(state) do |conn| - conn.execute(busser.sync_cmd) + conn.execute(busser.cleanup_cmd) + dirs = busser.local_payload.map {|f| File.dirname(f)}.uniq + conn.upload!(dirs, "/tmp/busser/suites") conn.execute(busser.run_cmd) end end diff --git a/lib/kitchen/transport/winrm.rb b/lib/kitchen/transport/winrm.rb index 52ebc631d..1cf5a6572 100644 --- a/lib/kitchen/transport/winrm.rb +++ b/lib/kitchen/transport/winrm.rb @@ -31,7 +31,8 @@ require "kitchen/errors" require "kitchen/login_command" -require "zip" +require 'kitchen/transport/winrm_file_transfer/remote_file' +require 'kitchen/transport/winrm_file_transfer/remote_zip_file' module Kitchen @@ -91,20 +92,13 @@ def wql(query) run(query, :wql) end - # (see Base#upload!) - def upload!(local, remote) - logger.debug("Upload: #{local} -> #{remote}") - local = Array.new(1) { local } if local.is_a? String - shell_id = session.open_shell - local.each do |path| - if File.directory?(path) - upload_directory(shell_id, path, remote) - else - upload_file(path, File.join(remote, File.basename(path)), shell_id) - end - end + # # (see Base#upload!) + def upload!(local_path, remote_path, &block) + local_path = [local_path] if local_path.is_a? String + file = create_remote_file(local_path, remote_path) + file.upload(&block) ensure - session.close_shell(shell_id) + file.close unless file.nil? end # Convert a complex CLIXML Error to a human readable format @@ -183,6 +177,15 @@ def default_port private + def create_remote_file(local_paths, remote_path) + if local_paths.count == 1 && !File.directory?(local_paths[0]) + return WinRMFileTransfer::RemoteFile.new(logger, session, local_paths[0], remote_path) + end + zip_file = WinRMFileTransfer::RemoteZipFile.new(logger, session, remote_path) + local_paths.each { |path| zip_file.add_file(path) } + zip_file + end + # (see Base#establish_connection) def establish_connection rescue_exceptions = [ @@ -306,152 +309,6 @@ def build_winrm_options [endpoint, :plaintext, opts] end - - def upload_file(local, remote, shell_id) - logger.debug("Upload: #{local} -> #{remote}") - remote = full_remote_path(remote, shell_id) - if should_upload_file?(shell_id, local, remote) - upload_to_remote(shell_id, local, remote) - decode_remote_file(shell_id, local, remote) - end - end - - def full_remote_path(path, shell_id) - command = <<-EOH - $dest_file_path = [System.IO.Path]::GetFullPath('#{path}') - - if (!(Test-Path $dest_file_path)) { - $dest_dir = ([System.IO.Path]::GetDirectoryName($dest_file_path)) - New-Item -ItemType directory -Force -Path $dest_dir | Out-Null - } - - $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("#{path}") - EOH - - powershell(command, shell_id)[:data][0][:stdout].chomp - end - - # Checks to see if the target file on the guest is missing or out of date. - # - # @param [String] The id of a shell instance to run the command from - # @param [String] The source file path on the host - # @param [String] The destination file path on the guest - # @return [Boolean] True if the file is missing or out of date - def should_upload_file?(shell_id, local, remote) - logger.debug("comparing #{local} to #{remote}") - local_md5 = Digest::MD5.file(local).hexdigest - command = <<-EOH - $dest_file_path = [System.IO.Path]::GetFullPath('#{remote}') - - if (Test-Path $dest_file_path) { - $crypto_prov = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider - try { - $file = [System.IO.File]::Open($dest_file_path, - [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read) - $guest_md5 = ([System.BitConverter]::ToString($crypto_prov.ComputeHash($file))) - $guest_md5 = $guest_md5.Replace("-","").ToLower() - } - finally { - $file.Dispose() - } - if ($guest_md5 -eq '#{local_md5}') { - exit 0 - } - } - remove-item $dest_file_path -Force - exit 1 - EOH - powershell(command, shell_id)[:exitcode] == 1 - end - - # Uploads the given file to the guest - # - # @param [String] The id of a shell instance to run the command from - # @param [String] The source file path on the host - # @return [String] The temp file path on the guest - def upload_to_remote(shell_id, local, remote) - logger.debug("Uploading '#{local}' to temp file '#{remote}'") - base64_host_file = Base64.encode64(IO.binread(local)).gsub("\n", "") - base64_host_file.chars.to_a.each_slice(8000 - remote.size) do |chunk| - output = cmd("echo #{chunk.join} >> \"#{remote}\"", shell_id) - raise_upload_error_if_failed(output, local, remote) - end - end - - # Recursively uploads the given directory from the host to the guest - # - # @param [String] The id of a shell instance to run the command from - # @param [String] The source file or directory path on the host - # @param [String] The destination file or directory path on the host - def upload_directory(shell_id, local, remote) - zipped = zip_path(local) - return if !File.exist?(zipped) - remote_zip = File.join(remote, File.basename(zipped)) - logger.debug("uploading #{zipped} to #{remote_zip}") - upload_file(zipped, remote_zip, shell_id) - extract_zip(remote_zip, local, shell_id) - end - - def zip_path(path) - path.sub!(%r[/$],'') - archive = File.join(path,File.basename(path))+'.zip' - FileUtils.rm archive, :force=>true - - Zip::File.open(archive, 'w') do |zipfile| - Dir["#{path}/**/**"].reject{|f|f==archive}.each do |file| - entry = Zip::Entry.new(archive, file.sub(path+'/',''), nil, nil, nil, nil, nil, nil, ::Zip::DOSTime.new(2000)) - zipfile.add(entry,file) - end - end - - archive - end - - def extract_zip(remote_zip, local, shell_id) - logger.debug("extracting #{remote_zip} to #{remote_zip.gsub('/','\\').gsub('.zip','')}") - command = <<-EOH - $shellApplication = new-object -com shell.application - $zip_path = "$($env:systemDrive)#{remote_zip.gsub('/','\\')}" - - $zipPackage = $shellApplication.NameSpace($zip_path) - $dest_path = "$($env:systemDrive)#{remote_zip.gsub('/','\\').gsub('.zip','')}" - mkdir $dest_path -ErrorAction SilentlyContinue - $destinationFolder = $shellApplication.NameSpace($dest_path) - $destinationFolder.CopyHere($zipPackage.Items(),0x10) - EOH - - output = powershell(command, shell_id) - raise_upload_error_if_failed(output, local, remote_zip) - end - - # Moves and decodes the given file temp file on the guest to its - # permanent location - # - # @param [String] The id of a shell instance to run the command from - # @param [String] The source base64 encoded temp file path on the guest - # @param [String] The destination file path on the guest - def decode_remote_file(shell_id, local, remote) - logger.debug("Decoding temp file '#{remote}'") - command = <<-EOH - $tmp_file_path = [System.IO.Path]::GetFullPath('#{remote}') - - $dest_dir = ([System.IO.Path]::GetDirectoryName($tmp_file_path)) - New-Item -ItemType directory -Force -Path $dest_dir - - $base64_string = Get-Content $tmp_file_path - $bytes = [System.Convert]::FromBase64String($base64_string) - [System.IO.File]::WriteAllBytes($tmp_file_path, $bytes) - EOH - output = powershell(command, shell_id) - raise_upload_error_if_failed(output, local, remote) - end - - def raise_upload_error_if_failed(output, from, to) - raise TransportFailed, - :from => from, - :to => to, - :message => output.inspect unless output[:exitcode].zero? - end end end end diff --git a/lib/kitchen/transport/winrm_file_transfer/remote_file.rb b/lib/kitchen/transport/winrm_file_transfer/remote_file.rb new file mode 100644 index 000000000..07896cdf5 --- /dev/null +++ b/lib/kitchen/transport/winrm_file_transfer/remote_file.rb @@ -0,0 +1,172 @@ +require 'io/console' +require 'json' +require 'kitchen/transport/winrm_file_transfer/shell' + +module Kitchen + module Transport + module WinRMFileTransfer + class RemoteFile + + attr_reader :local_path + attr_reader :remote_path + attr_reader :closed + attr_reader :options + attr_reader :service + attr_reader :shell + + def initialize(logger, service, local_path, remote_path) + @closed = false + @service = service + @shell = Shell.new(logger, service) + @local_path = local_path + @remote_path = full_remote_path(local_path, remote_path) + @logger = logger + logger.debug("Creating RemoteFile of local '#{local_path}' at '#{@remote_path}'") + ensure + if !shell.nil? + ObjectSpace.define_finalizer( self, self.class.close(shell) ) + end + end + + def upload(&block) + raise TransportFailed.new("This RemoteFile is closed.") if closed + raise TransportFailed.new("Cannot find path: '#{local_path}'") unless File.exist?(local_path) + + @remote_path, should_upload = powershell_batch do | builder | + builder << resolve_remote_command + builder << is_dirty_command + end + + if should_upload == 'True' + size = upload_to_remote(&block) + else + size = 0 + logger.debug("Files are equal. Not copying #{local_path} to #{remote_path}") + end + powershell_batch {|builder| builder << create_post_upload_command} + size + end + + def close + shell.close unless shell.nil? or closed + @closed = true + end + + protected + + attr_reader :logger + + def self.close(shell) + proc { shell.close } + end + + def full_remote_path(local_path, remote_path) + base_file_name = File.basename(local_path) + if File.basename(remote_path) != base_file_name + remote_path = File.join(remote_path, base_file_name) + end + remote_path + end + + def resolve_remote_command + <<-EOH + $dest_file_path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("#{remote_path}") + + if (!(Test-Path $dest_file_path)) { + $dest_dir = ([System.IO.Path]::GetDirectoryName($dest_file_path)) + New-Item -ItemType directory -Force -Path $dest_dir | Out-Null + } + + $dest_file_path + EOH + end + + def is_dirty_command + local_md5 = Digest::MD5.file(local_path).hexdigest + <<-EOH + $dest_file_path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("#{remote_path}") + + if (Test-Path $dest_file_path) { + $crypto_prov = new-object -TypeName System.Security.Cryptography.MD5CryptoServiceProvider + try { + $file = [System.IO.File]::Open($dest_file_path, + [System.IO.Filemode]::Open, [System.IO.FileAccess]::Read) + $guest_md5 = ([System.BitConverter]::ToString($crypto_prov.ComputeHash($file))) + $guest_md5 = $guest_md5.Replace("-","").ToLower() + } + finally { + $file.Dispose() + } + if ($guest_md5 -eq '#{local_md5}') { + return $false + } + } + if(Test-Path $dest_file_path){remove-item $dest_file_path -Force} + return $true + EOH + end + + def upload_to_remote(&block) + logger.debug("Uploading '#{local_path}' to temp file '#{remote_path}'") + base64_host_file = Base64.encode64(IO.binread(local_path)).gsub("\n", "") + base64_array = base64_host_file.chars.to_a + bytes_copied = 0 + base64_array.each_slice(8000 - remote_path.size) do |chunk| + shell.cmd("echo #{chunk.join} >> \"#{remote_path}\"") + bytes_copied += chunk.count + logger.debug("Uploading chunk #{bytes_copied} bytes copied of #{base64_array.count} total bytes") + yield bytes_copied, base64_array.count, local_path, remote_path if block_given? + end + base64_array.length + end + + def decode_command + <<-EOH + $base64_string = Get-Content '#{remote_path}' + try { + $bytes = [System.Convert]::FromBase64String($base64_string) + if($bytes -ne $null){ + [System.IO.File]::WriteAllBytes('#{remote_path}', $bytes) | Out-Null + } + } + catch{} + EOH + end + + def create_post_upload_command + [decode_command] + end + + def powershell_batch(&block) + ps_builder = [] + yield ps_builder + + commands = [ "$result = @{}" ] + idx = 0 + ps_builder.flatten.each do |cmd_item| + commands << <<-EOH + $result.ret#{idx} = Invoke-Command { #{cmd_item} } + EOH + idx += 1 + end + commands << "\"{\";$result.keys | % { write-output \"`\"$_`\": `\"$($result[$_])`\",\".Replace('\\','\\\\')};\"}\"" + + result = [] + begin + result_hash = JSON.parse(shell.powershell(commands.join("\n")).gsub(",\r\n}","\n}")) + result_hash.keys.sort.each do |key| + logger.debug("result key: #{key} is '#{result_hash[key]}'") + result << result_hash[key] unless result_hash[key].nil? + end + rescue TransportFailed => tf + raise TransportFailed, + :from => local_path, + :to => remote_path, + :message => tf.message + end + result unless result.empty? + end + end + end + end +end \ No newline at end of file diff --git a/lib/kitchen/transport/winrm_file_transfer/remote_zip_file.rb b/lib/kitchen/transport/winrm_file_transfer/remote_zip_file.rb new file mode 100644 index 000000000..43f31fb09 --- /dev/null +++ b/lib/kitchen/transport/winrm_file_transfer/remote_zip_file.rb @@ -0,0 +1,63 @@ +require 'zip' + +module Kitchen + module Transport + module WinRMFileTransfer + class RemoteZipFile < RemoteFile + + attr_reader :archive + + def initialize(logger, service, remote_path) + @archive = create_archive(remote_path) + @unzip_remote_path = remote_path + remote_path = "$env:temp/WinRM_ft" + super(logger, service, @archive, remote_path) + end + + def add_file(path) + path = path.gsub("\\","/") + logger.debug("adding '#{path}' to zip file") + raise TransportFailed.new("Cannot find path: '#{path}'") unless File.exist?(path) + File.directory?(path) ? glob = File.join(path, "**/*") : glob = path + logger.debug("iterating files in '#{glob}'") + Zip::File.open(archive, 'w') do |zipfile| + Dir.glob(glob).each do |file| + logger.debug("adding zip entry for '#{file}'") + entry = Zip::Entry.new(archive, file.sub(File.dirname(path)+'/',''), nil, nil, nil, nil, nil, nil, ::Zip::DOSTime.new(2000)) + zipfile.add(entry,file) + end + end + end + + protected + + def create_post_upload_command + super << extract_zip_command + end + + private + + def create_archive(remote_path) + archive_folder = File.join(ENV['TMP'] || ENV['TMPDIR'] || '/tmp', 'WinRM_file_transfer_local') + Dir.mkdir(archive_folder) unless File.exist?(archive_folder) + archive = File.join(archive_folder,File.basename(remote_path))+'.zip' + FileUtils.rm archive, :force=>true + + archive + end + + def extract_zip_command + <<-EOH + $destination = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("#{@unzip_remote_path}") + $shellApplication = new-object -com shell.application + + $zipPackage = $shellApplication.NameSpace('#{remote_path}') + mkdir $destination -ErrorAction SilentlyContinue | Out-Null + $destinationFolder = $shellApplication.NameSpace($destination) + $destinationFolder.CopyHere($zipPackage.Items(),0x10) | Out-Null + EOH + end + end + end + end +end \ No newline at end of file diff --git a/lib/kitchen/transport/winrm_file_transfer/shell.rb b/lib/kitchen/transport/winrm_file_transfer/shell.rb new file mode 100644 index 000000000..53e7004aa --- /dev/null +++ b/lib/kitchen/transport/winrm_file_transfer/shell.rb @@ -0,0 +1,63 @@ +module Kitchen + module Transport + module WinRMFileTransfer + class Shell + def initialize(logger, service) + @logger = logger + @service = service + @shell = reset + os_version = powershell("[environment]::OSVersion.Version.tostring()") + os_version < "6.2" ? @op_limit = 15 : @op_limit = 1500 + @op_limit = @op_limit - 2 #to be safe + end + + def powershell(script) + script = "$ProgressPreference='SilentlyContinue';" + script + @logger.debug("executing powershell script: \n#{script}") + script = script.encode('UTF-16LE', 'UTF-8') + script = Base64.strict_encode64(script) + cmd("powershell", ['-encodedCommand', script]) + end + + def cmd(command, arguments = []) + check_op_count! + + command_output = nil + out_stream = [] + err_stream = [] + @op_count = @op_count + 1 + command_id = @service.run_command(@shell, command, arguments) + command_output = @service.get_command_output(@shell, command_id) do |stdout, stderr| + out_stream << stdout if stdout + err_stream << stderr if stderr + end + @service.cleanup_command(@shell, command_id) + + if !command_output[:exitcode].zero? or !err_stream.empty? + raise TransportFailed, :message => command_output.inspect + end + out_stream.join.chomp + end + + def close + @service.close_shell(@shell) + end + + private + + def reset + close unless @shell.nil? + @shell = @service.open_shell + @op_count = 0 + @logger.debug("resetting winrm shell curent operation limit is #{@op_limit}") + @shell + end + + def check_op_count! + return if @op_limit.nil? + reset if @op_count > @op_limit + end + end + end + end +end \ No newline at end of file