From 5bffc8dd5ab5acb084ae188aab4af7aa42113960 Mon Sep 17 00:00:00 2001 From: Thomas Heinen Date: Thu, 23 Apr 2020 18:52:24 +0200 Subject: [PATCH] Implementation of script-based host queueing --- .rubocop.yml | 10 ++- Gemfile | 2 +- Guardfile | 4 +- README.md | 91 +++++++++++++++++++++++---- Rakefile | 11 ++++ kitchen-static.gemspec | 3 + lib/kitchen/driver/queueing/base.rb | 87 +++++++++++++++++++++++++ lib/kitchen/driver/queueing/script.rb | 49 +++++++++++++++ lib/kitchen/driver/static.rb | 87 ++++++++++++++++++++++++- lib/kitchen/driver/static_version.rb | 2 +- 10 files changed, 326 insertions(+), 20 deletions(-) create mode 100644 lib/kitchen/driver/queueing/base.rb create mode 100644 lib/kitchen/driver/queueing/script.rb diff --git a/.rubocop.yml b/.rubocop.yml index b57096e..c7af514 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,5 +1,9 @@ -# Offense count: 36 -# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. -# URISchemes: http, https +AllCops: + TargetRubyVersion: 2.6 + Exclude: + - Guardfile + - Rakefile + Metrics/LineLength: Max: 147 + diff --git a/Gemfile b/Gemfile index 18d806e..d8aafc1 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ end group :debug do gem "pry" gem "guard" - gem "guard-shell" + gem "guard-rake" end # If you want to load debugging tools into the bundle exec sandbox, diff --git a/Guardfile b/Guardfile index 2cccb15..4ca3613 100644 --- a/Guardfile +++ b/Guardfile @@ -1,5 +1,5 @@ notification :terminal_title, display_message: true -guard :shell do - watch(/\.rb$/) { `rake install:local` } +guard 'rake', task: 'install:local' do + watch(/\.rb$/) end diff --git a/README.md b/README.md index ee91d43..342a496 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ This is a Test Kitchen driver for use in cases, where you have an existing machine, such as a physical server which you want to use for your tests. -The static driver is directly derived from TK's "proxy" driver, -which is relying on legacy plugin infrastructure - making it directly incompatible -with Windows platforms. +The static driver is directly derived from TK's "proxy" driver, +which is relying on legacy plugin infrastructure - making it directly +incompatible with Windows platforms. ## Usage @@ -19,19 +19,88 @@ driver: # now the rest of your kitchen.yml follows ``` -The `host` configuration setting, which specifies the hostname/IP you want tests to run against. +The `host` configuration setting, which specifies the hostname/IP you want tests +to run against. -If you have more than one server, for example when testing specific -hardware drivers, just add a suite for each and override the -`host` value in its section +If you have more than one server, for example when testing specific hardware +drivers, just add a suite for each and override the `host` value in its +section ## Supported Platforms -As this is a pure driver which does not interact with the -instances/VMs, it supports all platforms. Specifically Linux -and Windows work. +As this is a pure driver which does not interact with the instances/VMs, it +supports all platforms. Specifically Linux and Windows are known to work. -## License +## Queueing Feature + +As physical machines are a limited resource and are rarely bought or thrown +away in a TestKitchen context, some sort of queueing mechanism is needed in +bigger environments. + +To enable this feature, set `queueing` to `true` (default: `false`) + +```yaml +driver: + name: static + queueing: true + request: + execute: /usr/local/bin/get-host.sh + release: + execute: /usr/local/bin/release-host.sh $STATIC_HOSTNAME + ... +``` + +Queueing knows two Actions: + +* `request` to obtain the hostname or IP of the machine to use +* `release` to return this host into the pool + +If you are using non-ephemeral test systems, like physical machines, you will +need to trigger some procedure to reset them back to the defined default. Otherwise, +every test will modify the system further until results get unpredictable. + +There currently is just one handler for queueing scenarios: + +* the `script` handler, which executes a local script + +## Driver Options + +| Name | Default | Description | +| ------------------- | --------- | --------------------------------------------- | +| `queueing` | false | If to invoke external actions to get hostname | +| `queueing_timeout` | 3600 | Timeout for queueing operations in seconds. | +| `queueing_handlers` | - | Glob to load external queueing handlers | + +## Queueing Handler `static` + +This handler only executes local commands. These could query remote databases or +even issue more complex programs to obtain/release machines. + +### Parameters for `request` + +| Name | Default | Description | +| ---------------- | --------- | ---------------------------------------------------------- | +| `type` | `script` | | +| `execute` | - | Command to execute | +| `match_hostname` | `^(.*)$` | Regex to specify what to grab from output. Default: All | +| `match_banner` | - | Regex to specify optional banner to grab. Default: Nothing | + +If a banner is grabbed, it's contents are displaed after the message reporting the +hostname. This field can be used for warnings or additional information like access +to management interfaces (ILO, BMC, ...). + +### Parameters for `release` + +| Name | Default | Description | +| ---------- | --------- | ------------------------------------------------------- | +| `type` | `script` | | +| `execute` | - | Command to execute | + +The executed script gets the following environment variables: + +* `STATIC_HOSTNAME`: Hostname or IP of the host to be released + +## License Apache 2.0 (see [LICENSE][license]) diff --git a/Rakefile b/Rakefile index 4f99dd2..de04005 100644 --- a/Rakefile +++ b/Rakefile @@ -19,3 +19,14 @@ YARD::Rake::YardocTask.new do |t| end task default: [:style] + +require "guard" +require "guard/commander" + +desc "Watch for source changes and redeploy Gem" +task :guard do + Guard.start({ no_interactions: true }) + while ::Guard.running do + sleep 0.5 + end +end diff --git a/kitchen-static.gemspec b/kitchen-static.gemspec index 30949b9..8261937 100644 --- a/kitchen-static.gemspec +++ b/kitchen-static.gemspec @@ -19,8 +19,11 @@ Gem::Specification.new do |spec| spec.required_ruby_version = ">= 2.3" spec.add_dependency "test-kitchen", ">= 1.16", "< 3.0" + spec.add_dependency "mixlib-shellout", "~> 3.0" spec.add_development_dependency "bundler", ">= 1.16" + spec.add_development_dependency "guard", "~> 2.16" + spec.add_development_dependency "guard-rake", "~> 1.0" spec.add_development_dependency "rake", "~> 12.0" spec.add_development_dependency "yard", "~> 0.9" end diff --git a/lib/kitchen/driver/queueing/base.rb b/lib/kitchen/driver/queueing/base.rb new file mode 100644 index 0000000..05a8f3f --- /dev/null +++ b/lib/kitchen/driver/queueing/base.rb @@ -0,0 +1,87 @@ +module Kitchen::Driver + class Static + module Queueing + class Base + @options = {} + @request_options = {} + @release_options = {} + + @hostname = nil + @banner = nil + + @env_vars = {} + + attr_reader :options, :request_options, :release_options, :env_vars, :banner + + def initialize(options) + @options = { + queueing_timeout: 3600, + } + + @request_options = {} + @release_options = {} + + setup(options) + + process_kitchen_options(options) + end + + def request(state) + handle_request(state) + end + + def release(state) + @env_vars = { + STATIC_HOSTNAME: state[:hostname], + } + + handle_release(state) + end + + def banner? + ! @banner.nil? + end + + def self.descendants + ObjectSpace.each_object(Class).select { |klass| klass < self } + end + + private + + def setup(_options) + # Add setup and defaults in specific handler + end + + def handle_request(_state) + raise "Implement request handler" + end + + def handle_release(_state) + raise "Implement release handler" + end + + def default_request_options(options = {}) + @request_options.merge!(options) + end + + def default_release_options(options = {}) + @release_options.merge!(options) + end + + def process_kitchen_options(kitchen_options) + @options = kitchen_options + + @request_options.merge!(options[:request]) + @options.delete(:request) + + @release_options.merge!(options[:release]) + @options.delete(:release) + end + + def timeout + options[:queueing_timeout] + end + end + end + end +end diff --git a/lib/kitchen/driver/queueing/script.rb b/lib/kitchen/driver/queueing/script.rb new file mode 100644 index 0000000..8c18c11 --- /dev/null +++ b/lib/kitchen/driver/queueing/script.rb @@ -0,0 +1,49 @@ +require "mixlib/shellout" + +require_relative "base.rb" + +module Kitchen::Driver + class Static + module Queueing + class Script < Base + def setup(_kitchen_options) + default_request_options({ + match_hostname: "^(.*)$", + match_banner: nil, + }) + + default_release_options({}) + end + + def handle_request(_state) + stdout = execute(request_options[:execute]) + + matched = stdout.match(request_options[:match_hostname]) + raise format("Could not extract hostname from '%s' with regular expression /%s/", stdout, request_options[:match_hostname]) unless matched + + # Allow additional feedback from command + @banner = stdout.match(request_options[:match_banner])&.captures&.first if request_options[:match_banner] + + matched.captures.first + end + + def handle_release(_state) + execute(release_options[:execute]) + end + + private + + def execute(command) + raise format("Received empty command") if command.nil? || command.empty? + + cmd = Mixlib::ShellOut.new(command, environment: env_vars, timeout: timeout) + cmd.run_command + + raise format("Error executing `%s`: %s", command, cmd.stderr) if cmd.status != 0 + + cmd.stdout.strip + end + end + end + end +end diff --git a/lib/kitchen/driver/static.rb b/lib/kitchen/driver/static.rb index c863cd9..36d6698 100644 --- a/lib/kitchen/driver/static.rb +++ b/lib/kitchen/driver/static.rb @@ -1,4 +1,5 @@ require "kitchen" + require_relative "static_version" module Kitchen @@ -12,16 +13,98 @@ class Static < Kitchen::Driver::Base plugin_version Kitchen::Driver::STATIC_VERSION - required_config :host + default_config :host, nil + + default_config :queueing, false + default_config :queueing_timeout, 3600 + default_config :queueing_handlers, [] + default_config :request, {} + default_config :release, {} + + BANNER_FORMAT = "[kitchen-static] >>> %s <<<".freeze def create(state) - state[:hostname] = config[:host] + print_version + + state[:hostname] = queueing? ? request(state) : config[:host] + + if queueing? + info format("[kitchen-static] Received %s for testing", state[:hostname]) + info format(BANNER_FORMAT, queueing_handler.banner) if queueing_handler.banner? + end end def destroy(state) + print_version return if state[:hostname].nil? + + release(state) if queueing? + info format("[kitchen-static] Released %s from testing", state[:hostname]) if queueing? + state.delete(:hostname) end + + private + + def request(state) + info format("[kitchen-static] Queueing request via %s handler", queueing_type) + + queueing_handler.request(state) + end + + def release(state) + info format("[kitchen-static] Queueing release via %s handler", queueing_type) + + queueing_handler.release(state) + end + + def queueing? + config[:queueing] === true + end + + def queueing_handler + @queueing_handler ||= queueing_registry[queueing_type].new(config) + end + + def queueing_type + return unless queueing? + + config.fetch(:type, "script") + end + + def queueing_registry + return unless queueing? + + @queueing_registry unless @queueing_registry.nil? || @queueing_registry.empty? + @queueing_registry = {} + + bundled_handlers = File.join(__dir__, "queueing", "*.rb") + require_queueing_handlers(bundled_handlers) + + additional_handlers = config[:queueing_handlers] + additional_handlers.each { |glob| require_queueing_handlers(glob) } + + Queueing::Base.descendants.each do |handler_class| + type = queueing_type_from_class(handler_class) + @queueing_registry[type] = handler_class + + debug format("[kitchen-static] Added queueing handler: %s", type) + end + + @queueing_registry + end + + def require_queueing_handlers(glob) + Dir.glob(glob).each { |file| require file } + end + + def queueing_type_from_class(handler_class) + handler_class.to_s.split("::").last.downcase + end + + def print_version + debug format("Starting kitchen-static %s", Kitchen::Driver::STATIC_VERSION) + end end end end diff --git a/lib/kitchen/driver/static_version.rb b/lib/kitchen/driver/static_version.rb index 6e6dc66..da628f6 100644 --- a/lib/kitchen/driver/static_version.rb +++ b/lib/kitchen/driver/static_version.rb @@ -1,5 +1,5 @@ module Kitchen module Driver - STATIC_VERSION = "0.9.1".freeze + STATIC_VERSION = "0.10.0".freeze end end