From 5a3494906e2981435e773ef6cf5b5d76c8bd9949 Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Sat, 29 Apr 2017 14:25:30 -0400 Subject: [PATCH 01/10] (SIMP-3109) Create a .meta key to store type * Create a .meta key to store metadata for each put. This will be necessary to properly deserialize types. * Add spec tests for atomic_list and make sure that list/ atomic_list filter out .meta keys. * Fix bug in consul's atomic_list implementation SIMP-3109 #close --- lib/puppet_x/libkv/consul_provider.rb | 31 ++++--- lib/puppet_x/libkv/loader.rb | 63 ++++++++++++- lib/puppet_x/libkv/mock_provider.rb | 16 +++- spec/functions/libkv_atomic_get_spec.rb | 4 +- spec/functions/libkv_atomic_list_spec.rb | 112 +++++++++++++++++++++++ spec/functions/libkv_atomic_put_spec.rb | 38 +++++++- spec/functions/libkv_get_spec.rb | 4 +- spec/functions/libkv_put_spec.rb | 32 ++++++- spec/spec_helper.rb | 22 +++-- 9 files changed, 290 insertions(+), 32 deletions(-) create mode 100644 spec/functions/libkv_atomic_list_spec.rb diff --git a/lib/puppet_x/libkv/consul_provider.rb b/lib/puppet_x/libkv/consul_provider.rb index 46af999..1fdcee6 100644 --- a/lib/puppet_x/libkv/consul_provider.rb +++ b/lib/puppet_x/libkv/consul_provider.rb @@ -262,32 +262,41 @@ def atomic_list(params) if (key == nil) throw Exception end + retval = {} begin response = consul_request(path: "/v1/kv" + @basepath + key + "?recurse") if (response.class == Net::HTTPOK) json = response.body value = JSON.parse(json) else - nil + return retval end rescue - nil + return retval end - end - def list(params) - list = atomic_list(params) - retval = {} - key = params['key'] + last_char = key.slice(key.size - 1,1) if (last_char != "/") key = key + "/" end reg = Regexp.new("^" + @basepath.gsub(/\//, "") + key) - - unless (list == nil) - list.each do |entry| + unless (value == nil) + value.each do |entry| nkey = entry["Key"].gsub(reg,"") - retval[nkey] = Base64.decode64(entry["Value"]) + retval[nkey] = entry + retval[nkey]["value"] = Base64.decode64(entry["Value"]) + retval[nkey].delete("Value") + retval[nkey].delete("Key") + end + end + retval + end + def list(params) + list = atomic_list(params) + retval = {} + unless (list == nil) + list.each do |key, entry| + retval[key] = entry["value"] end end retval diff --git a/lib/puppet_x/libkv/loader.rb b/lib/puppet_x/libkv/loader.rb index e1cf0c6..87ef395 100644 --- a/lib/puppet_x/libkv/loader.rb +++ b/lib/puppet_x/libkv/loader.rb @@ -53,9 +53,68 @@ def method_missing(symbol, url, auth, *args, &block) urls[instance] = classes[provider].new(url, auth) end object = urls[instance]; - object.send(symbol, *args, &block); + case symbol + when :put + meta = [] + meta[0] = Hash.new(params) + meta[0]["key"] = "#{params['key']}.meta" + nvalue = {} + nvalue["type"] = puppetype(params["value"]) + nvalue["format"] = "json" + nvalue["mode"] = "puppet" + meta[0]["value"] = nvalue.to_json + object.send(:put, *meta, &block); + when :atomic_put + meta = [] + meta[0] = Hash.new(params) + meta[0]["key"] = "#{params['key']}.meta" + nvalue = {} + nvalue["type"] = puppetype(params["value"]) + nvalue["format"] = "json" + nvalue["mode"] = "puppet" + meta[0]["value"] = nvalue.to_json + object.send(:put, *meta, &block); + end + retval = object.send(symbol, *args, &block); + case symbol + when :list + filtered_list = {} + retval.each do |entry, value| + unless (entry =~ /.*\.meta$/) + filtered_list[entry] = value + end + end + return filtered_list + when :atomic_list + filtered_list = {} + retval.each do |entry, value| + unless (entry =~ /.*\.meta$/) + filtered_list[entry] = value + end + end + return filtered_list + else + return retval + end + end + def puppetype(klass) + retval = nil + case klass.class.to_s + when "Fixnum" + retval = "Integer" + when "Float" + retval = "Float" + when "Array" + retval = "Array" + when "Hash" + retval = "Hash" + when "TrueClass" + retval = "Boolean" + when "FalseClass" + retval = "Boolean" + end + retval end - end # Use the adapter pattern to inject an anonymous diff --git a/lib/puppet_x/libkv/mock_provider.rb b/lib/puppet_x/libkv/mock_provider.rb index 8c8dd17..9ba45f6 100644 --- a/lib/puppet_x/libkv/mock_provider.rb +++ b/lib/puppet_x/libkv/mock_provider.rb @@ -194,7 +194,7 @@ def exists(params) def list(params) retval = {} unless(params.key?('key')) - throw Exception + raise "'key' must be specified" end key = params['key'] hash = @root.select do |k, v| @@ -219,14 +219,24 @@ def list(params) end def atomic_list(params) retval = {} + unless(params.key?('key')) + raise "'key' must be specified" + end key = params['key'] hash = @root.select do |k, v| if (k =~ Regexp.new(key + '/')) - retval["result"] = true + true else - retval["result"] = false + false end end + nlist = {} + hash.each do |k, v| + reg = Regexp.new("^" + key + "/") + rkey = k.gsub(reg,"") + nlist[rkey] = v + end + retval["result"] = nlist if (params['debug'] == true) retval else diff --git a/spec/functions/libkv_atomic_get_spec.rb b/spec/functions/libkv_atomic_get_spec.rb index 5d0cacc..f96c13f 100644 --- a/spec/functions/libkv_atomic_get_spec.rb +++ b/spec/functions/libkv_atomic_get_spec.rb @@ -89,7 +89,7 @@ def set_value(shared) end end datatype_testspec.each do |hash| - it "should return an object of type #{hash[:class]} for /atomic_get/#{hash[:key]}" do + it "should return an object of type #{hash[:nonserial_class]} for /atomic_get/#{hash[:key]}" do params = { 'key' => "/atomic_get/" + hash[:key], 'value' => hash[:value], @@ -100,7 +100,7 @@ def set_value(shared) 'key' => "/atomic_get/" + hash[:key], }.merge(shared_params) result = subject.execute(params) - expect(result["value"].class).to eql(hash[:class]) + expect(result["value"].class).to eql(hash[:nonserial_class]) end it "should return '#{hash[:value]}' for /atomic_get/#{hash[:key]}" do params = { diff --git a/spec/functions/libkv_atomic_list_spec.rb b/spec/functions/libkv_atomic_list_spec.rb new file mode 100644 index 0000000..dbb2ae8 --- /dev/null +++ b/spec/functions/libkv_atomic_list_spec.rb @@ -0,0 +1,112 @@ +#!/usr/bin/env ruby -S rspec +# vim: set expandtab ts=2 sw=2: +require 'spec_helper' +require 'pry' + +valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' +invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" + +describe 'libkv::atomic_list' do + it 'should throw an exception with empty parameters' do + is_expected.to run.with_params({}).and_raise_error(Exception); + end + providers.each do |providerinfo| + provider = providerinfo["name"] + url = providerinfo["url"] + auth = providerinfo["auth"] + shared_params = { + "url" => url, + "auth" => auth, + } + context "when provider = #{provider}" do + context "when the key doesn't exist" do + it 'should return a hash' do + params = { + 'key' => '/test9' + }.merge(shared_params) + result = subject.execute(params) + expect(result.class).to eql(Hash) + end + it 'should return an empty hash' do + params = { + 'key' => '/test9' + }.merge(shared_params) + result = subject.execute(params) + expect(result).to eql({}) + end + end + context "and when the key exists" do + def set_value(shared) + call_function("libkv::put", shared.merge({"value" => "value3"})) + end + it 'should return a hash' do + params = { + 'key' => '/test10' + }.merge(shared_params) + set_value(params.merge({'key' => '/test10/test1', 'value' => 'value10'})) + set_value(params.merge({'key' => '/test10/test2', 'value' => 'value10'})) + result = subject.execute(params) + expect(result.class).to eql(Hash) + end + it 'should return a hash of strings' do + params = { + 'key' => '/test10' + }.merge(shared_params) + set_value(params.merge({'key' => '/test10/test1', 'value' => 'value10'})) + set_value(params.merge({'key' => '/test10/test2', 'value' => 'value10'})) + result = subject.execute(params) + found_non_string = false + result.keys.each do |key| + if (key.class != String) + found_non_string = true + end + end + expect(found_non_string).to eql(false) + end + it 'when passed "fruits" it should return "apple" and "banana"' do + params = { + 'key' => '/test11/fruits' + }.merge(shared_params) + + set_value(params.merge({'key' => '/test11/fruits/apple', 'value' => 'value11'})) + set_value(params.merge({'key' => '/test11/fruits/banana', 'value' => 'value11'})) + set_value(params.merge({'key' => '/test11/meats/beef', 'value' => 'value11'})) + set_value(params.merge({'key' => '/test11/meats/pork', 'value' => 'value11'})) + result = subject.execute(params); + contains = result.key?("apple") and result.key?("banana"); + expect(contains).to eql(true) + end + it 'when passed "meats" it should return "beef" and "pork"' do + params = { + 'key' => '/test12/meats' + }.merge(shared_params) + + set_value(params.merge({'key' => '/test12/fruits/apple', 'value' => 'value12'})) + set_value(params.merge({'key' => '/test12/fruits/banana', 'value' => 'value12'})) + set_value(params.merge({'key' => '/test12/meats/beef', 'value' => 'value12'})) + set_value(params.merge({'key' => '/test12/meats/pork', 'value' => 'value12'})) + + result = subject.execute(params); + contains = result.key?("beef") and result.key?("pork"); + expect(contains).to eql(true) + end + it 'when passed "/test/list/fire" it should return "fox" and "starter" and not "/test/list/fire2/big" or "/test/list/fire2/water"' do + params = { + 'key' => '/test/list/fire' + }.merge(shared_params) + + set_value(params.merge({'key' => '/test/list/fire/fox', 'value' => 'value12'})) + set_value(params.merge({'key' => '/test/list/fire/starter', 'value' => 'value12'})) + set_value(params.merge({'key' => '/test/list/fire2/water', 'value' => 'value12'})) + set_value(params.merge({'key' => '/test/list/fire2/big', 'value' => 'value12'})) + + result = subject.execute(params); + contains = result.key?("fox") and result.key?("starter"); + size = result.size; + expect(contains).to eql(true) + expect(size).to eql(2) + end + end + end + end +end diff --git a/spec/functions/libkv_atomic_put_spec.rb b/spec/functions/libkv_atomic_put_spec.rb index e059683..de098b5 100644 --- a/spec/functions/libkv_atomic_put_spec.rb +++ b/spec/functions/libkv_atomic_put_spec.rb @@ -72,7 +72,7 @@ def set_value(shared) end end datatype_testspec.each do |hash| - it "should return an object of type #{hash[:class]} for /atomic_put/#{hash[:key]}" do + it "should return an object of type #{hash[:nonserial_class]} for /atomic_put/#{hash[:key]}" do params = { 'key' => "/atomic_put/" + hash[:key], }.merge(shared_params) @@ -85,7 +85,7 @@ def set_value(shared) }.merge(shared_params) subject.execute(params) result = call_function("libkv::atomic_get", params) - expect(result["value"].class).to eql(hash[:class]) + expect(result["value"].class).to eql(hash[:nonserial_class]) end it "should return '#{hash[:value]}' for /atomic_put/#{hash[:key]}" do params = { @@ -102,6 +102,40 @@ def set_value(shared) result = call_function("libkv::atomic_get", params) expect(result["value"]).to eql(hash[:retval]) end + unless (hash[:class] == "String") + it "should create the key '/put/#{hash[:key]}.meta' and contain a type = #{hash[:class]}" do + params = { + 'key' => "/atomic_put/" + hash[:key], + }.merge(shared_params) + original = call_function("libkv::atomic_get", params) + + params = { + 'key' => "/atomic_put/" + hash[:key], + 'value' => hash[:value], + 'previous' => original, + }.merge(shared_params) + subject.execute(params) + + params = { + 'key' => "/atomic_put/" + hash[:key] + ".meta", + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result).to_not eql(nil) + expect(result.class).to eql(String) + attempt_to_parse = nil + res = nil + begin + res = JSON.parse(result) + attempt_to_parse = true + rescue + attempt_to_parse = false + end + expect(attempt_to_parse).to eql(true) + expect(res.class).to eql(Hash) + expect(res["type"]).to eql(hash[:class].to_s) + end + + end end end end diff --git a/spec/functions/libkv_get_spec.rb b/spec/functions/libkv_get_spec.rb index ce99440..dd669fb 100644 --- a/spec/functions/libkv_get_spec.rb +++ b/spec/functions/libkv_get_spec.rb @@ -33,7 +33,7 @@ end datatype_testspec.each do |hash| - it "should return an object of type #{hash[:class]} for /get/#{hash[:key]}" do + it "should return an object of type #{hash[:nonserial_class]} for /get/#{hash[:key]}" do params = { 'key' => "/get/" + hash[:key], 'value' => hash[:value], @@ -44,7 +44,7 @@ 'key' => "/get/" + hash[:key], }.merge(shared_params) result = subject.execute(params) - expect(result.class).to eql(hash[:class]) + expect(result.class).to eql(hash[:nonserial_class]) end it "should return '#{hash[:value]}' for /get/#{hash[:key]}" do params = { diff --git a/spec/functions/libkv_put_spec.rb b/spec/functions/libkv_put_spec.rb index d407688..a30d0af 100644 --- a/spec/functions/libkv_put_spec.rb +++ b/spec/functions/libkv_put_spec.rb @@ -36,7 +36,7 @@ expect(result.class).to eql(TrueClass) end datatype_testspec.each do |hash| - it "should create an object of type #{hash[:class]} for /put/#{hash[:key]}" do + it "should create an object of type #{hash[:nonserial_class]} for /put/#{hash[:key]}" do params = { 'key' => "/put/" + hash[:key], 'value' => hash[:value], @@ -47,7 +47,7 @@ 'key' => "/put/" + hash[:key], }.merge(shared_params) result = call_function("libkv::get", params) - expect(result.class).to eql(hash[:class]) + expect(result.class).to eql(hash[:nonserial_class]) end it "should create the value '#{hash[:value]}' for /put/#{hash[:key]}" do params = { @@ -62,6 +62,34 @@ result = call_function("libkv::get", params) expect(result).to eql(hash[:retval]) end + unless (hash[:class] == "String") + it "should create the key '/put/#{hash[:key]}.meta' and contain a type = #{hash[:class]}" do + params = { + 'key' => "/put/" + hash[:key], + 'value' => hash[:value], + }.merge(shared_params) + subject.execute(params) + + params = { + 'key' => "/put/" + hash[:key] + ".meta", + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result).to_not eql(nil) + expect(result.class).to eql(String) + attempt_to_parse = nil + res = nil + begin + res = JSON.parse(result) + attempt_to_parse = true + rescue + attempt_to_parse = false + end + expect(attempt_to_parse).to eql(true) + expect(res.class).to eql(Hash) + expect(res["type"]).to eql(hash[:class].to_s) + end + + end end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ed098f1..3dddb97 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -55,7 +55,7 @@ def set_environment(environment = :production) # # Example: # -# describe 'some::class' do +# describe 'some::nonserial_class' do # context 'with version 10' do # let(:hieradata){ "#{class_name}_v10" } # ... @@ -160,45 +160,51 @@ def datatype_testspec [ # Test String { - :key => "test_string", + :key => "test_string", :value => "test1", :retval => "test1", - :class => String, + :nonserial_class => String, + :class => "String", }, # Test Boolean { :key => "test_boolean", :value => true, :retval => "true", - :class => String, + :nonserial_class => String, + :class => "Boolean", }, # Test Number { :key => "test_number", :value => 255, :retval => '255', - :class => String, + :nonserial_class => String, + :class => "Integer", }, # Test Float { :key => "test_float", :value => 2.38490, :retval => '2.3849', - :class => String, + :nonserial_class => String, + :class => "Float", }, # Test Array { :key => "test_array", :value => [ "test3", "test4"], :retval => '["test3", "test4"]', - :class => String, + :nonserial_class => String, + :class => "Array", }, # Test Hash { :key => "test_hash", :value => { "key" => "test", "value" => "test2" }, :retval => '{"key"=>"test", "value"=>"test2"}', - :class => String, + :nonserial_class => String, + :class => "Hash", }, ] end From b4343cab6386928909b69e7256ef669142376304 Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Mon, 1 May 2017 18:21:55 -0400 Subject: [PATCH 02/10] (SIMP-3060) Fix travisci tests SIMP-3060 #close --- .travis.yml | 8 ++++++-- lib/puppet_x/libkv/consul_provider.rb | 1 + prep_ci.sh | 12 ++++++++++++ spec/functions/libkv_atomic_delete_spec.rb | 1 - spec/functions/libkv_atomic_get_spec.rb | 1 - spec/functions/libkv_atomic_list_spec.rb | 1 - spec/functions/libkv_atomic_put_spec.rb | 1 - spec/functions/libkv_delete_spec.rb | 1 - spec/functions/libkv_deletetree_spec.rb | 1 - spec/functions/libkv_exists_spec.rb | 1 - spec/functions/libkv_get_spec.rb | 1 - spec/functions/libkv_list_spec.rb | 1 - spec/functions/libkv_put_spec.rb | 1 - spec/functions/libkv_supports_spec.rb | 1 - 14 files changed, 19 insertions(+), 13 deletions(-) create mode 100755 prep_ci.sh diff --git a/.travis.yml b/.travis.yml index ed54b79..39ca832 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,14 +7,18 @@ # S6.0.0 4.7 2.1.9 --- language: ruby -sudo: false +sudo: true cache: bundler +services: + - docker + - vagrant before_script: + - ./prep_ci.sh - bundle bundler_args: --without development system_tests before_install: rm Gemfile.lock || true script: - - bundle exec rake test + - docker run -e LOCAL_USER_ID=1000 -it --rm -v $(pwd):/data:Z simpproject/centos:6-ruby21 bundle exec rake spec notifications: email: false rvm: diff --git a/lib/puppet_x/libkv/consul_provider.rb b/lib/puppet_x/libkv/consul_provider.rb index 1fdcee6..e751c6c 100644 --- a/lib/puppet_x/libkv/consul_provider.rb +++ b/lib/puppet_x/libkv/consul_provider.rb @@ -1,6 +1,7 @@ # vim: set expandtab ts=2 sw=2: require 'net/http' require 'uri' +require 'base64' libkv.load("consul") do def initialize(url, auth) @uri = URI.parse(url) diff --git a/prep_ci.sh b/prep_ci.sh new file mode 100755 index 0000000..03b8d0a --- /dev/null +++ b/prep_ci.sh @@ -0,0 +1,12 @@ +#!/bin/sh +docker pull consul +docker run -d -p "8500:8500" -p "8501:8501" -v "$(pwd):/vagrant" -e CONSUL_LOCAL_CONFIG='{ "addresses": { "https":"0.0.0.0" }, "ports" : { "https" : 8501 }, "key_file" : "/vagrant/test/server.key", "cert_file" : "/vagrant/test/server.crt", "ca_file" : "/vagrant/test/ca.crt"}' consul:0.8.0 +docker run -d -p "8504:8500" -p "8503:8501" -v "$(pwd):/vagrant" -e CONSUL_LOCAL_CONFIG='{ "addresses": { "https":"0.0.0.0" }, "ports" : { "https" : 8501 }, "key_file" : "/vagrant/test/server.key", "cert_file" : "/vagrant/test/server.crt", "ca_file" : "/vagrant/test/ca.crt", "verify_incoming": true}' consul:0.8.0 +sleep 5 +for i in $(docker ps -aq) +do + docker inspect "${i}" + docker logs "${i}" +done +curl -kvvvv https://172.17.0.1:8501 +curl -kvvvv --cacert test/ca.pem --cert test/server.crt --key test/server.key https://172.17.0.1:8503 diff --git a/spec/functions/libkv_atomic_delete_spec.rb b/spec/functions/libkv_atomic_delete_spec.rb index a57bf87..b23b6a2 100644 --- a/spec/functions/libkv_atomic_delete_spec.rb +++ b/spec/functions/libkv_atomic_delete_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@\#$%^&*()\"" diff --git a/spec/functions/libkv_atomic_get_spec.rb b/spec/functions/libkv_atomic_get_spec.rb index f96c13f..7fd1fcd 100644 --- a/spec/functions/libkv_atomic_get_spec.rb +++ b/spec/functions/libkv_atomic_get_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_atomic_list_spec.rb b/spec/functions/libkv_atomic_list_spec.rb index dbb2ae8..9e2c621 100644 --- a/spec/functions/libkv_atomic_list_spec.rb +++ b/spec/functions/libkv_atomic_list_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_atomic_put_spec.rb b/spec/functions/libkv_atomic_put_spec.rb index de098b5..80bc784 100644 --- a/spec/functions/libkv_atomic_put_spec.rb +++ b/spec/functions/libkv_atomic_put_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_delete_spec.rb b/spec/functions/libkv_delete_spec.rb index 386fb6f..d40cb6f 100644 --- a/spec/functions/libkv_delete_spec.rb +++ b/spec/functions/libkv_delete_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_deletetree_spec.rb b/spec/functions/libkv_deletetree_spec.rb index cb55b79..b5138ef 100644 --- a/spec/functions/libkv_deletetree_spec.rb +++ b/spec/functions/libkv_deletetree_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_exists_spec.rb b/spec/functions/libkv_exists_spec.rb index b425ea6..8b52620 100644 --- a/spec/functions/libkv_exists_spec.rb +++ b/spec/functions/libkv_exists_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_get_spec.rb b/spec/functions/libkv_get_spec.rb index dd669fb..ad5ce1b 100644 --- a/spec/functions/libkv_get_spec.rb +++ b/spec/functions/libkv_get_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@\#$%^&*()\"" diff --git a/spec/functions/libkv_list_spec.rb b/spec/functions/libkv_list_spec.rb index 70da965..fe33fa0 100644 --- a/spec/functions/libkv_list_spec.rb +++ b/spec/functions/libkv_list_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" diff --git a/spec/functions/libkv_put_spec.rb b/spec/functions/libkv_put_spec.rb index a30d0af..aeacf8e 100644 --- a/spec/functions/libkv_put_spec.rb +++ b/spec/functions/libkv_put_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@\#$%^&*()\"" diff --git a/spec/functions/libkv_supports_spec.rb b/spec/functions/libkv_supports_spec.rb index 83c32e4..d5e8f25 100644 --- a/spec/functions/libkv_supports_spec.rb +++ b/spec/functions/libkv_supports_spec.rb @@ -1,7 +1,6 @@ #!/usr/bin/env ruby -S rspec # vim: set expandtab ts=2 sw=2: require 'spec_helper' -require 'pry' valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' invalid_characters = ";':,./<>?[]\{}|=`~!@#$%^&*()\"" From 9083ed8e3af830b0f03b348b0d4eff0b9a66bb96 Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Sat, 29 Apr 2017 15:37:22 -0400 Subject: [PATCH 03/10] (SIMP-3110) Use .meta to convert a value to the correct type * When serialize is true, convert data into json on put/atomic_put * When serialize is true, convert data from json on get/atomic_get/list/atomic_list based on the data type in .meta * Update all spec tests to support this change * Add a sanitize_input() function that uses symbol_table() to figure out required parameters, and check them for sanity. This should replace the provider specific error checking. SIMP-3110 #close --- lib/puppet_x/libkv/loader.rb | 245 +++++++++++++++++++---- spec/functions/libkv_atomic_get_spec.rb | 16 +- spec/functions/libkv_atomic_list_spec.rb | 40 ++++ spec/functions/libkv_atomic_put_spec.rb | 79 ++++---- spec/functions/libkv_get_spec.rb | 16 +- spec/functions/libkv_list_spec.rb | 40 ++++ spec/functions/libkv_put_spec.rb | 64 +++--- spec/spec_helper.rb | 61 +++--- 8 files changed, 429 insertions(+), 132 deletions(-) diff --git a/lib/puppet_x/libkv/loader.rb b/lib/puppet_x/libkv/loader.rb index 87ef395..01a9973 100644 --- a/lib/puppet_x/libkv/loader.rb +++ b/lib/puppet_x/libkv/loader.rb @@ -23,24 +23,119 @@ def initialize() attr_accessor :classes attr_accessor :urls attr_accessor :default_url - + def load(name, &block) @classes[name] = Class.new(&block) end def parseurl(url) - hash = {} - colonsplit = url.split(":"); - hash['provider'] = colonsplit[0].split("+")[0]; - return hash + hash = {} + colonsplit = url.split(":"); + hash['provider'] = colonsplit[0].split("+")[0]; + return hash + end + def symbol_table() + { + :params => { + 'key' => "KeySpecification", + 'previous' => "Hash", + 'url' => "String", + 'auth' => "Hash", + 'value' => "", + }, + :get => { + 'key' => "required", + }, + :put => { + 'key' => "required", + 'value' => "required", + }, + :delete => { + 'key' => "required", + }, + :exists => { + 'key' => "required", + }, + :list => { + 'key' => "required", + }, + :deletetree => { + 'key' => "required", + }, + :atomic_create => { + 'key' => "required", + 'value' => "required", + }, + :atomic_delete => { + 'key' => "required", + 'previous' => "required", + }, + :atomic_get => { + 'key' => "required", + }, + :atomic_put => { + 'key' => "required", + 'value' => "required", + 'previous' => "required", + }, + :atomic_list => { + 'key' => "required", + }, + } + end + def sanitize_input(symbol, params) + if (params.class.to_s != "Hash") + raise "parameter 0 needs to be a Hash, found #{params.class.to_s}" + end + table = symbol_table + if (table.key?(symbol)) + function_parameters = table[symbol] + function_parameters.each do |name, status| + found = params.key?(name) + case status + when "required" + if (found == false) + raise "parameter: #{name} not found" + end + end + if (found == true) + definition = table[:params][name] + case definition + when "" + if (params[name] == nil) + raise "parameter #{name} should not be nil" + end + when "KeySpecification" + unless (params[name].class.to_s == "String") + raise "parameter #{name} should be String, found #{params[name].class.to_s}" + end + else + unless (params[name].class.to_s == definition) + raise "parameter #{name} should be #{definition}, found #{params[name].class.to_s}" + end + end + end + end + end end def method_missing(symbol, url, auth, *args, &block) - params = args[0] + sanitize_input(symbol, args[0]) + # For safety make a new hash. This doesn't prevent side effects + # but reduces them somewhat + params = args[0].dup + nargs = [ params ] + # ddb hook for testing. + # if (params['dd'] == true) + # binding.pry + # end + unless (params.key?("serialize")) - params["serialize"] = false + params["serialize"] = true end - unless (params.key?("mode")) + serialize = params["serialize"] + if (params.key?("mode") == false or params["mode"] == "" or params["mode"] == nil) params["mode"] = 'puppet' end + if (auth == nil) auth_hash = "" else @@ -55,50 +150,122 @@ def method_missing(symbol, url, auth, *args, &block) object = urls[instance]; case symbol when :put - meta = [] - meta[0] = Hash.new(params) - meta[0]["key"] = "#{params['key']}.meta" - nvalue = {} - nvalue["type"] = puppetype(params["value"]) - nvalue["format"] = "json" - nvalue["mode"] = "puppet" - meta[0]["value"] = nvalue.to_json - object.send(:put, *meta, &block); + if (serialize == true) + meta = get_metadata(params, object) + params["value"] = pack(meta, params["value"]) + end + retval = object.send(symbol, *nargs, &block); when :atomic_put - meta = [] - meta[0] = Hash.new(params) - meta[0]["key"] = "#{params['key']}.meta" - nvalue = {} - nvalue["type"] = puppetype(params["value"]) - nvalue["format"] = "json" - nvalue["mode"] = "puppet" - meta[0]["value"] = nvalue.to_json - object.send(:put, *meta, &block); + if (serialize == true) + meta = get_metadata(params, object) + params["value"] = pack(meta, params["value"]) + end + retval = object.send(symbol, *nargs, &block); + else + retval = object.send(symbol, *nargs, &block); end - retval = object.send(symbol, *args, &block); + case symbol + when :get + if (serialize == true and params["key"] !~ /.*\.meta$/) + metadata = get_metadata(params, object); + return unpack(metadata,retval) + else + return retval + end + when :atomic_get + if (serialize == true and params["key"] !~ /.*\.meta$/) + metadata = get_metadata(params, object); + if (retval.key?("value")) + value = unpack(metadata,retval["value"]) + retval["value"] = value + end + return retval + else + return retval + end when :list - filtered_list = {} - retval.each do |entry, value| - unless (entry =~ /.*\.meta$/) + filtered_list = {} + retval.each do |entry, value| + unless (entry =~ /.*\.meta$/) + if (serialize == true) + metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + filtered_list[entry] = unpack(metadata, value) + else filtered_list[entry] = value end - end - return filtered_list + end + end + return filtered_list when :atomic_list - filtered_list = {} - retval.each do |entry, value| - unless (entry =~ /.*\.meta$/) + filtered_list = {} + retval.each do |entry, value| + unless (entry =~ /.*\.meta$/) + if (serialize == true) + metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + value["value"] = unpack(metadata, value["value"]) filtered_list[entry] = value + else + filtered_list[entry] = value + end + end + end + return filtered_list + else + return retval + end + end + def get_metadata(params, object) + meta = [] + meta[0] = params.dup + meta[0]["key"] = "#{params['key']}.meta" + # XXX FIXME: Make this atomic + if (object.send(:exists, *meta)) + metadata = object.send(:get, *meta) + retval = JSON.parse(metadata) + else + retval = {} + retval["format"] = "json" + retval["mode"] = params["mode"] + if (params.key?("value")) + retval["type"] = puppetype(params["value"]) + meta[0]["value"] = retval.to_json + object.send(:put, *meta); + end + end + return retval + end + def pack(meta, value) + unless (meta["type"] == "String") + # JSON objects need to be real objects, or else the parser blows up. So wrap in a hash + encapsulation = { "value" => value } + encapsulation.to_json + else + value + end + end + def unpack(meta, value) + retval = value + case meta["mode"] + when "puppet" + unless (meta["type"] == "String") + case meta["format"] + when "json" + unless value == nil + object = JSON.parse(value) + retval = object["value"] end - end - return filtered_list + else + raise "Unknown format: #{meta["format"]}" + end + end else - return retval + raise "Unknown mode: #{meta["mode"]}" end + return retval end def puppetype(klass) - retval = nil + retval = klass.class.to_s case klass.class.to_s when "Fixnum" retval = "Integer" diff --git a/spec/functions/libkv_atomic_get_spec.rb b/spec/functions/libkv_atomic_get_spec.rb index 7fd1fcd..15c6162 100644 --- a/spec/functions/libkv_atomic_get_spec.rb +++ b/spec/functions/libkv_atomic_get_spec.rb @@ -88,7 +88,17 @@ def set_value(shared) end end datatype_testspec.each do |hash| - it "should return an object of type #{hash[:nonserial_class]} for /atomic_get/#{hash[:key]}" do + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should return an object of type #{klass} for /atomic_get/#{hash[:key]}" do params = { 'key' => "/atomic_get/" + hash[:key], 'value' => hash[:value], @@ -99,7 +109,7 @@ def set_value(shared) 'key' => "/atomic_get/" + hash[:key], }.merge(shared_params) result = subject.execute(params) - expect(result["value"].class).to eql(hash[:nonserial_class]) + expect(result["value"].class.to_s).to eql(klass) end it "should return '#{hash[:value]}' for /atomic_get/#{hash[:key]}" do params = { @@ -112,7 +122,7 @@ def set_value(shared) 'key' => "/atomic_get/" + hash[:key], }.merge(shared_params) result = subject.execute(params) - expect(result["value"]).to eql(hash[:retval]) + expect(result["value"]).to eql(expected_retval) end end diff --git a/spec/functions/libkv_atomic_list_spec.rb b/spec/functions/libkv_atomic_list_spec.rb index 9e2c621..ca27c1f 100644 --- a/spec/functions/libkv_atomic_list_spec.rb +++ b/spec/functions/libkv_atomic_list_spec.rb @@ -13,9 +13,11 @@ provider = providerinfo["name"] url = providerinfo["url"] auth = providerinfo["auth"] + serialize = providerinfo["serialize"] shared_params = { "url" => url, "auth" => auth, + "serialize" => serialize, } context "when provider = #{provider}" do context "when the key doesn't exist" do @@ -106,6 +108,44 @@ def set_value(shared) expect(size).to eql(2) end end + datatype_testspec.each do |hash| + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should return an object of type #{klass} for /list/#{hash[:key]}" do + params = { + 'key' => "/list/" + hash[:key] + "/value", + 'value' => hash[:value], + }.merge(shared_params) + call_function("libkv::put", params) + + params = { + 'key' => "/list/" + hash[:key], + }.merge(shared_params) + result = subject.execute(params) + expect(result["value"]["value"].class.to_s).to eql(klass) + end + it "should return '#{hash[:value]}' for /list/#{hash[:key]}" do + params = { + 'key' => "/list/" + hash[:key] + "/value", + 'value' => hash[:value], + }.merge(shared_params) + call_function("libkv::put", params) + + params = { + 'key' => "/list/" + hash[:key], + }.merge(shared_params) + result = subject.execute(params) + expect(result["value"]["value"]).to eql(expected_retval) + end + end end end end diff --git a/spec/functions/libkv_atomic_put_spec.rb b/spec/functions/libkv_atomic_put_spec.rb index 80bc784..e44d983 100644 --- a/spec/functions/libkv_atomic_put_spec.rb +++ b/spec/functions/libkv_atomic_put_spec.rb @@ -55,8 +55,8 @@ def set_value(shared) 'key' => '/test/atomic_put/test4' }.merge(shared_params) call_function("libkv::put", params.merge({'key' => '/test/atomic_put/test5', 'value' => 'value5'})) - random = call_function("libkv::get", params.merge({'key' => '/test/atomic_put/test5'})) - result = subject.execute(params.merge({'previous' => random, 'value' => 'value4'})) + random = call_function("libkv::atomic_get", params.merge({'key' => '/test/atomic_put/test5'})) + result = subject.execute(params.merge( { 'previous' => random, 'value' => 'value4'})) expect(result).to eql(false) end it 'should not set the key to value' do @@ -64,14 +64,24 @@ def set_value(shared) 'key' => '/test/atomic_put/test6' }.merge(shared_params) call_function("libkv::put", params.merge({'key' => '/test/atomic_put/test5', 'value' => 'value5'})) - random = call_function("libkv::get", params.merge({'key' => '/test/atomic_put/test5'})) + random = call_function("libkv::atomic_get", params.merge({'key' => '/test/atomic_put/test5'})) subject.execute(params.merge({'previous' => random, 'value' => 'value6'})) result = call_function("libkv::get", params) expect(result).to eql(nil) end end datatype_testspec.each do |hash| - it "should return an object of type #{hash[:nonserial_class]} for /atomic_put/#{hash[:key]}" do + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should return an object of type #{klass} for /atomic_put/#{hash[:key]}" do params = { 'key' => "/atomic_put/" + hash[:key], }.merge(shared_params) @@ -84,9 +94,9 @@ def set_value(shared) }.merge(shared_params) subject.execute(params) result = call_function("libkv::atomic_get", params) - expect(result["value"].class).to eql(hash[:nonserial_class]) + expect(result["value"].class.to_s).to eql(klass) end - it "should return '#{hash[:value]}' for /atomic_put/#{hash[:key]}" do + it "should return '#{expected_retval}' for /atomic_put/#{hash[:key]}" do params = { 'key' => "/atomic_put/" + hash[:key], }.merge(shared_params) @@ -99,41 +109,42 @@ def set_value(shared) }.merge(shared_params) subject.execute(params) result = call_function("libkv::atomic_get", params) - expect(result["value"]).to eql(hash[:retval]) + expect(result["value"]).to eql(expected_retval) end unless (hash[:class] == "String") - it "should create the key '/put/#{hash[:key]}.meta' and contain a type = #{hash[:class]}" do - params = { - 'key' => "/atomic_put/" + hash[:key], - }.merge(shared_params) - original = call_function("libkv::atomic_get", params) + if (providerinfo["serialize"] == true) + it "should create the key '/atomic_put/#{hash[:key]}.meta' and contain a type = #{hash[:puppet_type]}" do + params = { + 'key' => "/atomic_put/" + hash[:key], + }.merge(shared_params) + original = call_function("libkv::atomic_get", params) - params = { - 'key' => "/atomic_put/" + hash[:key], - 'value' => hash[:value], - 'previous' => original, - }.merge(shared_params) - subject.execute(params) + params = { + 'key' => "/atomic_put/" + hash[:key], + 'value' => hash[:value], + 'previous' => original, + }.merge(shared_params) + subject.execute(params) - params = { + params = { 'key' => "/atomic_put/" + hash[:key] + ".meta", - }.merge(shared_params) - result = call_function("libkv::get", params) - expect(result).to_not eql(nil) - expect(result.class).to eql(String) - attempt_to_parse = nil - res = nil - begin - res = JSON.parse(result) - attempt_to_parse = true - rescue - attempt_to_parse = false + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result).to_not eql(nil) + expect(result.class).to eql(String) + attempt_to_parse = nil + res = nil + begin + res = JSON.parse(result) + attempt_to_parse = true + rescue + attempt_to_parse = false + end + expect(attempt_to_parse).to eql(true) + expect(res.class).to eql(Hash) + expect(res["type"]).to eql(hash[:puppet_type].to_s) end - expect(attempt_to_parse).to eql(true) - expect(res.class).to eql(Hash) - expect(res["type"]).to eql(hash[:class].to_s) end - end end end diff --git a/spec/functions/libkv_get_spec.rb b/spec/functions/libkv_get_spec.rb index ad5ce1b..cc315a7 100644 --- a/spec/functions/libkv_get_spec.rb +++ b/spec/functions/libkv_get_spec.rb @@ -32,7 +32,17 @@ end datatype_testspec.each do |hash| - it "should return an object of type #{hash[:nonserial_class]} for /get/#{hash[:key]}" do + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should return an object of type #{klass} for /get/#{hash[:key]}" do params = { 'key' => "/get/" + hash[:key], 'value' => hash[:value], @@ -43,7 +53,7 @@ 'key' => "/get/" + hash[:key], }.merge(shared_params) result = subject.execute(params) - expect(result.class).to eql(hash[:nonserial_class]) + expect(result.class.to_s).to eql(klass) end it "should return '#{hash[:value]}' for /get/#{hash[:key]}" do params = { @@ -56,7 +66,7 @@ 'key' => "/get/" + hash[:key], }.merge(shared_params) result = subject.execute(params) - expect(result).to eql(hash[:retval]) + expect(result).to eql(expected_retval) end end diff --git a/spec/functions/libkv_list_spec.rb b/spec/functions/libkv_list_spec.rb index fe33fa0..00d00fb 100644 --- a/spec/functions/libkv_list_spec.rb +++ b/spec/functions/libkv_list_spec.rb @@ -13,9 +13,11 @@ provider = providerinfo["name"] url = providerinfo["url"] auth = providerinfo["auth"] + serialize = providerinfo["serialize"] shared_params = { "url" => url, "auth" => auth, + "serialize" => serialize, } context "when provider = #{provider}" do context "when the key doesn't exist" do @@ -107,6 +109,44 @@ def set_value(shared) expect(size).to eql(2) end end + datatype_testspec.each do |hash| + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should return an object of type #{klass} for /list/#{hash[:key]}" do + params = { + 'key' => "/list/" + hash[:key] + "/value", + 'value' => hash[:value], + }.merge(shared_params) + call_function("libkv::put", params) + + params = { + 'key' => "/list/" + hash[:key], + }.merge(shared_params) + result = subject.execute(params) + expect(result["value"].class.to_s).to eql(klass) + end + it "should return '#{hash[:value]}' for /list/#{hash[:key]}" do + params = { + 'key' => "/list/" + hash[:key] + "/value", + 'value' => hash[:value], + }.merge(shared_params) + call_function("libkv::put", params) + + params = { + 'key' => "/list/" + hash[:key], + }.merge(shared_params) + result = subject.execute(params) + expect(result["value"]).to eql(expected_retval) + end + end end end end diff --git a/spec/functions/libkv_put_spec.rb b/spec/functions/libkv_put_spec.rb index aeacf8e..66024a3 100644 --- a/spec/functions/libkv_put_spec.rb +++ b/spec/functions/libkv_put_spec.rb @@ -19,6 +19,7 @@ "mode" => providerinfo["mode"], "auth" => auth, } + context "when provider = #{provider}" do it 'should throw an exception when "key" is missing' do is_expected.to run.with_params(shared_params).and_raise_error(Exception) @@ -35,7 +36,17 @@ expect(result.class).to eql(TrueClass) end datatype_testspec.each do |hash| - it "should create an object of type #{hash[:nonserial_class]} for /put/#{hash[:key]}" do + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should create an object of type #{klass} for /put/#{hash[:key]}" do params = { 'key' => "/put/" + hash[:key], 'value' => hash[:value], @@ -46,7 +57,7 @@ 'key' => "/put/" + hash[:key], }.merge(shared_params) result = call_function("libkv::get", params) - expect(result.class).to eql(hash[:nonserial_class]) + expect(result.class.to_s).to eql(klass) end it "should create the value '#{hash[:value]}' for /put/#{hash[:key]}" do params = { @@ -59,35 +70,36 @@ 'key' => "/put/" + hash[:key], }.merge(shared_params) result = call_function("libkv::get", params) - expect(result).to eql(hash[:retval]) + expect(result).to eql(expected_retval) end unless (hash[:class] == "String") - it "should create the key '/put/#{hash[:key]}.meta' and contain a type = #{hash[:class]}" do - params = { - 'key' => "/put/" + hash[:key], - 'value' => hash[:value], - }.merge(shared_params) - subject.execute(params) + if (providerinfo["serialize"] == true) + it "should create the key '/put/#{hash[:key]}.meta' and contain a type = #{hash[:puppet_type]}" do + params = { + 'key' => "/put/" + hash[:key], + 'value' => hash[:value], + }.merge(shared_params) + subject.execute(params) - params = { - 'key' => "/put/" + hash[:key] + ".meta", - }.merge(shared_params) - result = call_function("libkv::get", params) - expect(result).to_not eql(nil) - expect(result.class).to eql(String) - attempt_to_parse = nil - res = nil - begin - res = JSON.parse(result) - attempt_to_parse = true - rescue - attempt_to_parse = false + params = { + 'key' => "/put/" + hash[:key] + ".meta", + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result).to_not eql(nil) + expect(result.class).to eql(String) + attempt_to_parse = nil + res = nil + begin + res = JSON.parse(result) + attempt_to_parse = true + rescue + attempt_to_parse = false + end + expect(attempt_to_parse).to eql(true) + expect(res.class).to eql(Hash) + expect(res["type"]).to eql(hash[:puppet_type].to_s) end - expect(attempt_to_parse).to eql(true) - expect(res.class).to eql(Hash) - expect(res["type"]).to eql(hash[:class].to_s) end - end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3dddb97..7ae4b1d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -162,49 +162,55 @@ def datatype_testspec { :key => "test_string", :value => "test1", - :retval => "test1", - :nonserial_class => String, + :nonserial_retval => "test1", + :nonserial_class => "String", :class => "String", + :puppet_type => "String", }, # Test Boolean { :key => "test_boolean", :value => true, - :retval => "true", - :nonserial_class => String, - :class => "Boolean", + :nonserial_retval => "true", + :nonserial_class => "String", + :class => "TrueClass", + :puppet_type => "Boolean", }, # Test Number { :key => "test_number", :value => 255, - :retval => '255', - :nonserial_class => String, - :class => "Integer", + :nonserial_retval => '255', + :nonserial_class => "String", + :class => "Fixnum", + :puppet_type => "Integer", }, # Test Float { :key => "test_float", :value => 2.38490, - :retval => '2.3849', - :nonserial_class => String, + :nonserial_retval => '2.3849', + :nonserial_class => "String", :class => "Float", + :puppet_type => "Float", }, # Test Array { :key => "test_array", :value => [ "test3", "test4"], - :retval => '["test3", "test4"]', - :nonserial_class => String, + :nonserial_retval => '["test3", "test4"]', + :nonserial_class => "String", :class => "Array", + :puppet_type => "Array", }, # Test Hash { :key => "test_hash", :value => { "key" => "test", "value" => "test2" }, - :retval => '{"key"=>"test", "value"=>"test2"}', - :nonserial_class => String, + :nonserial_retval => '{"key"=>"test", "value"=>"test2"}', + :nonserial_class => "String", :class => "Hash", + :puppet_type => "Hash", }, ] end @@ -220,11 +226,12 @@ def providers() "url" => "mock://", "serialize" => true, }, - { - "name" => "mock with serialize true and mode is 'native'", - "url" => "mock://", - "serialize" => true, - }, + # { + # "name" => "mock with serialize true and mode is 'native'", + # "url" => "mock://", + # "serialize" => true, + # "mode" => 'native', + # }, { "name" => "consul with serialize false and with daemon", "url" => "consul://172.17.0.1:8500/puppet", @@ -239,14 +246,14 @@ def providers() "softfail" => false, "should_error" => false, }, - { - "name" => "consul with serialize true and mode is 'native' and with daemon", - "url" => "consul://172.17.0.1:8500/puppet", - "serialize" => true, - "mode" => 'native', - "softfail" => false, - "should_error" => false, - }, +# { + # "name" => "consul with serialize true and mode is 'native' and with daemon", + # "url" => "consul://172.17.0.1:8500/puppet", + # "serialize" => true, + # "mode" => 'native', + # "softfail" => false, + # "should_error" => false, + # }, { "name" => "consul with ssl and without auth and with daemon", "url" => "consul+ssl+noverify://172.17.0.1:8501/puppet", From ca1887f09459867fdf4e4745e349ec37d062ccee Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Mon, 1 May 2017 18:09:20 -0400 Subject: [PATCH 04/10] (SIMP-3125) Move key regex match into libkv wrapper Now that there is a sanitize_input function, move the 'key' regex check inside it and remove it from the individual functions. SIMP-3125 #close --- lib/puppet/functions/libkv/atomic_create.rb | 12 ------------ lib/puppet/functions/libkv/atomic_delete.rb | 12 ------------ lib/puppet/functions/libkv/atomic_get.rb | 12 ------------ lib/puppet/functions/libkv/atomic_list.rb | 12 ------------ lib/puppet/functions/libkv/atomic_put.rb | 12 ------------ lib/puppet/functions/libkv/delete.rb | 12 ------------ lib/puppet/functions/libkv/deletetree.rb | 12 ------------ lib/puppet/functions/libkv/empty_value.rb | 12 ------------ lib/puppet/functions/libkv/exists.rb | 12 ------------ lib/puppet/functions/libkv/get.rb | 12 ------------ lib/puppet/functions/libkv/info.rb | 12 ------------ lib/puppet/functions/libkv/list.rb | 12 ------------ lib/puppet/functions/libkv/newlock.rb | 12 ------------ lib/puppet/functions/libkv/pop_error.rb | 12 ------------ lib/puppet/functions/libkv/provider.rb | 12 ------------ lib/puppet/functions/libkv/put.rb | 12 ------------ lib/puppet/functions/libkv/supports.rb | 12 ------------ lib/puppet/functions/libkv/watch.rb | 12 ------------ lib/puppet/functions/libkv/watchtree.rb | 12 ------------ lib/puppet_x/libkv/loader.rb | 6 ++++++ scripts/template.erb | 12 ------------ 21 files changed, 6 insertions(+), 240 deletions(-) diff --git a/lib/puppet/functions/libkv/atomic_create.rb b/lib/puppet/functions/libkv/atomic_create.rb index ae10a39..32f78d6 100644 --- a/lib/puppet/functions/libkv/atomic_create.rb +++ b/lib/puppet/functions/libkv/atomic_create.rb @@ -74,18 +74,6 @@ def atomic_create(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.atomic_create(url, auth, params); diff --git a/lib/puppet/functions/libkv/atomic_delete.rb b/lib/puppet/functions/libkv/atomic_delete.rb index 525153f..070de4f 100644 --- a/lib/puppet/functions/libkv/atomic_delete.rb +++ b/lib/puppet/functions/libkv/atomic_delete.rb @@ -74,18 +74,6 @@ def atomic_delete(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.atomic_delete(url, auth, params); diff --git a/lib/puppet/functions/libkv/atomic_get.rb b/lib/puppet/functions/libkv/atomic_get.rb index b8c755e..13951c6 100644 --- a/lib/puppet/functions/libkv/atomic_get.rb +++ b/lib/puppet/functions/libkv/atomic_get.rb @@ -70,18 +70,6 @@ def atomic_get(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.atomic_get(url, auth, params); diff --git a/lib/puppet/functions/libkv/atomic_list.rb b/lib/puppet/functions/libkv/atomic_list.rb index 1df3cf2..9403a23 100644 --- a/lib/puppet/functions/libkv/atomic_list.rb +++ b/lib/puppet/functions/libkv/atomic_list.rb @@ -70,18 +70,6 @@ def atomic_list(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.atomic_list(url, auth, params); diff --git a/lib/puppet/functions/libkv/atomic_put.rb b/lib/puppet/functions/libkv/atomic_put.rb index 518eb39..2398bee 100644 --- a/lib/puppet/functions/libkv/atomic_put.rb +++ b/lib/puppet/functions/libkv/atomic_put.rb @@ -78,18 +78,6 @@ def atomic_put(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.atomic_put(url, auth, params); diff --git a/lib/puppet/functions/libkv/delete.rb b/lib/puppet/functions/libkv/delete.rb index 2567419..16765af 100644 --- a/lib/puppet/functions/libkv/delete.rb +++ b/lib/puppet/functions/libkv/delete.rb @@ -70,18 +70,6 @@ def delete(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = false - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.delete(url, auth, params); diff --git a/lib/puppet/functions/libkv/deletetree.rb b/lib/puppet/functions/libkv/deletetree.rb index d21e037..f465559 100644 --- a/lib/puppet/functions/libkv/deletetree.rb +++ b/lib/puppet/functions/libkv/deletetree.rb @@ -70,18 +70,6 @@ def deletetree(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = false - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.deletetree(url, auth, params); diff --git a/lib/puppet/functions/libkv/empty_value.rb b/lib/puppet/functions/libkv/empty_value.rb index c0a2b25..908b6c7 100644 --- a/lib/puppet/functions/libkv/empty_value.rb +++ b/lib/puppet/functions/libkv/empty_value.rb @@ -57,18 +57,6 @@ def empty_value(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = nil - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.empty_value(url, auth, params); diff --git a/lib/puppet/functions/libkv/exists.rb b/lib/puppet/functions/libkv/exists.rb index 0912f81..c8decfb 100644 --- a/lib/puppet/functions/libkv/exists.rb +++ b/lib/puppet/functions/libkv/exists.rb @@ -70,18 +70,6 @@ def exists(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = nil - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.exists(url, auth, params); diff --git a/lib/puppet/functions/libkv/get.rb b/lib/puppet/functions/libkv/get.rb index b189c82..75797c1 100644 --- a/lib/puppet/functions/libkv/get.rb +++ b/lib/puppet/functions/libkv/get.rb @@ -70,18 +70,6 @@ def get(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = nil - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.get(url, auth, params); diff --git a/lib/puppet/functions/libkv/info.rb b/lib/puppet/functions/libkv/info.rb index 96e9c37..f8cfa8d 100644 --- a/lib/puppet/functions/libkv/info.rb +++ b/lib/puppet/functions/libkv/info.rb @@ -57,18 +57,6 @@ def info(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.info(url, auth, params); diff --git a/lib/puppet/functions/libkv/list.rb b/lib/puppet/functions/libkv/list.rb index 50f7440..0a99436 100644 --- a/lib/puppet/functions/libkv/list.rb +++ b/lib/puppet/functions/libkv/list.rb @@ -70,18 +70,6 @@ def list(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = {} - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.list(url, auth, params); diff --git a/lib/puppet/functions/libkv/newlock.rb b/lib/puppet/functions/libkv/newlock.rb index d618dac..e5e3159 100644 --- a/lib/puppet/functions/libkv/newlock.rb +++ b/lib/puppet/functions/libkv/newlock.rb @@ -51,18 +51,6 @@ def newlock(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.newlock(url, auth, params); diff --git a/lib/puppet/functions/libkv/pop_error.rb b/lib/puppet/functions/libkv/pop_error.rb index 7afdf30..91b4d00 100644 --- a/lib/puppet/functions/libkv/pop_error.rb +++ b/lib/puppet/functions/libkv/pop_error.rb @@ -57,18 +57,6 @@ def pop_error(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = "" - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.pop_error(url, auth, params); diff --git a/lib/puppet/functions/libkv/provider.rb b/lib/puppet/functions/libkv/provider.rb index 2d94b79..80a76b6 100644 --- a/lib/puppet/functions/libkv/provider.rb +++ b/lib/puppet/functions/libkv/provider.rb @@ -57,18 +57,6 @@ def provider(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = "" - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.provider(url, auth, params); diff --git a/lib/puppet/functions/libkv/put.rb b/lib/puppet/functions/libkv/put.rb index 11c43ed..6bb2dd7 100644 --- a/lib/puppet/functions/libkv/put.rb +++ b/lib/puppet/functions/libkv/put.rb @@ -74,18 +74,6 @@ def put(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = false - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.put(url, auth, params); diff --git a/lib/puppet/functions/libkv/supports.rb b/lib/puppet/functions/libkv/supports.rb index bd973d8..afb7a94 100644 --- a/lib/puppet/functions/libkv/supports.rb +++ b/lib/puppet/functions/libkv/supports.rb @@ -57,18 +57,6 @@ def supports(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = [] - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.supports(url, auth, params); diff --git a/lib/puppet/functions/libkv/watch.rb b/lib/puppet/functions/libkv/watch.rb index f950841..936016c 100644 --- a/lib/puppet/functions/libkv/watch.rb +++ b/lib/puppet/functions/libkv/watch.rb @@ -51,18 +51,6 @@ def watch(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.watch(url, auth, params); diff --git a/lib/puppet/functions/libkv/watchtree.rb b/lib/puppet/functions/libkv/watchtree.rb index 12eddb9..097b5d8 100644 --- a/lib/puppet/functions/libkv/watchtree.rb +++ b/lib/puppet/functions/libkv/watchtree.rb @@ -51,18 +51,6 @@ def watchtree(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.watchtree(url, auth, params); diff --git a/lib/puppet_x/libkv/loader.rb b/lib/puppet_x/libkv/loader.rb index 01a9973..114d199 100644 --- a/lib/puppet_x/libkv/loader.rb +++ b/lib/puppet_x/libkv/loader.rb @@ -108,6 +108,12 @@ def sanitize_input(symbol, params) unless (params[name].class.to_s == "String") raise "parameter #{name} should be String, found #{params[name].class.to_s}" end + regex = /^\/[a-zA-Z0-9._\-\/]*$/ + error_msg = "the value of '#{name}': '#{params[name]}' does not match regex '#{regex}'" + unless (regex =~ params[name]) + raise error_msg + end + else unless (params[name].class.to_s == definition) raise "parameter #{name} should be #{definition}, found #{params[name].class.to_s}" diff --git a/scripts/template.erb b/scripts/template.erb index 030629d..e25e761 100644 --- a/scripts/template.erb +++ b/scripts/template.erb @@ -84,18 +84,6 @@ def <%= function %>(params) auth = call_function('lookup', 'libkv::auth', { 'default_value' => nil }) end params["auth"] = auth - if params.key?('key') - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the specified key, '#{params['key']}' does not match regex '#{regex}'" - unless (regex =~ params['key']) - if (params["softfail"] == true) - retval = <%= value[:softfail] %> - return retval - else - raise "the specified key, '#{params['key']}' does not match regex '#{regex}'" - end - end - end if (params["softfail"] == true) begin retval = libkv.<%= function %>(url, auth, params); From 1fc7fa244c1638fda2d56370c039fad8d85f3c3e Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Mon, 1 May 2017 20:56:46 -0400 Subject: [PATCH 05/10] (SIMP-3122) Move the libkv wrapper outside the loader SIMP-3122 #close --- lib/puppet_x/libkv/libkv.rb | 298 +++++++++++++++++++++++++++++++++++ lib/puppet_x/libkv/loader.rb | 290 +--------------------------------- 2 files changed, 301 insertions(+), 287 deletions(-) create mode 100644 lib/puppet_x/libkv/libkv.rb diff --git a/lib/puppet_x/libkv/libkv.rb b/lib/puppet_x/libkv/libkv.rb new file mode 100644 index 0000000..3ffd760 --- /dev/null +++ b/lib/puppet_x/libkv/libkv.rb @@ -0,0 +1,298 @@ +# vim: set expandtab ts=2 sw=2: +@classes = {} +@urls = {} +@default_url = "" +def classes() + @classes +end +def classes=(value) + @classes = value +end +def urls() + @urls +end +def urls=(value) + @urls = value +end +def default_url() + @default_url +end +def default_url=(value) + @default_url = value +end + +def load(name, &block) + @classes[name] = Class.new(&block) +end +def parseurl(url) + hash = {} + colonsplit = url.split(":"); + hash['provider'] = colonsplit[0].split("+")[0]; + return hash +end +def symbol_table() + { + :params => { + 'key' => "KeySpecification", + 'previous' => "Hash", + 'url' => "String", + 'auth' => "Hash", + 'value' => "", + }, + :get => { + 'key' => "required", + }, + :put => { + 'key' => "required", + 'value' => "required", + }, + :delete => { + 'key' => "required", + }, + :exists => { + 'key' => "required", + }, + :list => { + 'key' => "required", + }, + :deletetree => { + 'key' => "required", + }, + :atomic_create => { + 'key' => "required", + 'value' => "required", + }, + :atomic_delete => { + 'key' => "required", + 'previous' => "required", + }, + :atomic_get => { + 'key' => "required", + }, + :atomic_put => { + 'key' => "required", + 'value' => "required", + 'previous' => "required", + }, + :atomic_list => { + 'key' => "required", + }, + } +end +def sanitize_input(symbol, params) + if (params.class.to_s != "Hash") + raise "parameter 0 needs to be a Hash, found #{params.class.to_s}" + end + table = symbol_table + if (table.key?(symbol)) + function_parameters = table[symbol] + function_parameters.each do |name, status| + found = params.key?(name) + case status + when "required" + if (found == false) + raise "parameter: #{name} not found" + end + end + if (found == true) + definition = table[:params][name] + case definition + when "" + if (params[name] == nil) + raise "parameter #{name} should not be nil" + end + when "KeySpecification" + unless (params[name].class.to_s == "String") + raise "parameter #{name} should be String, found #{params[name].class.to_s}" + end + regex = /^\/[a-zA-Z0-9._\-\/]*$/ + error_msg = "the value of '#{name}': '#{params[name]}' does not match regex '#{regex}'" + unless (regex =~ params[name]) + raise error_msg + end + + else + unless (params[name].class.to_s == definition) + raise "parameter #{name} should be #{definition}, found #{params[name].class.to_s}" + end + end + end + end + end +end +def method_missing(symbol, url, auth, *args, &block) + sanitize_input(symbol, args[0]) + # For safety make a new hash. This doesn't prevent side effects + # but reduces them somewhat + params = args[0].dup + nargs = [ params ] + # ddb hook for testing. + # if (params['dd'] == true) + # binding.pry + # end + + unless (params.key?("serialize")) + params["serialize"] = true + end + serialize = params["serialize"] + if (params.key?("mode") == false or params["mode"] == "" or params["mode"] == nil) + params["mode"] = 'puppet' + end + + if (auth == nil) + auth_hash = "" + else + auth_hash = auth.hash + end + instance = url + "@" + auth_hash.to_s + if (urls[instance] == nil) + urlspec = parseurl(url) + provider = urlspec['provider'] + urls[instance] = classes[provider].new(url, auth) + end + object = urls[instance]; + case symbol + when :put + if (serialize == true) + meta = get_metadata(params, object) + params["value"] = pack(meta, params["value"]) + end + retval = object.send(symbol, *nargs, &block); + when :atomic_put + if (serialize == true) + meta = get_metadata(params, object) + params["value"] = pack(meta, params["value"]) + end + retval = object.send(symbol, *nargs, &block); + else + retval = object.send(symbol, *nargs, &block); + end + + case symbol + when :get + if (serialize == true and params["key"] !~ /.*\.meta$/) + metadata = get_metadata(params, object); + return unpack(metadata,retval) + else + return retval + end + when :atomic_get + if (serialize == true and params["key"] !~ /.*\.meta$/) + metadata = get_metadata(params, object); + if (retval.key?("value")) + value = unpack(metadata,retval["value"]) + retval["value"] = value + end + return retval + else + return retval + end + when :list + filtered_list = {} + retval.each do |entry, value| + unless (entry =~ /.*\.meta$/) + if (serialize == true) + metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + filtered_list[entry] = unpack(metadata, value) + else + filtered_list[entry] = value + end + end + end + return filtered_list + when :atomic_list + filtered_list = {} + retval.each do |entry, value| + unless (entry =~ /.*\.meta$/) + if (serialize == true) + metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + value["value"] = unpack(metadata, value["value"]) + filtered_list[entry] = value + else + filtered_list[entry] = value + end + end + end + return filtered_list + else + return retval + end +end +def get_metadata(params, object) + meta = [] + meta[0] = params.dup + meta[0]["key"] = "#{params['key']}.meta" + # XXX FIXME: Make this atomic + if (object.send(:exists, *meta)) + metadata = object.send(:get, *meta) + retval = JSON.parse(metadata) + else + retval = {} + retval["format"] = "json" + retval["mode"] = params["mode"] + if (params.key?("value")) + retval["type"] = puppetype(params["value"]) + meta[0]["value"] = retval.to_json + object.send(:put, *meta); + end + end + return retval +end +def pack(meta, value) + unless (meta["type"] == "String") + # JSON objects need to be real objects, or else the parser blows up. So wrap in a hash + encapsulation = { "value" => value } + encapsulation.to_json + else + value + end +end +def unpack(meta, value) + retval = value + case meta["mode"] + when "puppet" + unless (meta["type"] == "String") + case meta["format"] + when "json" + unless value == nil + object = JSON.parse(value) + retval = object["value"] + end + else + raise "Unknown format: #{meta["format"]}" + end + end + else + raise "Unknown mode: #{meta["mode"]}" + end + return retval +end +def puppetype(klass) + retval = klass.class.to_s + case klass.class.to_s + when "Fixnum" + retval = "Integer" + when "Float" + retval = "Float" + when "Array" + retval = "Array" + when "Hash" + retval = "Hash" + when "TrueClass" + retval = "Boolean" + when "FalseClass" + retval = "Boolean" + end + retval +end + +# Every file in lib/puppet_x/libkv/*_provider.rb is assumed +# to contain a libkv backend provider, and we load them in. +# +# Every provider uses $LIBKV.load() to actually define itself, +# which results in catalog.libkv.classes['providername'] to return +# the Class that implements the provider. +providerglob = File.dirname(File.dirname(File.dirname(File.dirname(File.dirname(__FILE__))))) + "/*/lib/puppet_x/libkv/*_provider.rb" +Dir.glob(providerglob) do |filename| + self.instance_eval File.read(filename), filename +end diff --git a/lib/puppet_x/libkv/loader.rb b/lib/puppet_x/libkv/loader.rb index 114d199..94578f6 100644 --- a/lib/puppet_x/libkv/loader.rb +++ b/lib/puppet_x/libkv/loader.rb @@ -14,281 +14,6 @@ def libkv=(value) # # This is anonymous to allow for code updating. -c = Class.new do - def initialize() - @classes = {} - @urls = {} - @default_url = "" - end - attr_accessor :classes - attr_accessor :urls - attr_accessor :default_url - - def load(name, &block) - @classes[name] = Class.new(&block) - end - def parseurl(url) - hash = {} - colonsplit = url.split(":"); - hash['provider'] = colonsplit[0].split("+")[0]; - return hash - end - def symbol_table() - { - :params => { - 'key' => "KeySpecification", - 'previous' => "Hash", - 'url' => "String", - 'auth' => "Hash", - 'value' => "", - }, - :get => { - 'key' => "required", - }, - :put => { - 'key' => "required", - 'value' => "required", - }, - :delete => { - 'key' => "required", - }, - :exists => { - 'key' => "required", - }, - :list => { - 'key' => "required", - }, - :deletetree => { - 'key' => "required", - }, - :atomic_create => { - 'key' => "required", - 'value' => "required", - }, - :atomic_delete => { - 'key' => "required", - 'previous' => "required", - }, - :atomic_get => { - 'key' => "required", - }, - :atomic_put => { - 'key' => "required", - 'value' => "required", - 'previous' => "required", - }, - :atomic_list => { - 'key' => "required", - }, - } - end - def sanitize_input(symbol, params) - if (params.class.to_s != "Hash") - raise "parameter 0 needs to be a Hash, found #{params.class.to_s}" - end - table = symbol_table - if (table.key?(symbol)) - function_parameters = table[symbol] - function_parameters.each do |name, status| - found = params.key?(name) - case status - when "required" - if (found == false) - raise "parameter: #{name} not found" - end - end - if (found == true) - definition = table[:params][name] - case definition - when "" - if (params[name] == nil) - raise "parameter #{name} should not be nil" - end - when "KeySpecification" - unless (params[name].class.to_s == "String") - raise "parameter #{name} should be String, found #{params[name].class.to_s}" - end - regex = /^\/[a-zA-Z0-9._\-\/]*$/ - error_msg = "the value of '#{name}': '#{params[name]}' does not match regex '#{regex}'" - unless (regex =~ params[name]) - raise error_msg - end - - else - unless (params[name].class.to_s == definition) - raise "parameter #{name} should be #{definition}, found #{params[name].class.to_s}" - end - end - end - end - end - end - def method_missing(symbol, url, auth, *args, &block) - sanitize_input(symbol, args[0]) - # For safety make a new hash. This doesn't prevent side effects - # but reduces them somewhat - params = args[0].dup - nargs = [ params ] - # ddb hook for testing. - # if (params['dd'] == true) - # binding.pry - # end - - unless (params.key?("serialize")) - params["serialize"] = true - end - serialize = params["serialize"] - if (params.key?("mode") == false or params["mode"] == "" or params["mode"] == nil) - params["mode"] = 'puppet' - end - - if (auth == nil) - auth_hash = "" - else - auth_hash = auth.hash - end - instance = url + "@" + auth_hash.to_s - if (urls[instance] == nil) - urlspec = parseurl(url) - provider = urlspec['provider'] - urls[instance] = classes[provider].new(url, auth) - end - object = urls[instance]; - case symbol - when :put - if (serialize == true) - meta = get_metadata(params, object) - params["value"] = pack(meta, params["value"]) - end - retval = object.send(symbol, *nargs, &block); - when :atomic_put - if (serialize == true) - meta = get_metadata(params, object) - params["value"] = pack(meta, params["value"]) - end - retval = object.send(symbol, *nargs, &block); - else - retval = object.send(symbol, *nargs, &block); - end - - case symbol - when :get - if (serialize == true and params["key"] !~ /.*\.meta$/) - metadata = get_metadata(params, object); - return unpack(metadata,retval) - else - return retval - end - when :atomic_get - if (serialize == true and params["key"] !~ /.*\.meta$/) - metadata = get_metadata(params, object); - if (retval.key?("value")) - value = unpack(metadata,retval["value"]) - retval["value"] = value - end - return retval - else - return retval - end - when :list - filtered_list = {} - retval.each do |entry, value| - unless (entry =~ /.*\.meta$/) - if (serialize == true) - metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) - filtered_list[entry] = unpack(metadata, value) - else - filtered_list[entry] = value - end - end - end - return filtered_list - when :atomic_list - filtered_list = {} - retval.each do |entry, value| - unless (entry =~ /.*\.meta$/) - if (serialize == true) - metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) - value["value"] = unpack(metadata, value["value"]) - filtered_list[entry] = value - else - filtered_list[entry] = value - end - end - end - return filtered_list - else - return retval - end - end - def get_metadata(params, object) - meta = [] - meta[0] = params.dup - meta[0]["key"] = "#{params['key']}.meta" - # XXX FIXME: Make this atomic - if (object.send(:exists, *meta)) - metadata = object.send(:get, *meta) - retval = JSON.parse(metadata) - else - retval = {} - retval["format"] = "json" - retval["mode"] = params["mode"] - if (params.key?("value")) - retval["type"] = puppetype(params["value"]) - meta[0]["value"] = retval.to_json - object.send(:put, *meta); - end - end - return retval - end - def pack(meta, value) - unless (meta["type"] == "String") - # JSON objects need to be real objects, or else the parser blows up. So wrap in a hash - encapsulation = { "value" => value } - encapsulation.to_json - else - value - end - end - def unpack(meta, value) - retval = value - case meta["mode"] - when "puppet" - unless (meta["type"] == "String") - case meta["format"] - when "json" - unless value == nil - object = JSON.parse(value) - retval = object["value"] - end - else - raise "Unknown format: #{meta["format"]}" - end - end - else - raise "Unknown mode: #{meta["mode"]}" - end - return retval - end - def puppetype(klass) - retval = klass.class.to_s - case klass.class.to_s - when "Fixnum" - retval = "Integer" - when "Float" - retval = "Float" - when "Array" - retval = "Array" - when "Hash" - retval = "Hash" - when "TrueClass" - retval = "Boolean" - when "FalseClass" - retval = "Boolean" - end - retval - end -end # Use the adapter pattern to inject an anonymous # module into the catalog, so we get a libkv @@ -298,17 +23,8 @@ def puppetype(klass) # Basically, there is no constants assigned to any # libkv code, so there is no risk of environment or catalog # poisoning if the underlying module is updated. +libkv = Object.new() +libkv.instance_eval(File.read(File.dirname(__FILE__) + "/libkv.rb"), File.dirname(__FILE__) + "/libkv.rb") -self.libkv = c.new() -libkv = self.libkv +self.libkv = libkv -# Every file in lib/puppet_x/libkv/*_provider.rb is assumed -# to contain a libkv backend provider, and we load them in. -# -# Every provider uses $LIBKV.load() to actually define itself, -# which results in catalog.libkv.classes['providername'] to return -# the Class that implements the provider. -providerglob = File.dirname(File.dirname(File.dirname(File.dirname(File.dirname(__FILE__))))) + "/*/lib/puppet_x/libkv/*_provider.rb" -Dir.glob(providerglob) do |filename| - self.instance_eval File.read(filename), filename -end From a97bf2f891c6ad57b948d41d0a91bb58d478d0e3 Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Tue, 2 May 2017 09:34:17 -0400 Subject: [PATCH 06/10] (SIMP-3127) libkv can't list / since metadata update SIMP-3127 #close --- lib/puppet_x/libkv/libkv.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/puppet_x/libkv/libkv.rb b/lib/puppet_x/libkv/libkv.rb index 3ffd760..fc60773 100644 --- a/lib/puppet_x/libkv/libkv.rb +++ b/lib/puppet_x/libkv/libkv.rb @@ -192,7 +192,11 @@ def method_missing(symbol, url, auth, *args, &block) retval.each do |entry, value| unless (entry =~ /.*\.meta$/) if (serialize == true) - metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + unless (params['key'] == '/') + metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + else + metadata = get_metadata(params.merge({ "key" => "/#{entry}" }), object) + end filtered_list[entry] = unpack(metadata, value) else filtered_list[entry] = value @@ -205,7 +209,11 @@ def method_missing(symbol, url, auth, *args, &block) retval.each do |entry, value| unless (entry =~ /.*\.meta$/) if (serialize == true) - metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + unless (params['key'] == '/') + metadata = get_metadata(params.merge({ "key" => "#{params['key']}/#{entry}" }), object) + else + metadata = get_metadata(params.merge({ "key" => "/#{entry}" }), object) + end value["value"] = unpack(metadata, value["value"]) filtered_list[entry] = value else From 904b183aa330dca5a2e19f5adc340be696ae2f6b Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Tue, 2 May 2017 16:29:14 -0400 Subject: [PATCH 07/10] (SIMP-3129) atomic_create needs to create metadata * atomic_create needs to create metadata and pack/unpack. * add spec tests that were curiously missing SIMP-3129 #close --- lib/puppet_x/libkv/consul_provider.rb | 7 +- lib/puppet_x/libkv/libkv.rb | 6 ++ lib/puppet_x/libkv/mock_provider.rb | 27 +----- spec/functions/libkv_atomic_create_spec.rb | 107 +++++++++++++++++++++ spec/spec_helper.rb | 2 +- 5 files changed, 121 insertions(+), 28 deletions(-) create mode 100644 spec/functions/libkv_atomic_create_spec.rb diff --git a/lib/puppet_x/libkv/consul_provider.rb b/lib/puppet_x/libkv/consul_provider.rb index e751c6c..0e51f45 100644 --- a/lib/puppet_x/libkv/consul_provider.rb +++ b/lib/puppet_x/libkv/consul_provider.rb @@ -181,7 +181,10 @@ def atomic_get(params) throw Exception end end - + def atomic_create(params) + empty = empty_value() + atomic_put(params.merge({ 'previous' => empty})) + end def atomic_put(params) key = params['key'] value = params['value'] @@ -318,7 +321,7 @@ def exists(params) end end - def empty_value(params) + def empty_value(params = {}) { "ModifyIndex" => 0, "value" => nil diff --git a/lib/puppet_x/libkv/libkv.rb b/lib/puppet_x/libkv/libkv.rb index fc60773..9b8e446 100644 --- a/lib/puppet_x/libkv/libkv.rb +++ b/lib/puppet_x/libkv/libkv.rb @@ -164,6 +164,12 @@ def method_missing(symbol, url, auth, *args, &block) params["value"] = pack(meta, params["value"]) end retval = object.send(symbol, *nargs, &block); + when :atomic_create + if (serialize == true) + meta = get_metadata(params, object) + params["value"] = pack(meta, params["value"]) + end + retval = object.send(symbol, *nargs, &block); else retval = object.send(symbol, *nargs, &block); end diff --git a/lib/puppet_x/libkv/mock_provider.rb b/lib/puppet_x/libkv/mock_provider.rb index 9ba45f6..f22f787 100644 --- a/lib/puppet_x/libkv/mock_provider.rb +++ b/lib/puppet_x/libkv/mock_provider.rb @@ -95,31 +95,8 @@ def atomic_put(params) end end def atomic_create(params) - retval = {} - key = params['key']; - if (key == nil) - throw Exception - end - value = params['value']; - previous = self.empty_value(); - @mutex.synchronize do - previous_entry = atomic_get({'key' => key}) - if (previous_entry['sequence'] == previous['sequence']) - @sequence += 1; - retval["result"] = @root[key] = { - 'sequence' => @sequence, - 'key' => key, - 'value' => value.to_s, - } - else - throw Exception - end - end - if (params['debug'] == true) - retval - else - retval["result"] - end + empty = empty_value() + atomic_put(params.merge({ 'previous' => empty})) end def delete(params) diff --git a/spec/functions/libkv_atomic_create_spec.rb b/spec/functions/libkv_atomic_create_spec.rb new file mode 100644 index 0000000..0e9796e --- /dev/null +++ b/spec/functions/libkv_atomic_create_spec.rb @@ -0,0 +1,107 @@ +#!/usr/bin/env ruby -S rspec +# vim: set expandtab ts=2 sw=2: +require 'spec_helper' + +valid_characters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890/_-+' +invalid_characters = ";':,./<>?[]\{}|=`~!@\#$%^&*()\"" + +describe 'libkv::atomic_create' do + it 'should throw an exception with empty parameters' do + is_expected.to run.with_params({}).and_raise_error(Exception); + end + providers.each do |providerinfo| + provider = providerinfo["name"] + url = providerinfo["url"] + auth = providerinfo["auth"] + shared_params = { + "url" => url, + "serialize" => providerinfo["serialize"], + "mode" => providerinfo["mode"], + "auth" => auth, + } + + context "when provider = #{provider}" do + it 'should throw an exception when "key" is missing' do + is_expected.to run.with_params(shared_params).and_raise_error(Exception) + end + it 'should throw an exception when "value" is missing' do + is_expected.to run.with_params(shared_params.merge({"key" => "/test1"})).and_raise_error(Exception) + end + it 'should return a Boolean' do + params = { + 'key' => '/test1', + 'value' => 'value1', + }.merge(shared_params) + result = subject.execute(params) + expect(result.class).to eql(TrueClass) + end + datatype_testspec.each do |hash| + if (providerinfo["serialize"] == true) + klass = hash[:class] + else + klass = hash[:nonserial_class] + end + if (providerinfo["serialize"] == true) + expected_retval = hash[:value] + else + expected_retval = hash[:nonserial_retval] + end + it "should create an object of type #{klass} for /put/#{hash[:key]}" do + params = { + 'key' => "/put/" + hash[:key], + 'value' => hash[:value], + }.merge(shared_params) + subject.execute(params) + + params = { + 'key' => "/put/" + hash[:key], + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result.class.to_s).to eql(klass) + end + it "should create the value '#{hash[:value]}' for /put/#{hash[:key]}" do + params = { + 'key' => "/put/" + hash[:key], + 'value' => hash[:value], + }.merge(shared_params) + subject.execute(params) + + params = { + 'key' => "/put/" + hash[:key], + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result).to eql(expected_retval) + end + unless (hash[:class] == "String") + if (providerinfo["serialize"] == true) + it "should create the key '/put/#{hash[:key]}.meta' and contain a type = #{hash[:puppet_type]}" do + params = { + 'key' => "/put/" + hash[:key], + 'value' => hash[:value], + }.merge(shared_params) + subject.execute(params) + + params = { + 'key' => "/put/" + hash[:key] + ".meta", + }.merge(shared_params) + result = call_function("libkv::get", params) + expect(result).to_not eql(nil) + expect(result.class).to eql(String) + attempt_to_parse = nil + res = nil + begin + res = JSON.parse(result) + attempt_to_parse = true + rescue + attempt_to_parse = false + end + expect(attempt_to_parse).to eql(true) + expect(res.class).to eql(Hash) + expect(res["type"]).to eql(hash[:puppet_type].to_s) + end + end + end + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 7ae4b1d..c16eed2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -139,7 +139,7 @@ def set_hieradata(hieradata) set_hieradata(class_name.gsub(':','_')) end `curl -sX DELETE http://172.17.0.1:8500/v1/kv/puppet?recurse` - `curl -sX DELETE https://172.17.0.1:8504/v1/kv/puppet?recurse` + `curl -sX DELETE http://172.17.0.1:8504/v1/kv/puppet?recurse` end c.after(:each) do From d68ea45804da000d8b51ac4c9e8d80eb026acfb7 Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Tue, 2 May 2017 14:02:05 -0400 Subject: [PATCH 08/10] (SIMP-3130) metadata needs to default to 'String'. SIMP-3130 #close --- lib/puppet_x/libkv/libkv.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/puppet_x/libkv/libkv.rb b/lib/puppet_x/libkv/libkv.rb index 9b8e446..bee826f 100644 --- a/lib/puppet_x/libkv/libkv.rb +++ b/lib/puppet_x/libkv/libkv.rb @@ -248,6 +248,8 @@ def get_metadata(params, object) retval["type"] = puppetype(params["value"]) meta[0]["value"] = retval.to_json object.send(:put, *meta); + else + retval["type"] = "String" end end return retval From 159757ce1c0e7f13e1ba853621e47313bb6b7a53 Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Fri, 14 Apr 2017 12:36:13 -0400 Subject: [PATCH 09/10] (SIMP-2961) Add automatic cluster creation for consul. - Add a bootstrap manifest to mark a consul server for bootstrapping, and generates the certificate, keys, and tokens necessary for libkv - Add hiera data-in-modules for the consul configuration data. - Fix debugging output for libkv::put - Add 'consul-create-acl' command that can use a master token to generate a new acl from a hard coded template, for use by libkv --- bootstrap/consul.pp | 38 ++++++ data/common.yaml | 16 +++ files/consul/consul-create-acl | 26 ++++ hiera.yaml | 10 ++ lib/puppet_x/libkv/consul_provider.rb | 4 +- manifests/consul.pp | 183 ++++++++++++++++++++++++-- 6 files changed, 261 insertions(+), 16 deletions(-) create mode 100644 bootstrap/consul.pp create mode 100644 data/common.yaml create mode 100644 files/consul/consul-create-acl create mode 100644 hiera.yaml diff --git a/bootstrap/consul.pp b/bootstrap/consul.pp new file mode 100644 index 0000000..c89409a --- /dev/null +++ b/bootstrap/consul.pp @@ -0,0 +1,38 @@ +file { "/etc/simp": + ensure => directory, +} + file { "/etc/simp/bootstrap/": + ensure => directory, + } + file { "/etc/simp/bootstrap/consul": + ensure => directory, + } +exec { "/usr/bin/uuidgen >/etc/simp/bootstrap/consul/master_token": + creates => '/etc/simp/bootstrap/consul/master_token', + require => File["/etc/simp/bootstrap/consul"], +} -> +exec { "/opt/puppetlabs/bin/puppet cert generate server.dc1.consul": + creates => '/etc/puppetlabs/puppet/ssl/private_keys/server.dc1.consul.pem', +} -> +file { "/etc/simp/bootstrap/consul/server.dc1.consul.private.pem": +source => '/etc/puppetlabs/puppet/ssl/private_keys/server.dc1.consul.pem', +} -> +file { "/etc/simp/bootstrap/consul/server.dc1.consul.cert.pem": +source => '/etc/puppetlabs/puppet/ssl/certs/server.dc1.consul.pem', +} -> +file { "/etc/simp/bootstrap/consul/ca.pem": +source => '/etc/puppetlabs/puppet/ssl/ca/ca_crt.pem', +} -> +class { "libkv::consul": + dont_copy_files => true, + bootstrap => true, + server => true, +} -> +exec { "/usr/local/bin/consul keygen >/etc/simp/bootstrap/consul/key": + path => $::path, + creates => '/etc/simp/bootstrap/consul/key', +} -> +file { "/opt/puppetlabs/facter/facts.d/consul_bootstrap.sh": + mode => "a+x", + content => "#!/bin/sh\necho 'consul_bootstrap=true'", +} diff --git a/data/common.yaml b/data/common.yaml new file mode 100644 index 0000000..1b2d0ac --- /dev/null +++ b/data/common.yaml @@ -0,0 +1,16 @@ +lookup_options: + libkv::consul::config_hash: + merge: hash +libkv::consul::config_hash: + acl_datacenter: "dc1" + acl_default_policy: "deny" + addresses: + http: '127.0.0.1' + https: '0.0.0.0' + ports: + https: 8501 + http: 8500 + data_dir: '/opt/consul' + node_name: "%{::hostname}" + client_addr: '0.0.0.0' + ui_dir: /opt/consul/ui diff --git a/files/consul/consul-create-acl b/files/consul/consul-create-acl new file mode 100644 index 0000000..0b46c58 --- /dev/null +++ b/files/consul/consul-create-acl @@ -0,0 +1,26 @@ +#!/bin/sh +TOKEN=$(cat $1) +OUTPUTFILE=$2 +# Give consul some time to attempt a join, then realize it's bootstrapping +# a new cluster +sleep 10 +if [ "${TYPE}" = "" ] ; then + TYPE="libkv" +fi +case "${TYPE}" in + libkv) + POLICY='{ + "Name": "libkv-acl", + "Type": "client", + "Rules": "{\"key\":{\"puppet/\":{\"policy\":\"write\"}},\"operator\":\"read\"}" +}' + ;; + agent) + POLICY='{ + "Name": "agent-acl", + "Taype": "client", + "Rules": "{\"key\":{\"\":{\"policy\":\"write\"}, \"puppet/\":{\"policy\":\"deny\"}},\"operator\":\"read\"}" +}' + ;; +esac +curl --request PUT --data "${POLICY}" -q http://localhost:8500/v1/acl/create?token="${TOKEN}" | cut -d '"' -f 4 >"${OUTPUTFILE}" diff --git a/hiera.yaml b/hiera.yaml new file mode 100644 index 0000000..fd217b7 --- /dev/null +++ b/hiera.yaml @@ -0,0 +1,10 @@ +--- +version: 4 +datadir: data +hierarchy: + - name: "OSFamily + Release" + backend: "yaml" + path: "os/%{facts.osfamily}-%{facts.operatingsystemmajrelease}" + - name: "Common" + backend: "yaml" + path: "common" diff --git a/lib/puppet_x/libkv/consul_provider.rb b/lib/puppet_x/libkv/consul_provider.rb index 0e51f45..ff6c08d 100644 --- a/lib/puppet_x/libkv/consul_provider.rb +++ b/lib/puppet_x/libkv/consul_provider.rb @@ -134,11 +134,11 @@ def put(params) value = params['value'] if (key == nil) - throw Exception + raise "Put requires 'key' to be specified" end if (value == nil) - throw Exception + raise "Put requires 'value' to be specified" end response = consul_request(path: "/v1/kv" + @basepath + key, method: 'PUT', body: value) if (debug == true) diff --git a/manifests/consul.pp b/manifests/consul.pp index 59a7d7b..f617175 100644 --- a/manifests/consul.pp +++ b/manifests/consul.pp @@ -5,26 +5,181 @@ # class libkv::consul( $server = false, + $version = '0.8.0', + $use_puppet_pki = true, $bootstrap = false, - $key = undef, - $version = '0.7.4', - $client_addr = '0.0.0.0', + $dont_copy_files = false, + $serverhost = undef, + $advertise = undef, + $datacenter = undef, + $ca_file_name = undef, + $private_file_name = undef, + $cert_file_name = undef, + $config_hash = undef, ) { + if ($firewall) { + $ports = [ + '8300', + '8301', + '8302', + '8501', + ] + $ports.each |$port| { + iptables::listen::tcp_stateful { "libkv::consul - tcp - ${port}": + dports => $port, + } + iptables::listen::udp { "libkv::consul - udp - ${port}": + dports => $port, + } + } + } package { "unzip": } if ($bootstrap == true) { - $bootstrap_expect = 1 + $_bootstrap_hash = { "bootstrap_expect" => 1 } + } else { + $type = type($facts['consul_bootstrap']) + notify { "consul_bootstrap = ${type}": } + if ($facts["consul_bootstrap"] == "true") { + $_bootstrap_hash = { "bootstrap_expect" => 1 } + ## Create real token + file { "/usr/bin/consul-create-acl": + mode => "a+x", + source => "puppet:///modules/libkv/consul/consul-create-acl" + } -> + exec { "/usr/bin/consul-create-acl -t libkv /etc/simp/bootstrap/consul/master_token /etc/simp/bootstrap/consul/libkv_token": + creates => "/etc/simp/bootstrap/consul/libkv_token", + require => [ + Service['consul'], + File["/usr/bin/consul-create-acl"], + ], + } + exec { "/usr/bin/consul-create-acl -t agent_token /etc/simp/bootstrap/consul/master_token /etc/simp/bootstrap/consul/agent_token": + creates => "/etc/simp/bootstrap/consul/libkv_token", + require => [ + Service['consul'], + File["/usr/bin/consul-create-acl"], + ], + } + } else { + $_bootstrap_hash = {} + } + } + if ($datacenter == undef) { + $_datacenter = {} + } else { + $_datacenter = { "datacenter" => $datacenter } + } + if ($serverhost == undef) { + if ($::servername == undef) { + $_serverhost = $::fqdn + } else { + $_serverhost = $::servername + } + } else { + $_serverhost = $serverhost + } + if ($advertise == undef) { + $_advertise = $::ipaddress + } else { + $_advertise = $advertise + } + $keypath = '/etc/simp/bootstrap/consul/key' + $keydata = file($keypath, "/dev/null") + if ($keydata != undef) { + $_key_hash = { 'encrypt' => $keydata.chomp } + } else { + $_key_hash = {} + } + $master_token_path = '/etc/simp/bootstrap/consul/master_token' + $master_token = file($master_token_path, "/dev/null") + if ($master_token != undef) { + $_token_hash = { + "acl_master_token" => $master_token.chomp, + "acl_token" => $master_token.chomp, + } + } else { + $_token_hash = {} + } + if ($use_puppet_pki == true) { + if ($bootstrap == false) { + if (!defined(File['/etc/simp'])) { + file { "/etc/simp": + ensure => directory, + } + } + } + file { "/etc/simp/consul": + ensure => directory, + } + if ($server == true) { + $_cert_file_name = '/etc/simp/bootstrap/consul/server.dc1.consul.cert.pem' + $_private_file_name = '/etc/simp/bootstrap/consul/server.dc1.consul.private.pem' + $_ca_file_name = '/etc/simp/bootstrap/consul/ca.pem' + if ($dont_copy_files == false) { + file { "/etc/simp/bootstrap/": + ensure => directory, + } + file { "/etc/simp/bootstrap/consul": + ensure => directory, + } + file { $_cert_file_name: + content => file($_cert_file_name) + } + file { $_private_file_name: + content => file($_private_file_name) + } + file { $_ca_file_name: + content => file($_ca_file_name) + } + file { '/etc/simp/consul/cert.pem': + content => file($_cert_file_name) + } + file { '/etc/simp/consul/key.pem': + content => file($_private_file_name) + } + file { '/etc/simp/consul/ca.pem': + content => file($_ca_file_name) + } + } + } else { + $_cert_file_name_source = "/etc/puppetlabs/puppet/ssl/certs/${::clientcert}.pem" + $_ca_file_name_source = '/etc/puppetlabs/puppet/ssl/certs/ca.pem' + $_private_file_name_source = "/etc/puppetlabs/puppet/ssl/private_keys/${::clientcert}.pem" + file { '/etc/simp/consul/cert.pem': + source => $_cert_file_name_source + } + file { '/etc/simp/consul/ca.pem': + source => $_ca_file_name_source + } + file { '/etc/simp/consul/key.pem': + source => $_private_file_name_source + } + } + if ($bootstrap == false) { + $_cert_hash = { + "cert_file" => '/etc/simp/consul/cert.pem', + "ca_file" => '/etc/simp/consul/ca.pem', + "key_file" => '/etc/simp/consul/key.pem', + "verify_outgoing" => true, + "verify_incoming" => true, + "verify_server_hostname" => true, + } + } else { + $_cert_hash = {} + } + } + # Attempt to store bootstrap info into consul directly via libkv. + # Use softfail to get around issues if the service isn't up + $hash = lookup('consul::config_hash', { "default_value" => {} }) + $class_hash = { + 'server' => $server, + 'node_name' => $::hostname, + 'retry_join' => [ $_serverhost ], + 'advertise_addr' => $_advertise, } + $merged_hash = $hash + $class_hash + $_datacenter + $config_hash + $_key_hash + $_token_hash + $_bootstrap_hash + $_cert_hash class { '::consul': - config_hash => { - 'data_dir' => '/opt/consul', - 'bootstrap_expect' => $bootstrap_expect, - 'server' => $server, - 'node_name' => $::hostname, - 'retry_join' => [ $serverip ], - 'advertise_addr' => $::ipaddress, - 'client_addr' => $client_addr, - 'ui_dir' => '/opt/consul/ui', - }, + config_hash => $merged_hash, version => $version, } } From 69ddddd1cd60e34ceb55b930d05e144ec7f0c31d Mon Sep 17 00:00:00 2001 From: Dylan Cochran Date: Mon, 10 Jul 2017 16:46:50 -0400 Subject: [PATCH 10/10] Release 0.3.0 --- CHANGELOG | 11 +++++++++++ metadata.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 59bb98e..98d4243 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +* Sat Jul 10 2017 Dylan Cochran - 0.3.0 +- (SIMP-2961) Add automatic cluster creation for consul. +- (SIMP-3130) metadata needs to default to 'String'. +- (SIMP-3129) atomic_create needs to create metadata +- (SIMP-3127) libkv can't list / since metadata update +- (SIMP-3122) Move the libkv wrapper outside the loader +- (SIMP-3125) Move key regex match into libkv wrapper +- (SIMP-3110) Use .meta to convert a value to the correct type +- (SIMP-3060) Fix travisci tests +- (SIMP-3109) Create a .meta key to store type + * Sat Apr 29 2017 Dylan Cochran - 0.2.0 - (SIMP-2978) Fix readme generation - (SIMP-3019) Add ssl/tls support to consul backend diff --git a/metadata.json b/metadata.json index 71b30b7..64072f4 100644 --- a/metadata.json +++ b/metadata.json @@ -1,6 +1,6 @@ { "name": "simp-libkv", - "version": "0.2.0", + "version": "0.3.0", "author": "simp", "summary": "", "license": "Apache-2.0",