Merge branch 'release/0.4.2'

CHANGELOG

@@ -1,3 +1,8 @@
+* Tue Jul 18 2017 Dylan Cochran <[email protected]> - 0.4.2
+- (SIMP-3001) Prevent '.' and '..' from being used in keys
+- (SIMP-3446) Add parameters to reconfigure http and https listen
+- (SIMP-3429) libkv::list isn't always removing keys
+
 * Tue Jul 18 2017 Dylan Cochran <[email protected]> - 0.4.1
 - Always copy over consul-acl, and update metadata
 

README.md

@@ -38,6 +38,27 @@
 
 libkv is an abstract library that allows puppet to access a distributed key value store, like consul or etcd. This library implements all the basic key/value primitives, get, put, list, delete. It also exposes any 'check and set' functionality the underlying store supports. This allows building of safe atomic operations, to build complex distributed systems. This library supports loading 'provider' modules that exist in other modules, and provides a first class api.
 
+For example, you can use the following to store hostnames, and then read all the known hostnames from consul and generate a hosts file:
+
+```puppet
+libkv::put("/hosts/${::clientcert}", $::ipaddress)
+
+$hosts = libkv::list("/hosts")
+$hosts.each |$host, $ip | {
+  host { $host:
+    ip => $ip,
+  }
+}
+```
+
+Each key specified *must* contain only the following characters:
+* a-z
+* A-Z
+* 0-9
+* The following special characters: ._:-/
+
+Additionally, '/./' and '/../' are disallowed in all providers as key components. The key name also *must* begin with '/'
+
 When any libkv function is called, it will first call `lookup()` and attempt to find a value for libkv::url from hiera. This url specifies the provider name, the host, the port, and the path in the underlying store. For example:
 ```yaml
 libkv::url: 'consul://127.0.0.1:8500/puppet'
@@ -47,7 +68,7 @@ libkv::url: 'etcd://127.0.0.1:2380/puppet/%{environment}/'
 libkv::url: 'consul://127.0.0.1:8500/puppet/%{trusted.extensions.pp_department}/%{environment}'
 ```
 
-Additionally libkv uses lookup to store authentication information. This information can range from ssl client certificates, access tokens, or usernames and passwords. The options are provider specific, so it's best
+Additionally libkv uses lookup to store authentication information. This information can range from ssl client certificates, access tokens, or usernames and passwords. It is exposed as a hash named libkv::auth, and will be merged by default. The keys in the auth token are passed as is to the provider, and can vary between providers. Please read the documentation on configuring 'libkv::auth' for each provider
 
 libkv currently supports the following providers:
 

data/common.yaml

@@ -5,9 +5,6 @@ libkv::consul::puppet_cert_path: '/etc/puppetlabs/puppet/ssl'
 libkv::consul::config_hash:
   acl_datacenter: "dc1"
   acl_default_policy: "deny"
-  addresses:
-    http: '127.0.0.1'
-    https: '0.0.0.0'
   ports:
     https: 8501
     http: 8500

lib/puppet_x/libkv/consul_provider.rb

@@ -283,7 +283,7 @@ def atomic_list(params)
     if (last_char != "/")
       key = key + "/"
     end
-    reg = Regexp.new("^" + @basepath.gsub(/\//, "") + key)
+    reg = Regexp.new("^" + @basepath.gsub(/^\//, "") + key)
     unless (value == nil)
       value.each do |entry|
         nkey = entry["Key"].gsub(reg,"")

lib/puppet_x/libkv/libkv.rb

@@ -110,6 +110,10 @@ def sanitize_input(symbol, params)
           unless (regex =~ params[name])
             raise error_msg
           end
+          dot_regex = /\/\.\.*\//
+          if (dot_regex =~ params[name])
+            raise "the value of '#{name}': '#{params[name]}' contains ./ or ../, which is invalid in a key name"
+          end
 
         else
           unless (params[name].class.to_s == definition)
@@ -130,7 +134,7 @@ def method_missing(symbol, url, auth, *args, &block)
   # if (params['dd'] == true)
   #   binding.pry
   # end
- 
+
   # Pre-provider mangling
   unless (params.key?("serialize"))
     params["serialize"] = true
@@ -246,11 +250,11 @@ def delete_metadata(params, object)
   meta[0] = params.dup
   meta[0]["key"] = "#{params['key']}.meta"
   begin
-  metadata = object.send(:delete, *meta)
+    metadata = object.send(:delete, *meta)
   rescue
   end
 end
-  
+
 def get_metadata(params, object)
   meta = []
   meta[0] = params.dup

manifests/consul.pp

@@ -10,6 +10,8 @@
   $bootstrap = false,
   $dont_copy_files = false,
   $serverhost = undef,
+  $http_listen = '127.0.0.1',
+  $https_listen = '0.0.0.0',
   $advertise = undef,
   $datacenter = undef,
   $puppet_cert_path,
@@ -203,10 +205,14 @@
   # Use softfail to get around issues if the service isn't up
   $hash = lookup('consul::config_hash', { "default_value" => {} })
   $class_hash =     {
-    'server'           => $server,
-    'node_name'        => $::hostname,
-    'retry_join'       => [ $_serverhost ],
-    'advertise_addr'   => $_advertise,
+    'server'         => $server,
+    'node_name'      => $::hostname,
+    'retry_join'     => [ $_serverhost ],
+    'advertise_addr' => $_advertise,
+    'addresses'      => {
+      'http'         => $http_listen,
+      'https'        => $https_listen,
+    },
   }
   $merged_hash = $hash + $class_hash + $_datacenter + $config_hash + $_key_hash + $_token_hash + $_bootstrap_hash + $_cert_hash
   class { '::consul':

metadata.json

@@ -1,6 +1,6 @@
 {
   "name": "simp-libkv",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "author": "simp",
   "summary": "",
   "license": "Apache-2.0",

scripts/readme.erb

@@ -20,6 +20,27 @@
 
 libkv is an abstract library that allows puppet to access a distributed key value store, like consul or etcd. This library implements all the basic key/value primitives, get, put, list, delete. It also exposes any 'check and set' functionality the underlying store supports. This allows building of safe atomic operations, to build complex distributed systems. This library supports loading 'provider' modules that exist in other modules, and provides a first class api.
 
+For example, you can use the following to store hostnames, and then read all the known hostnames from consul and generate a hosts file:
+
+```puppet
+libkv::put("/hosts/${::clientcert}", $::ipaddress)
+
+$hosts = libkv::list("/hosts")
+$hosts.each |$host, $ip | {
+  host { $host:
+    ip => $ip,
+  }
+}
+```
+
+Each key specified *must* contain only the following characters:
+* a-z
+* A-Z
+* 0-9
+* The following special characters: ._:-/
+
+Additionally, '/./' and '/../' are disallowed in all providers as key components. The key name also *must* begin with '/'
+
 When any libkv function is called, it will first call `lookup()` and attempt to find a value for libkv::url from hiera. This url specifies the provider name, the host, the port, and the path in the underlying store. For example:
 ```yaml
 libkv::url: 'consul://127.0.0.1:8500/puppet'
@@ -29,7 +50,7 @@ libkv::url: 'etcd://127.0.0.1:2380/puppet/%{environment}/'
 libkv::url: 'consul://127.0.0.1:8500/puppet/%{trusted.extensions.pp_department}/%{environment}'
 ```
 
-Additionally libkv uses lookup to store authentication information. This information can range from ssl client certificates, access tokens, or usernames and passwords. The options are provider specific, so it's best
+Additionally libkv uses lookup to store authentication information. This information can range from ssl client certificates, access tokens, or usernames and passwords. It is exposed as a hash named libkv::auth, and will be merged by default. The keys in the auth token are passed as is to the provider, and can vary between providers. Please read the documentation on configuring 'libkv::auth' for each provider
 
 libkv currently supports the following providers:
 

spec/functions/libkv_get_spec.rb

@@ -30,7 +30,23 @@
         result = subject.execute(params)
         expect(result).to eql(nil)
       end
+      it "should throw an exception if the key contains '..' as a component" do
+          params = {
+             'key' => "/get/directory1/test",
+             'value' => "directory1",
+          }.merge(shared_params)
+          call_function("libkv::put", params)
+          params = {
+             'key' => "/get/directory2/test",
+             'value' => "directory2",
+          }.merge(shared_params)
+          call_function("libkv::put", params)
 
+          params = {
+             'key' => "/get/directory1/../directory2/test",
+          }.merge(shared_params)
+        is_expected.to run.with_params(params).and_raise_error(Exception);
+      end
       datatype_testspec.each do |hash|
        if (providerinfo["serialize"] == true)
          klass = hash[:class]

spec/spec_helper.rb

@@ -255,6 +255,13 @@ def providers()
 	  # "softfail" => false,
 	  # "should_error" => false,
   # },
+  {
+	  "name" => "consul with ssl and without auth and with daemon and multiple path segments",
+	  "url" => "consul+ssl+noverify://172.17.0.1:10501/puppet/default/",
+          "serialize" => true,
+	  "softfail" => false,
+	  "should_error" => false,
+  },
   {
 	  "name" => "consul with ssl and without auth and with daemon",
 	  "url" => "consul+ssl+noverify://172.17.0.1:10501/puppet",