(SIMP-2961) Add automatic cluster creation for consul.

- Add a bootstrap manifest to mark a consul server for bootstrapping,
and generates the certificate, keys, and tokens necessary for libkv
- Add hiera data-in-modules for the consul configuration data.
- Fix debugging output for libkv::put
- Add 'consul-create-acl' command that can use a master token to
generate a new acl from a hard coded template, for use by libkv

bootstrap/consul.pp

@@ -0,0 +1,38 @@
+file { "/etc/simp":
+	ensure => directory,
+}
+      file { "/etc/simp/bootstrap/":
+	ensure => directory,
+      }
+      file { "/etc/simp/bootstrap/consul":
+	ensure => directory,
+      }
+exec { "/usr/bin/uuidgen >/etc/simp/bootstrap/consul/master_token":
+	creates => '/etc/simp/bootstrap/consul/master_token',
+        require => File["/etc/simp/bootstrap/consul"],
+} ->
+exec { "/opt/puppetlabs/bin/puppet cert generate server.dc1.consul":
+	creates => '/etc/puppetlabs/puppet/ssl/private_keys/server.dc1.consul.pem',
+} ->
+file { "/etc/simp/bootstrap/consul/server.dc1.consul.private.pem":
+source => '/etc/puppetlabs/puppet/ssl/private_keys/server.dc1.consul.pem',
+} ->
+file { "/etc/simp/bootstrap/consul/server.dc1.consul.cert.pem":
+source => '/etc/puppetlabs/puppet/ssl/certs/server.dc1.consul.pem',
+} ->
+file { "/etc/simp/bootstrap/consul/ca.pem":
+source => '/etc/puppetlabs/puppet/ssl/ca/ca_crt.pem',
+} -> 
+class { "libkv::consul":
+	dont_copy_files => true,
+	bootstrap       => true,
+	server          => true,
+} ->
+exec { "/usr/local/bin/consul keygen >/etc/simp/bootstrap/consul/key":
+  path => $::path,
+  creates => '/etc/simp/bootstrap/consul/key',
+} ->
+file { "/opt/puppetlabs/facter/facts.d/consul_bootstrap.sh":
+	mode    => "a+x",
+	content => "#!/bin/sh\necho 'consul_bootstrap=true'",
+}

data/common.yaml

@@ -0,0 +1,16 @@
+lookup_options:
+  libkv::consul::config_hash:
+    merge: hash
+libkv::consul::config_hash:
+  acl_datacenter: "dc1"
+  acl_default_policy: "deny"
+  addresses:
+    http: '127.0.0.1'
+    https: '0.0.0.0'
+  ports:
+    https: 8501
+    http: 8500
+  data_dir: '/opt/consul'
+  node_name: "%{::hostname}"
+  client_addr: '0.0.0.0'
+  ui_dir: /opt/consul/ui

files/consul/consul-create-acl

@@ -0,0 +1,26 @@
+#!/bin/sh
+TOKEN=$(cat $1)
+OUTPUTFILE=$2
+# Give consul some time to attempt a join, then realize it's bootstrapping
+# a new cluster
+sleep 10
+if [ "${TYPE}" = "" ] ; then
+  TYPE="libkv"
+fi
+case "${TYPE}" in
+	libkv)
+		POLICY='{
+  "Name": "libkv-acl",
+  "Type": "client",
+  "Rules": "{\"key\":{\"puppet/\":{\"policy\":\"write\"}},\"operator\":\"read\"}"
+}'
+		;;
+	agent)
+		POLICY='{
+  "Name": "agent-acl",
+  "Taype": "client",
+  "Rules": "{\"key\":{\"\":{\"policy\":\"write\"}, \"puppet/\":{\"policy\":\"deny\"}},\"operator\":\"read\"}"
+}'
+		;;
+esac
+curl --request PUT --data "${POLICY}" -q http://localhost:8500/v1/acl/create?token="${TOKEN}" | cut -d '"' -f 4 >"${OUTPUTFILE}"

hiera.yaml

@@ -0,0 +1,10 @@
+---
+version: 4
+datadir: data
+hierarchy:
+  - name: "OSFamily + Release"
+    backend: "yaml"
+    path: "os/%{facts.osfamily}-%{facts.operatingsystemmajrelease}"
+  - name: "Common"
+    backend: "yaml"
+    path: "common"

lib/puppet_x/libkv/consul_provider.rb

@@ -134,11 +134,11 @@ def put(params)
     value = params['value']
 
     if (key == nil)
-      throw Exception
+      raise "Put requires 'key' to be specified"
     end
 
     if (value == nil)
-      throw Exception
+      raise "Put requires 'value' to be specified"
     end
     response = consul_request(path: "/v1/kv" + @basepath + key, method: 'PUT', body: value)
     if (debug == true)

manifests/consul.pp

@@ -5,26 +5,181 @@
 #
 class libkv::consul(
   $server = false,
+  $version = '0.8.0',
+  $use_puppet_pki = true,
   $bootstrap = false,
-  $key = undef,
-  $version = '0.7.4',
-  $client_addr = '0.0.0.0',
+  $dont_copy_files = false,
+  $serverhost = undef,
+  $advertise = undef,
+  $datacenter = undef,
+  $ca_file_name = undef,
+  $private_file_name = undef,
+  $cert_file_name = undef,
+  $config_hash = undef,
 ) {
+  if ($firewall) {
+    $ports = [
+      '8300',
+      '8301',
+      '8302',
+      '8501',
+    ]
+    $ports.each |$port| {
+      iptables::listen::tcp_stateful { "libkv::consul - tcp - ${port}":
+        dports => $port,
+      }
+      iptables::listen::udp { "libkv::consul - udp - ${port}":
+        dports => $port,
+      }
+    }
+  }
   package { "unzip": }
   if ($bootstrap == true) {
-    $bootstrap_expect = 1
+    $_bootstrap_hash = { "bootstrap_expect" => 1 }
+  } else {
+    $type = type($facts['consul_bootstrap'])
+    notify { "consul_bootstrap = ${type}": }
+    if ($facts["consul_bootstrap"] == "true") {
+      $_bootstrap_hash = { "bootstrap_expect" => 1 }
+      ## Create real token
+      file { "/usr/bin/consul-create-acl":
+        mode   => "a+x",
+        source => "puppet:///modules/libkv/consul/consul-create-acl"
+      } ->
+      exec { "/usr/bin/consul-create-acl -t libkv /etc/simp/bootstrap/consul/master_token /etc/simp/bootstrap/consul/libkv_token":
+        creates => "/etc/simp/bootstrap/consul/libkv_token",
+        require => [
+		Service['consul'],
+		File["/usr/bin/consul-create-acl"],
+	],
+      }
+      exec { "/usr/bin/consul-create-acl -t agent_token /etc/simp/bootstrap/consul/master_token /etc/simp/bootstrap/consul/agent_token":
+        creates => "/etc/simp/bootstrap/consul/libkv_token",
+        require => [
+		Service['consul'],
+		File["/usr/bin/consul-create-acl"],
+	],
+      }
+    } else {
+      $_bootstrap_hash = {}
+    }
+  }
+  if ($datacenter == undef) {
+    $_datacenter = {}
+  } else {
+    $_datacenter = { "datacenter" => $datacenter }
+  }
+  if ($serverhost == undef) {
+    if ($::servername == undef) {
+      $_serverhost = $::fqdn
+    } else {
+      $_serverhost = $::servername
+    }
+  } else {
+    $_serverhost = $serverhost
+  }
+  if ($advertise == undef) {
+    $_advertise = $::ipaddress
+  } else {
+    $_advertise = $advertise
+  }
+  $keypath = '/etc/simp/bootstrap/consul/key'
+  $keydata = file($keypath, "/dev/null")
+  if ($keydata != undef) {
+    $_key_hash = { 'encrypt' => $keydata.chomp }
+  } else {
+    $_key_hash = {}
+  }
+  $master_token_path = '/etc/simp/bootstrap/consul/master_token'
+  $master_token = file($master_token_path, "/dev/null")
+  if ($master_token != undef) {
+    $_token_hash = { 
+    "acl_master_token" => $master_token.chomp,
+    "acl_token"        => $master_token.chomp,
+    }
+  } else {
+    $_token_hash = {}
+  }
+  if ($use_puppet_pki == true) {
+    if ($bootstrap == false) {
+      if (!defined(File['/etc/simp'])) {
+      file { "/etc/simp":
+        ensure => directory,
+      }
+      }
+    }
+    file { "/etc/simp/consul":
+      ensure => directory,
+    }
+    if ($server == true) {
+      $_cert_file_name = '/etc/simp/bootstrap/consul/server.dc1.consul.cert.pem'
+      $_private_file_name = '/etc/simp/bootstrap/consul/server.dc1.consul.private.pem'
+      $_ca_file_name = '/etc/simp/bootstrap/consul/ca.pem'
+      if ($dont_copy_files == false) {
+        file { "/etc/simp/bootstrap/":
+          ensure => directory,
+        }
+        file { "/etc/simp/bootstrap/consul":
+          ensure => directory,
+        }
+        file { $_cert_file_name:
+          content => file($_cert_file_name)
+        }
+        file { $_private_file_name:
+          content => file($_private_file_name)
+        }
+        file { $_ca_file_name:
+          content => file($_ca_file_name)
+        }
+        file { '/etc/simp/consul/cert.pem':
+          content => file($_cert_file_name)
+        }
+        file { '/etc/simp/consul/key.pem':
+          content => file($_private_file_name)
+        }
+        file { '/etc/simp/consul/ca.pem':
+          content => file($_ca_file_name)
+        }
+      }
+    } else {
+      $_cert_file_name_source = "/etc/puppetlabs/puppet/ssl/certs/${::clientcert}.pem"
+      $_ca_file_name_source = '/etc/puppetlabs/puppet/ssl/certs/ca.pem'
+      $_private_file_name_source = "/etc/puppetlabs/puppet/ssl/private_keys/${::clientcert}.pem"
+      file { '/etc/simp/consul/cert.pem':
+        source => $_cert_file_name_source
+      }
+      file { '/etc/simp/consul/ca.pem':
+        source => $_ca_file_name_source
+      }
+      file { '/etc/simp/consul/key.pem':
+        source => $_private_file_name_source
+      }
+    }
+    if ($bootstrap == false) {
+      $_cert_hash = {
+        "cert_file"              => '/etc/simp/consul/cert.pem',
+        "ca_file"                => '/etc/simp/consul/ca.pem',
+        "key_file"               => '/etc/simp/consul/key.pem',
+        "verify_outgoing"        => true,
+        "verify_incoming"        => true,
+        "verify_server_hostname" => true,
+      }
+    } else {
+      $_cert_hash = {}
+    }
+  }
+  # Attempt to store bootstrap info into consul directly via libkv.
+  # Use softfail to get around issues if the service isn't up
+  $hash = lookup('consul::config_hash', { "default_value" => {} })
+  $class_hash =     {
+    'server'           => $server,
+    'node_name'        => $::hostname,
+    'retry_join'       => [ $_serverhost ],
+    'advertise_addr'   => $_advertise,
   }
+  $merged_hash = $hash + $class_hash + $_datacenter + $config_hash + $_key_hash + $_token_hash + $_bootstrap_hash + $_cert_hash
   class { '::consul':
-    config_hash          => {
-      'data_dir'         => '/opt/consul',
-      'bootstrap_expect' => $bootstrap_expect,
-      'server'           => $server,
-      'node_name'        => $::hostname,
-      'retry_join'       => [ $serverip ],
-      'advertise_addr'   => $::ipaddress,
-      'client_addr'      => $client_addr,
-      'ui_dir'           => '/opt/consul/ui',
-    },
+    config_hash          => $merged_hash,
     version => $version,
   }
 }