Merge pull request #33 from cloudamatic/directory_services

Directory Services merge for v1.1

Berksfile

@@ -16,6 +16,8 @@ cookbook 'application_ruby', '~> 2.1.4'
 cookbook 'apt', '~> 2.7.0'
 cookbook 'aws', '~> 2.7.2'
 cookbook 'awscli', path: "#{cookbookPath}/awscli"
+cookbook 'bind', '~> 1.1.2'
+cookbook 'bind9-ng', '~> 0.1.0'
 cookbook 'bluepill', '~> 2.3.1'
 cookbook 'build-essential', '~> 2.2.3'
 cookbook 'mu-master', path: "#{cookbookPath}/mu-master"
@@ -39,13 +41,14 @@ cookbook 'gunicorn', '~> 1.1.2'
 cookbook 'iis', '~> 4.1.1'
 cookbook 'iptables', '~> 1.0.0'
 cookbook 'logrotate', '~> 1.9.2'
-cookbook 'java', '~> 1.35.0'
+cookbook 'java', '~> 1.36.0'
 cookbook 'jenkins', '~> 2.3.1'
 cookbook 'memcached', '~> 1.7.2'
 cookbook 'mongodb', '~> 0.16.2'
 cookbook 'mysql', '~> 6.1.0'
-cookbook 'nagios', '~> 7.1.8'
+cookbook 'nagios', path: "#{cookbookPath}/nagios"
 cookbook 'nginx', '~> 2.7.6'
+cookbook 'nrpe', '~> 1.5.2'
 cookbook 'nginx_simplecgi', '~> 0.1.2'
 cookbook 'mu-activedirectory', path: "#{cookbookPath}/mu-activedirectory"
 cookbook 'nginx-passenger', path: "#{cookbookPath}/nginx-passenger"
@@ -72,5 +75,5 @@ cookbook 'unicorn', '~> 1.3.0'
 cookbook 'windows', '~> 1.38.1'
 cookbook 'xfs', '~> 1.1.0'
 cookbook 'xml', '~> 1.2.4'
-cookbook 'yum', '~> 3.6.3'
-cookbook 'yum-epel', '~> 0.6.2'
+cookbook 'yum', '~> 3.8.2'
+cookbook 'yum-epel', '~> 0.6.3'

Berksfile.lock

@@ -7,6 +7,8 @@ DEPENDENCIES
   aws (~> 2.7.2)
   awscli
     path: cookbooks/awscli
+  bind (~> 1.1.2)
+  bind9-ng (~> 0.1.0)
   bluepill (~> 2.3.1)
   build-essential (~> 2.2.3)
   chef-splunk
@@ -25,7 +27,7 @@ DEPENDENCIES
   gunicorn (~> 1.1.2)
   iis (~> 4.1.1)
   iptables (~> 1.0.0)
-  java (~> 1.35.0)
+  java (~> 1.36.0)
   jenkins (~> 2.3.1)
   logrotate (~> 1.9.2)
   memcached (~> 1.7.2)
@@ -45,12 +47,14 @@ DEPENDENCIES
   mu-utility
     path: cookbooks/mu-utility
   mysql (~> 6.1.0)
-  nagios (~> 7.1.8)
+  nagios
+    path: cookbooks/nagios
   nginx (~> 2.7.6)
   nginx-passenger
     path: cookbooks/nginx-passenger
   nginx_simplecgi (~> 0.1.2)
   nodejs (~> 2.4.0)
+  nrpe (~> 1.5.2)
   ohai (~> 2.0.1)
   openssl (~> 4.0.0)
   oracle-instantclient (~> 1.1.0)
@@ -79,8 +83,8 @@ DEPENDENCIES
   windows (~> 1.38.1)
   xfs (~> 1.1.0)
   xml (~> 1.2.4)
-  yum (~> 3.6.3)
-  yum-epel (~> 0.6.2)
+  yum (~> 3.8.2)
+  yum-epel (~> 0.6.3)
 
 GRAPH
   7-zip (1.0.2)
@@ -105,24 +109,24 @@ GRAPH
     windows (>= 0.0.0)
   aws (2.7.2)
   awscli (0.2.1)
+  bind (1.1.2)
+  bind9-ng (0.1.0)
   bluepill (2.3.1)
     rsyslog (>= 0.0.0)
-  build-essential (2.2.3)
+  build-essential (2.2.4)
   chef-splunk (1.3.0)
     chef-vault (>= 1.0.4)
   chef-sugar (3.1.1)
-  chef-vault (1.3.0)
+  chef-vault (1.3.2)
   chef_handler (1.2.0)
   cpan (0.0.34)
-  database (4.0.8)
+  database (4.0.9)
     postgresql (>= 1.0.0)
-  demo (0.2.3)
+  demo (0.3.0)
     application (>= 0.0.0)
     application_python (>= 0.0.0)
     application_ruby (>= 0.0.0)
-    chef-vault (>= 0.0.0)
     git (>= 0.0.0)
-    java (>= 0.0.0)
     mysql (>= 0.0.0)
     nginx (>= 0.0.0)
     nodejs (>= 0.0.0)
@@ -131,19 +135,20 @@ GRAPH
   dmg (2.2.2)
   ec2-s3-api-tools (0.1.0)
   freebsd (0.1.10)
-  git (4.3.3)
+  git (4.3.4)
     build-essential (>= 0.0.0)
     dmg (>= 0.0.0)
     windows (>= 0.0.0)
     yum-epel (>= 0.0.0)
   gunicorn (1.1.6)
     python (>= 0.0.0)
-  homebrew (1.13.0)
+  homebrew (2.0.2)
     build-essential (>= 2.1.2)
-  iis (4.1.1)
+  iis (4.1.5)
     windows (>= 1.34.6)
   iptables (1.0.0)
-  java (1.35.0)
+  java (1.36.0)
+    apt (>= 0.0.0)
   jenkins (2.3.1)
     apt (~> 2.0)
     runit (~> 1.5)
@@ -168,13 +173,18 @@ GRAPH
     chef-vault (>= 0.0.0)
     java (>= 0.0.0)
     jenkins (>= 0.0.0)
+    mu-master (>= 0.0.0)
     mu-utility (>= 0.0.0)
   mu-master (0.8.1)
+    bind (>= 0.0.0)
+    bind9-ng (>= 0.0.0)
     jenkins (>= 0.0.0)
+    mu-activedirectory (>= 0.0.0)
     mu-jenkins (>= 0.0.0)
     mu-tools (>= 0.0.0)
     mu-utility (>= 0.0.0)
     nagios (>= 0.0.0)
+    nrpe (>= 0.0.0)
     postfix (>= 0.0.0)
     s3fs (>= 0.0.0)
   mu-openvpn (0.1.0)
@@ -207,11 +217,11 @@ GRAPH
     windows (>= 0.0.0)
     yum (>= 0.0.0)
     yum-epel (>= 0.0.0)
-  mysql (6.1.0)
+  mysql (6.1.2)
     smf (>= 0.0.0)
     yum-mysql-community (>= 0.0.0)
-  nagios (7.1.8)
-    apache2 (>= 2.0)
+  nagios (7.2.5)
+    apache2 (>= 2.0.0)
     build-essential (>= 0.0.0)
     nginx (>= 0.0.0)
     nginx_simplecgi (>= 0.0.0)
@@ -232,7 +242,7 @@ GRAPH
     nginx (>= 0.0.0)
     perl (>= 0.0.0)
     runit (>= 0.0.0)
-  nodejs (2.4.0)
+  nodejs (2.4.2)
     apt (>= 0.0.0)
     ark (>= 0.0.0)
     build-essential (>= 0.0.0)
@@ -241,14 +251,14 @@ GRAPH
   nrpe (1.5.2)
     build-essential (>= 0.0.0)
     yum-epel (>= 0.0.0)
-  ohai (2.0.1)
+  ohai (2.0.4)
   openssl (4.0.0)
     chef-sugar (>= 0.0.0)
   oracle-instantclient (1.1.0)
     build-essential (>= 0.0.0)
     cpan (>= 0.0.0)
     php (>= 0.0.0)
-  packagecloud (0.0.19)
+  packagecloud (0.1.1)
   pacman (1.1.1)
   passenger_apache2 (2.1.2)
     apache2 (>= 1.0.4)
@@ -262,24 +272,24 @@ GRAPH
     xml (>= 0.0.0)
     yum-epel (>= 0.0.0)
   postfix (3.6.2)
-  postgresql (3.4.20)
+  postgresql (3.4.24)
     apt (>= 1.9.0)
     build-essential (>= 0.0.0)
-    openssl (~> 4.0.0)
+    openssl (~> 4.0)
   python (1.4.7)
     build-essential (>= 0.0.0)
     yum-epel (>= 0.0.0)
   rbac (1.0.3)
   rsyslog (2.1.0)
   ruby-cookbook (0.9.2)
   ruby_build (0.8.0)
-  runit (1.7.2)
+  runit (1.7.4)
     packagecloud (>= 0.0.0)
   rvm (0.1.0)
   s3fs (0.2.0)
     build-essential (>= 0.0.0)
     mu-utility (>= 0.0.0)
-  simple_iptables (0.7.2)
+  simple_iptables (0.7.3)
   smf (2.2.7)
     rbac (>= 1.0.1)
   supervisor (0.4.12)
@@ -289,15 +299,15 @@ GRAPH
     openssl (>= 0.0.0)
     yum-epel (>= 0.0.0)
   unicorn (1.3.0)
-  windows (1.38.1)
+  windows (1.38.4)
     chef_handler (>= 0.0.0)
-  xfs (1.1.0)
+  xfs (1.1.1)
   xml (1.2.13)
     build-essential (>= 0.0.0)
     chef-sugar (>= 0.0.0)
-  yum (3.6.3)
-  yum-epel (0.6.2)
+  yum (3.8.2)
+  yum-epel (0.6.5)
     yum (~> 3.2)
-  yum-mysql-community (0.1.17)
-    yum (>= 3.0)
-  zap (0.8.6)
+  yum-mysql-community (0.1.21)
+    yum (>= 3.2)
+  zap (0.11.2)

bin/mu-aws-setup

@@ -22,7 +22,7 @@
 require 'etc'
 require 'securerandom'
 
-require File.expand_path(File.dirname(__FILE__))+"/mu-load-murc.rb"
+require File.expand_path(File.dirname(__FILE__))+"/mu-load-config.rb"
 
 require 'rubygems'
 require 'bundler/setup'
@@ -55,7 +55,7 @@ preferred_ip = MU.mu_public_ip
 # Create a security group, or manipulate an existing one, so that we have all
 # of the appropriate network holes.
 if $opts[:sg]
-  open_ports = [80, 443, 2260, 8443, 9443]
+  open_ports = [80, 443, 2260, 7443, 8443, 9443]
 
   # This doesn't make sense. we can have multiple security groups in our account with a name tag of "Mu Master". This will then find and modify a security group that has nothing to do with us.
   # found = MU::MommaCat.findStray("AWS", "firewall_rule", region: MU.myRegion, dummy_ok: true, tag_key: "Name", tag_value: "Mu Master")
@@ -250,20 +250,24 @@ if $opts[:logs]
   )
 
 
-  resp = MU::Cloud::AWS.cloudtrails.describe_trails(trail_name_list: ["Mu_Trails"])
-  if resp.trail_list.size == 0
+  resp = MU::Cloud::AWS.cloudtrail.describe_trails.trail_list
+  if resp.empty?
     MU.log "Enabling Cloud Trails, logged to bucket #{$bucketname}"
 
     begin
-      MU::Cloud::AWS.cloudtrails.create_trail(
-          name: "Mu_Trails",
+      MU::Cloud::AWS.cloudtrail.create_trail(
+          name: "cloudtrail",
           s3_bucket_name: $bucketname,
-          s3_key_prefix: "AWSLogs",
           include_global_service_events: true
       )
     rescue Aws::CloudTrail::Errors::MaximumNumberOfTrailsExceededException => e
       MU.log e.inspect, MU::ERR
     end
+
+  # Make sure we actually enable cloudtrail logging
+  MU::Cloud::AWS.cloudtrail.start_logging(
+    name: "cloudtrail"
+  )
   end
 
   # Now that we've got S3 logging, let's also create an Mu_Logs stack in
@@ -314,6 +318,14 @@ if $opts[:dns]
       rescue Aws::Route53::Errors::ConflictingDomainExists
       end
     end
+    resolver = Resolv::DNS.new
+    my_ip = ""
+    begin
+      my_ip = resolver.getaddress($MU_CFG['hostname']).to_s
+    end rescue Resolv::ResolvError
+    if my_ip != MU.mu_public_ip
+      MU::Cloud::AWS::DNSZone.manageRecord(ext_zone.id, $MU_CFG['hostname'], "A", targets: [MU.mu_public_ip], sync_wait: false)
+   end
   end
 end
 

bin/mu-cleanup

@@ -16,6 +16,9 @@
 
 require File.expand_path(File.dirname(__FILE__))+"/mu-load-murc.rb"
 
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+# now we have our global config available as the read-only hash $MU_CFG
+
 require 'rubygems'
 require 'bundler/setup'
 require 'trollop'

bin/mu-deploy

@@ -15,6 +15,9 @@
 
 require File.expand_path(File.dirname(__FILE__))+"/mu-load-murc.rb"
 
+require File.realpath(File.expand_path(File.dirname(__FILE__)+"/mu-load-config.rb"))
+# now we have our global config available as the read-only hash $MU_CFG
+
 require 'rubygems'
 require 'bundler/setup'
 require 'json'

bin/mu-load-config.rb

@@ -0,0 +1 @@
+../modules/mu-load-config.rb

bin/mu-load-murc.rb

@@ -24,6 +24,7 @@ def parseRCFile(path)
   if File.readable?(path)
     File.readlines(path).each { |line|
       line.strip!
+      next if !line.match(/^export.*?=/)
       name, value = line.split(/=/, 2)
       name.sub!(/^export /, "")
       if !value.nil? and !value.empty?

bin/mu-momma-cat

@@ -78,7 +78,7 @@ start()
 
 	echo -n $"Starting $prog: "
 	ulimit -s unlimited
-	$MU_RUBY $THIN --threaded --daemonize --port $PORT --pid $PID_FILE --log $LOG_FILE --ssl --ssl-key-file $MU_DATADIR/ssl/mommacat.key --ssl-cert-file $MU_DATADIR/ssl/mommacat.crt --ssl-disable-verify --tag "mu-momma-cat" -R mommacat.ru start && success || failure
+	$MU_RUBY $THIN --threaded --daemonize --port $PORT --pid $PID_FILE --log $LOG_FILE --ssl --ssl-key-file $MU_SSL_KEY --ssl-cert-file $MU_SSL_CERT --ssl-disable-verify --tag "mu-momma-cat" -R mommacat.ru start && success || failure
 	RETVAL=$?
 	[ $RETVAL -eq 0 ] && touch $lockfile
 	echo

bin/mu-self-update

@@ -47,7 +47,7 @@ DEFAULT_BRANCH="master"
 
 usage()
 {
-  echo "Updates CAP scripts in $MU_INSTALLDIR/bin. Optionally refreshes from git."
+  echo "Updates Mu scripts in $MU_INSTALLDIR/bin. Optionally refreshes from git."
   echo "Usage: $0 [-b <branch>] [-f [-c <commit>] ] [-d] [-u] [-r]"
   echo "    -f: Forcibly re-sync $MU_LIBDIR from Git.  Saves your"
   echo "        working changes unless -d is specified."
@@ -187,9 +187,14 @@ if ! bundle install > /dev/null;then
 fi
 generate_docs
 
+# Drop old local groups in favor of LDAP's versions
+/usr/sbin/groupdel mu-users > /dev/null 2>&1
+/usr/sbin/groupdel mu-admins > /dev/null 2>&1
+
 set +e
 install_ruby
 patch_knife_windows
+generate_ssl_certs skip_chef
 install_chef
 set_bash_defaults
 setup_chef_cache $update_chef_artifacts