Merge pull request #177 from cloudamatic/development

Bolt-Tightening for 3.0
cloudamatic · Nov 23, 2019 · 086048b · 086048b
2 parents 9b21149 + a9ca6f6
commit 086048b
Show file tree

Hide file tree

Showing 36 changed files with 690 additions and 218 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,7 +12,10 @@ bin/vault
 .vscode
 Berksfile.lock
 cloud-mu-*.gem
+.kitchen/.kitchen.local.yml
+.kitchen/
 coverage
 spec/mu.yaml
 spec/azure_creds
+.kitchen.local.yml
 mu-gen-docs-index*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,16 +1,21 @@
-image: ruby:2.5
+image: ruby:2.5-slim
 
 stages:
   - Lint Test
   - Test
   - Security Test
   - Build
+  - Docker Build
   - Smoke Test
   - Merge/Tag
-  - Dockerbuild
   - Deploy
+
+variables:
+  DOCKER_DRIVER: overlay
+  AWS_REGION: us-east-1
+  CHEF_LICENSE: "accept"
 
-Rubocop:
+.Rubocop:
   stage: Lint Test
   image: chef/chefdk:latest
   script:
@@ -21,7 +26,7 @@ Rubocop:
     variables:
     - $IMAGE_BUILD
 
-Cookstyle:
+.Cookstyle:
   stage: Lint Test
   image: chef/chefdk:latest
   script:
@@ -117,134 +122,154 @@ Gem Build:
   stage: Build
   script:
     - gem build cloud-mu.gemspec
-  only:
-    - master
-    - development
-    - gen_docs
-    - /^gem-.*$/
-    - /^cicd-.*$/
   artifacts:
     paths:
     - cloud-mu-*.gem
   except:
     variables:
     - $IMAGE_BUILD
 
-.Muby Build:
-  stage: Build
-  image: centos
-  script: 
-    - yum -y groupinstall 'Development Tools'
-    - yum -y install rpm-build
-    - yum-builddep -y extras/ruby_rpm/muby.spec
-    - /usr/bin/rpmbuild -ba extras/ruby_rpm/muby.spec
-    - echo "maybe do something cool and smart here?"
+Docker Build:
+  stage: Docker Build
+  image: docker:stable
+  services:
+    - docker:dind
+  dependencies:
+    - Gem Build
+  before_script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME || true
+    - export MU_VERSION=$(grep '\.version' cloud-mu.gemspec | grep -o "'[^']*'" | tr -d "\'")
+    - export OS=$(grep 'FROM' Dockerfile | grep -o " [a-zA-Z0-9:]*" | tr -d " " | tr ":" "-")
+  script:
+    - docker build --cache-from $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME -t mu .
+    - docker run mu mu-configure --help
+    - docker tag mu $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
   only:
     - master
     - development
-  allow_failure: true
+    - /^gem-.*$/
+    - /^cicd-.*$/
+    - /^docker-.*$/
   except:
     variables:
     - $IMAGE_BUILD
+  retry: 2
 
 Gem Parser Test:
   stage: Smoke Test
+  image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  before_script:
+    - myip="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+    - mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --mu-admin-email="[email protected]" --public-address=$myip --google-region=us-east4 --aws-region=us-east-1 --azure-region=eastus
   script:
-    - curl https://gist.githubusercontent.com/ryantiger658/87ee6aca72802ce55211a7e6c6bfa76f/raw/aaa54c255936dcb7495b6befeababd44c162922d/.mu.yaml >> /root/.mu.yaml
-    - gem install cloud-mu-*.gem
-    - MY_ADDR=`hostname -I | awk '{print $1}'`
-    - mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --mu-admin-email="[email protected]" --public-address="${MY_ADDR}" --google-project="egt-labs-admin" --google-region=us-east4 --aws-region=us-east-1 --azure-region=eastus
-    - cat /root/.mu.yaml
     - mu-deploy -d modules/tests/super_simple_bok.yml
     - mu-deploy -d modules/tests/super_complex_bok.yml
-  dependencies:
-  - Gem Build
   only:
     - master
     - development
     - /^gem-.*$/
     - /^cicd-.*$/
-  allow_failure: true
   except:
     variables:
     - $IMAGE_BUILD
 
+.Mu Install:
+  stage: Smoke Test
+  image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  before_script:
+    - myip="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+    - mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --mu-admin-email="[email protected]" --public-address=$myip --google-region=us-east4 --aws-region=us-east-1 --azure-region=eastus
+  script:
+    - for cloud in AWS Azure Google; do mu-deploy install/mu-master.yaml -p name=$cloud-MU-$CI_COMMIT_SHORT_SHA -p cloud=$cloud; done
+  only:
+  - master
+  - development
+  - gem-mess_with_test_kitchen
+  artifacts:
+    paths:
+    - /tmp/ssh_debug
+
+Test Kitchen:
+  stage: Smoke Test
+  image: chef/chefdk:latest
+  before_script:
+  - echo "export MU_BRANCH=$(CI_COMMIT_REF_NAME)" > ./kitchen_vars
+  script:
+  - kitchen test mu-install-aws-amazon2 mu-install-aws-centos-7 -c 5
+  after_script:
+  - kitchen destroy
+  only:
+  - master
+  - development
+  when: manual
+
 Smoke Test:
   stage: Smoke Test
+  before_script:
+    - apt-get -qq update
+    - apt-get -qq -y install dnsutils
+    - myip="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+    - mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --mu-admin-email="[email protected]" --public-address=$myip --google-region=us-east4 --aws-region=us-east-1 --azure-region=eastus
   script:
-    - MY_ADDR=`hostname -I | awk '{print $1}'`
-    - mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --mu-admin-email="[email protected]" --public-address="${MY_ADDR}" --google-project="egt-labs-admin" --google-region=us-east4 --aws-region=us-east-1 --azure-region=eastus
-    - mu-upload-chef-artifacts -sn
-    - mu-deploy /opt/mu/var/demo_platform/applications/gitlab-server.yml -p vpc_id=vpc-040da43493f894a8d
+  - mu-upload-chef-artifacts -sn
+  - mu-deploy /opt/mu/var/demo_platform/applications/gitlab-server.yml -p vpc_id=vpc-040da43493f894a8d
   tags: 
-    - mu-master
+  - mu-master
   only:
-    - master
-    - development
+  - master
+  - development
   when: manual
   except:
     variables:
     - $IMAGE_BUILD
 
-.Dockerbuild:
-  stage: Dockerbuild
-  image: docker:latest
-  script:
-    - echo "Build Mu Docker Container"
-  only:
-    - master
-    - development
-  except:
-    variables:
-    - $IMAGE_BUILD
 
 Gen Docs:
   stage: Merge/Tag
+  image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  before_script:
+    - mkdir public/
+    - mkdir -p /var/www/html/docs
+    - myip="$(dig +short myip.opendns.com @resolver1.opendns.com)"
+    - mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --mu-admin-email="[email protected]" --public-address="${myip}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --google-region=us-east4 --google-project="egt-labs-admin" --aws-region=us-east-1 --azure-region=eastus
   script:
-  - curl https://gist.githubusercontent.com/ryantiger658/87ee6aca72802ce55211a7e6c6bfa76f/raw/aaa54c255936dcb7495b6befeababd44c162922d/.mu.yaml >> /root/.mu.yaml
-  - MY_ADDR=`hostname -I | awk '{print $1}'`
-  - gem install cloud-mu-*.gem
-  - ruby bin/mu-configure -n --aws-access-key="${AWS_ACCESS_KEY_ID}" --aws-access-secret="${AWS_SECRET_ACCESS_KEY}" --azure-directory-id="${AZURE_DIRECTORY_ID}" --azure-client-id="${AZURE_CLIENT_ID}" --azure-client-secret="${AZURE_CLIENT_SECRET}" --azure-subscription="${AZURE_SUBSCIPTION_ID}" --google-credentials-encoded="${GOOGLE_CREDS_ENCODED}" --mu-admin-email="[email protected]" --public-address="${MY_ADDR}" --google-project="egt-labs-admin" --google-region=us-east4 --aws-region=us-east-1 --azure-region=eastus
-  - cat /root/.mu.yaml
-  - ruby bin/mu-gen-docs
-  - mkdir public
-  - cp -Rf /var/www/html/docs/* public
-  - ls public
-  dependencies:
-  - Gem Build
+    - mu-gen-docs
+    - mv -f /var/www/html/docs/* public/
   only:
   - master
   - development
   - gen_docs
+  retry: 2
   artifacts:
     paths:
     - public/
   except:
     variables:
     - $IMAGE_BUILD
 
-.GitHub Pages:
+DockerHub Upload:
   stage: Deploy
-  image: bitnami/git:latest
+  services:
+  - docker:dind
+  image: docker:stable
+  when: on_success
+  before_script:
+    - export MU_VERSION=$(grep '\.version' cloud-mu.gemspec | grep -o "'[^']*'" | tr -d "\'")
+    - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+    - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME mu
+    - docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PWD
   script:
-  - echo "Hello I am going to push to GitHub"
-  - ssh-keyscan -H github.com >> ~/.ssh/known_hosts
-  - git clone [email protected]:cloudamatic/cloudamatic.github.io.git
-  - cp -Rf public/* cloudamatic.github.io
-  - cd cloudamatic.github.io
-  - git status
-  - git add -a
-  - git commit -m "$CI_COMMIT_MESSAGE"
-  - git push origin master
-  dependencies:
-  - Gen Docs
+    - docker tag mu egtlabs/mu:$MU_VERSION
+    - docker tag mu egtlabs/mu:latest
+    - docker tag mu egtlabs/mu:development
+    - docker push egtlabs/mu:$MU_VERSION
+    - if [ $CI_COMMIT_REF_NAME == "master" ]; then docker push egtlabs/mu:latest; fi
+    - if [ $CI_COMMIT_REF_NAME == "development" ]; then  docker push egtlabs/mu:development; fi
   only:
-  - master
-  - gen_docs
-  allow_failure: true
-  except:
-    variables:
-    - $IMAGE_BUILD
+    - master
+    - development
 
 Upload Gem:
   stage: Deploy

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,25 @@
+FROM ruby:2.5-slim
+
+RUN mkdir -p /opt/mu/etc/ /home/mu /usr/local/ruby-current/lib/ruby/gems/2.5.0/gems/var/
+
+WORKDIR /home/mu
+
+RUN apt-get update
+
+RUN apt-get install -y ruby2.5-dev dnsutils ansible build-essential
+
+RUN apt-get upgrade -y
+
+COPY ./cloud-mu-*.gem /home/mu
+
+RUN gem install ./cloud-mu-*.gem thin -N
+
+RUN rm cloud-mu-*.gem
+
+RUN apt-get remove -y build-essential ruby2.5-dev
+
+RUN apt-get autoremove -y
+
+EXPOSE 2260
+
+CMD /usr/sbin/init
diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 mu -- Cloudamatic Automation Tooling
 ===
 [![pipeline status](https://gitlab.com/cloudamatic/mu/badges/master/pipeline.svg)](https://gitlab.com/cloudamatic/mu/commits/master)
-[![Gem Version](https://badge.fury.io/rb/cloud-mu.svg)](https://badge.fury.io/rb/cloud-mu)
 [![Maintainability](https://api.codeclimate.com/v1/badges/dd4e5d867890336accd1/maintainability)](https://codeclimate.com/github/cloudamatic/mu/maintainability)
-[![Inline docs](http://inch-ci.org/github/cloudamatic/mu.svg?branch=master)](http://inch-ci.org/github/cloudamatic/mu)
+[![Gem Version](https://badge.fury.io/rb/cloud-mu.svg)](https://badge.fury.io/rb/cloud-mu)
+[![Docker Version](https://images.microbadger.com/badges/version/egtlabs/mu.svg)](https://microbadger.com/images/egtlabs/mu)
 
 # About mu
 **Mu**  is the deployer and developer toolset for the Cloudamatic suite of services, designed to provision, orchestrate and manage complex platforms and applications. At [eGT Labs](https://www.eglobaltech.com/egt-labs/), we use mu for rapid prototyping of cloud migration efforts for federal customers, for managing cloud applications throughout their lifecycles, and as a tools library for cloud maintenance tasks.

diff --git a/bin/mu-node-manage b/bin/mu-node-manage
@@ -599,7 +599,7 @@ def updateAWSMetaData(deploys = MU::MommaCat.listDeploys, nodes = [])
 
       MU.log "Updating #{nodename} userdata (#{server["conf"]["platform"]})"
       begin
-        MU::Cloud::AWS.ec2(server['region']).modify_instance_attribute(
+        MU::Cloud::AWS.ec2(region: server['region']).modify_instance_attribute(
           instance_id: id,
           attribute: "userData",
           value: Base64.encode64(userdata)

diff --git a/chefignore b/chefignore
@@ -0,0 +1 @@
+.kitchen
diff --git a/cloud-mu.gemspec b/cloud-mu.gemspec
@@ -17,8 +17,8 @@ end
 
 Gem::Specification.new do |s|
   s.name        = 'cloud-mu'
-  s.version     = '3.0.0'
-  s.date        = '2019-11-11'
+  s.version     = '3.0.1'
+  s.date        = '2019-11-22'
   s.require_paths = ['modules']
   s.required_ruby_version = '>= 2.4'
   s.summary     = "The eGTLabs Mu toolkit for unified cloud deployments"