feat: add vault (#27)

feat: add vault refactor(playbook): use the vault cluster role instead build: reduce wait time for apt update to 90s ci: put cached binaries in /usr/bin ci: put cached binaries in /home/becker/bin ci: don't change path in workflow, call it explicitly ci: remove spurious vault statement ci: fix pip install step chore: remove explicit ansible-core dependency ci: add r flag to galaxy requirements install step build(ansible): rework the vault playbook fix(consul): remove empty task style(consul): use fqcn for consul playbooks fix(consul): fix typo in handler name fix(vault): fix group present in vault playbook style(consul): capitalise task name ci: add vault axis to matrix fix(vault): add fqcn to tasks fix(vault): add default values for tag name and autojoin token Signed-off-by: Bruce Becker <[email protected]> --------- Signed-off-by: Bruce Becker <[email protected]>
brucellino · Mar 15, 2023 · 90b3469 · 90b3469
1 parent a460ecc
commit 90b3469
Show file tree

Hide file tree

Showing 8 changed files with 262 additions and 30 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -13,54 +13,59 @@ jobs:
         template:
           - ubuntu-server
           - consul
+          - vault
     name: Validate
     runs-on: ubuntu-latest
     env:
       VAULT_ADDR: http://localhost:8200
       VAULT_TOKEN: token
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - name: Run ansible-lint
+        uses: ansible/[email protected]
+        with:
+          path: "${{ matrix.template }}"
+      - name: Create bin dir for cached binaries
+        run: mkdir "${HOME}/bin"
       - name: Start Vault Server
         run: |
           curl https://releases.hashicorp.com/vault/1.13.0/vault_1.13.0_linux_amd64.zip | \
-          gunzip -> vault ; \
-          chmod u+x vault ; \
-          ./vault server -dev -dev-root-token-id=${VAULT_TOKEN} | tee vault-output.txt &
+          gunzip -> "${HOME}/bin/vault" ; \
+          chmod u+x "${HOME}/bin/vault" ; \
+          ${HOME}/bin/vault server -dev -dev-root-token-id=${VAULT_TOKEN} | tee vault-output.txt &
       - name: Enable Secrets mounts
         run: |
-          ./vault secrets enable -path="digitalocean" -description="KV data" kv-v2 ; \
-          ./vault secrets enable -path="kv" kv-v2
+          ${HOME}/bin/vault secrets enable -path="digitalocean" -description="KV data" kv-v2 ; \
+          ${HOME}/bin/vault secrets enable -path="kv" kv-v2
       - name: Populate the DO secret
-        run: ./vault kv put -mount="digitalocean" tokens packer=${{ secrets.DO_TOKEN }}
-      - name: Create the consul secrets
-        run: ./vault kv put
+        run: ${HOME}/bin/vault kv put -mount="digitalocean" tokens packer=${{ secrets.DO_TOKEN }}
       - name: Populate the GH secret
-        run: ./vault kv put -mount kv github ghcr_token=${{ secrets.GITHUB_TOKEN }}
+        run: ${HOME}/bin/vault kv put -mount="kv" github ghcr_token=${{ secrets.GITHUB_TOKEN }}
       - name: Populate Consul Encryption Key
-        run: ./vault kv put -mount kv consul encrypt=${{ secrets.CONSUL_ENCRYPT_KEY }}
+        run: ${HOME}/bin/vault kv put -mount="kv" consul encrypt=${{ secrets.CONSUL_ENCRYPT_KEY }}
       - name: "Get Packer"
         run: |
           curl https://releases.hashicorp.com/packer/1.8.2/packer_1.8.2_linux_amd64.zip | \
-          gunzip -> packer ; \
-          chmod u+x packer
+          gunzip -> "${HOME}/bin/packer" ; \
+          chmod u+x "${HOME}/bin/packer"
       - name: Add Ansible requirements
-        run: python3 -m pip install requirements.txt
+        run: python3 -m pip install -r requirements.txt
       - name: Add Ansible collections
         run: ansible-galaxy collection install community.hashi_vault
       - name: Add Ansible roles
         run: |
           cd ${{ matrix.template }}
           if [[ -f requirements.yml ]] ; then
-            ansible-galaxy install requirements.yml
+            ansible-galaxy install -r requirements.yml
           fi
       - name: "Init Packer"
         run: |
-          cd ${{ matrix.template }} ; ../packer init .
+          cd ${{ matrix.template }} ; ${HOME}/bin/packer init .
       - name: "Validate Packer templates"
-        run: cd ${{ matrix.template }} ; PATH=${PATH}:../ packer validate .
+        run: cd ${{ matrix.template }} ; PATH=${PATH}:${HOME}/bin/ packer validate .
 
   release:
     needs:

diff --git a/consul/playbook.yml b/consul/playbook.yml
@@ -6,40 +6,41 @@
     consul_version: 1.15.0
   pre_tasks:
     - name: Wait
-      pause:
+      ansible.builtin.pause:
         seconds: 90
-    - name: force update
-      raw: apt-get -y update
+    - name: Force update
+      ansible.builtin.raw: apt-get -y update
+      changed_when: false
   tasks:
     - name: Get Consul
-      unarchive:
+      ansible.builtin.unarchive:
         src: "https://releases.hashicorp.com/consul/{{ consul_version }}/consul_{{ consul_version }}_linux_amd64.zip"
         dest: /usr/bin/consul
         remote_src: true
         mode: 0777
         owner: root.
         group: rooot
     - name: Add Consul group
-      group:
+      ansible.builtin.group:
         name: consul
         state: present
     - name: Add Consul user
-      user:
+      ansible.builtin.user:
         name: consul
         group: consul
         groups: consul
         append: true
         state: present
     - name: Add configuration directory
-      file:
+      ansible.builtin.file:
         path: /etc/consul.d
         state: directory
         recurse: true
         mode: 0644
         owner: consul
         group: consul
     - name: Create Consul configuration
-      template:
+      ansible.builtin.template:
         src: consul.hcl.j2
         dest: /etc/consul.d/consul.hcl
         backup: true
@@ -52,7 +53,7 @@
         variable_end_string: "]]"
 
     - name: Create Systemd Unit
-      copy:
+      ansible.builtin.copy:
         dest: /etc/systemd/system/consul.service
         content: |
           [Unit]
@@ -82,10 +83,9 @@
         owner: true
         group: true
       notify: daemon-reload
-    - name: Start service
   handlers:
-    - name: daemin-reload
-      systemd:
+    - name: daemon-reload
+      ansible.builtin.systemd:
         name: consul
         state: started
         enabled: true

diff --git a/requirements.txt b/requirements.txt
@@ -1,6 +1,5 @@
 # Python requirements
 ansible==7.3.0
-ansible-core==2.13.2
 ansible-lint==5.4.0
 attrs==21.2.0
 awxkit==19.4.0

diff --git a/vault/playbook.yml b/vault/playbook.yml
@@ -0,0 +1,82 @@
+---
+- hosts: all
+  name: Setup
+  become: true
+  vars:
+    vault_version: 1.13.0
+    vault_config_dir: /etc/vault
+    vault_bin_dir: /usr/local/bin
+    raft_data_dir: /data/raft
+    tag_name: vault
+    autojoin_token: ""
+    prerequisites:
+      - jq
+      - net-tools
+      - curl
+  pre_tasks:
+    - name: Wait
+      ansible.builtin.pause:
+        seconds: 60
+    - name: Force update
+      ansible.builtin.raw: apt-get -y update
+      changed_when: false
+  tasks:
+    - name: Ensure prerequisites
+      ansible.builtin.package:
+        name: "{{ prerequisites }}"
+        state: present
+    - name: Ensure Vault Group
+      ansible.builtin.group:
+        name: vault
+        state: present
+    - name: Ensure Vault user
+      ansible.builtin.user:
+        name: vault
+        comment: "Vault user added by Ansible"
+        groups: vault
+        append: true
+        state: present
+        create_home: false
+        generate_ssh_key: false
+    - name: Get Vault
+      ansible.builtin.unarchive:
+        src: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
+        dest: "{{ vault_bin_dir }}/vault"
+        remote_src: true
+        mode: 0777
+        owner: root
+        group: root
+    - name: Ensure Vault Configuration Directory
+      ansible.builtin.file:
+        path: "{{ vault_config_dir }}"
+        state: directory
+        recurse: true
+        mode: 0770
+        owner: vault
+        group: vault
+    - name: Template Vault Configuration
+      ansible.builtin.template:
+        src: vault.hcl.j2
+        dest: "{{ vault_config_dir }}/vault.hcl"
+        mode: 0660
+        owner: vault
+        group: vault
+        block_start_string: "{%"
+        block_end_string: "%}"
+        variable_start_string: "[["
+        variable_end_string: "]]"
+        # validate:
+    - name: Template systemd unit
+      ansible.builtin.template:
+        src: vault.service.j2
+        dest: /etc/systemd/system/vault.service
+        mode: 0644
+        owner: root
+        group: root
+    - name: Enable Vault service
+      ansible.builtin.systemd:
+        name: vault
+        state: reloaded
+        enabled: true
+        masked: false
+        daemon_reload: true
diff --git a/vault/requirements.yml b/vault/requirements.yml
@@ -0,0 +1,3 @@
+# Ansible role requirements
+- name: consul
+  src: brucellino.consul
diff --git a/vault/vault-do.pkr.hcl b/vault/vault-do.pkr.hcl
@@ -0,0 +1,69 @@
+packer {
+  required_plugins {
+    digitalocean = {
+      version = ">= v1.1.0"
+      source  = "github.com/digitalocean/digitalocean"
+    }
+  }
+}
+
+variable "region" {
+  type      = string
+  default   = "ams3"
+  sensitive = false
+}
+
+variable "size" {
+  type    = string
+  default = "s-1vcpu-512mb-10gb"
+}
+
+variable "base_image_name" {
+  type      = string
+  sensitive = false
+  default   = "20.04 (LTS) x64"
+}
+
+local "do_token" {
+  expression = vault("digitalocean/data/tokens", "packer")
+  sensitive  = true
+}
+
+
+variable "vpc_uuid" {
+  type      = string
+  sensitive = false
+  default   = "08a4d3ad-a229-40dd-8dd4-042bda3e09bc" # this is only available in AMS3 - a map is needed.
+}
+
+data "digitalocean-image" "base-ubuntu" {
+  api_token = vault("digitalocean/data/tokens", "packer")
+  name      = var.base_image_name
+  region    = var.region
+  type      = "distribution"
+}
+
+
+source "digitalocean" "server" {
+  api_token          = local.do_token
+  image              = data.digitalocean-image.base-ubuntu.image_id
+  region             = var.region
+  size               = var.size
+  ssh_username       = "root"
+  snapshot_name      = "vault_snap-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}"
+  snapshot_regions   = [var.region]
+  droplet_agent      = true
+  monitoring         = true
+  private_networking = true
+  droplet_name       = "vault-build-${formatdate("YYYY-MM-DD-hh-mm", timestamp())}"
+  tags               = ["packer", "vault"]
+  vpc_uuid           = var.vpc_uuid
+}
+
+build {
+  name    = "server"
+  sources = ["source.digitalocean.server"]
+  provisioner "ansible" {
+    playbook_file = "playbook.yml"
+  }
+}
diff --git a/vault/vault.hcl.j2 b/vault/vault.hcl.j2
@@ -0,0 +1,37 @@
+storage "raft" {
+ path = "[[ raft_data_dir ]]"
+ node_id = "digitalocean-host"
+
+ retry_join {
+  auto_join = "provider=digitalocean region=[[ region ]] tag_name=[[ tag_name ]] api_token=[[ autojoin_token ]]"
+  auto_join_scheme = "http"
+ }
+
+}
+
+cluster_name = "hah"
+disable_mlock = false
+
+listener "tcp" {
+  address = "127.0.0.1:8200"
+  tls_disable = true
+}
+
+listener "tcp" {
+  address = "{{ GetInterfaceIP \"eth0\" }}:8200"
+  tls_disable = true
+}
+
+api_addr = "http://{{ GetInterfaceIP \"eth0\" }}:8200"
+cluster_addr = "http://{{ GetInterfaceIP \"eth0\" }}:8201"
+
+{% if consul_agent | default (false) %}
+service_registration "consul" {
+  address = "127.0.0.1:8500"
+  tls_skip_verify = "true"
+  check_timeout = "30s"
+  scheme = "http"
+}
+{% endif %}
+
+ui = true
diff --git a/vault/vault.service.j2 b/vault/vault.service.j2
@@ -0,0 +1,37 @@
+# See https://medium.com/hashicorp-engineering/systemd-service-file-for-vault-3e339ff86bc6
+[Unit]
+Description="HashiCorp Vault - A tool for managing secrets"
+Documentation=https://www.vaultproject.io/docs/
+Requires=network-online.target
+After=network-online.target
+ConditionFileNotEmpty={{ vault_config_dir }}/vault.hcl
+StartLimitIntervalSec=60
+StartLimitBurst=3
+
+[Service]
+User=vault
+Group=vault
+ProtectSystem=full
+ProtectHome=read-only
+PrivateTmp=yes
+PrivateDevices=yes
+SecureBits=keep-caps
+AmbientCapabilities=CAP_IPC_LOCK
+CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK
+NoNewPrivileges=yes
+Environment="GODEBUG=x509ignoreCN=0"
+ExecStart={{ vault_bin_dir }}/vault server -config={{ vault_config_dir }}/vault.hcl
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillMode=process
+KillSignal=SIGINT
+Restart=on-failure
+RestartSec=5
+TimeoutStopSec=30
+StartLimitInterval=60
+StartLimitBurst=3
+LimitNOFILE=65536
+LimitMEMLOCK=infinity
+StandardOutput=append:/var/log/vault.log
+
+[Install]
+WantedBy=multi-user.target