refactor vault role (kubernetes-sigs#2733)

* Move front-proxy-client certs back to kube mount

We want the same CA for all k8s certs

* Refactor vault to use a third party module

The module adds idempotency and reduces some of the repetitive
logic in the vault role

Requires ansible-modules-hashivault on ansible node and hvac
on the vault hosts themselves

Add upgrade test scenario
Remove bootstrap-os tags from tasks

* fix upgrade issues

* improve unseal logic

* specify ca and fix etcd check

* Fix initialization check

bump machine size

.gitlab-ci.yml

@@ -304,6 +304,10 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
+.coreos_vault_upgrade_variables: &coreos_vault_upgrade_variables
+# stage: deploy-part1
+  UPGRADE_TEST: "basic"
+
 .ubuntu_flannel_variables: &ubuntu_flannel_variables
 # stage: deploy-special
   MOVED_TO_GROUP_VARS: "true"
@@ -560,7 +564,7 @@ gce_rhel7-canal-sep:
     <<: *rhel7_canal_sep_variables
   when: manual
   except: ['triggers']
-  only: ['master', /^pr-.*$/,]
+  only: ['master', /^pr-.*$/]
 
 gce_rhel7-canal-sep-triggers:
   stage: deploy-part2
@@ -638,6 +642,17 @@ gce_ubuntu-vault-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+gce_coreos-vault-upgrade:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *coreos_vault_upgrade_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 gce_ubuntu-flannel-sep:
   stage: deploy-special
   <<: *job

requirements.txt

@@ -1,4 +1,6 @@
-pbr>=1.6
 ansible>=2.4.0
-netaddr
 jinja2>=2.9.6
+netaddr
+pbr>=1.6
+ansible-modules-hashivault>=3.9.4
+hvac

roles/bootstrap-os/tasks/bootstrap-ubuntu.yml

@@ -8,6 +8,7 @@
   changed_when: false
   with_items:
     - python
+    - python-apt
     - pip
     - dbus-daemon
   tags:

roles/dnsmasq/tasks/main.yml

@@ -3,15 +3,11 @@
   file:
     path: /etc/dnsmasq.d
     state: directory
-  tags:
-    - bootstrap-os
 
 - name: ensure dnsmasq.d-available directory exists
   file:
     path: /etc/dnsmasq.d-available
     state: directory
-  tags:
-    - bootstrap-os
 
 - name: check system nameservers
   shell: awk '/^nameserver/ {print $NF}' /etc/resolv.conf

roles/download/defaults/main.yml

@@ -37,7 +37,7 @@ calico_rr_version: "v0.4.2"
 flannel_version: "v0.10.0"
 flannel_cni_version: "v0.3.0"
 istio_version: "0.2.6"
-vault_version: 0.8.1
+vault_version: 0.10.1
 weave_version: 2.3.0
 pod_infra_version: 3.0
 contiv_version: 1.1.7

roles/etcd/defaults/main.yml

@@ -50,7 +50,7 @@ etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr',
 
 etcd_compaction_retention: "8"
 
-etcd_vault_mount_path: etcd
+etcd_vault_mount_path: "/etcd"
 
 # Force clients like etcdctl to use TLS certs (different than peer security)
 etcd_secure_client: true

roles/kubernetes/master/defaults/main.yml

@@ -100,3 +100,6 @@ kube_encrypt_secret_data: false
 kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}"
 # Must be either: aescbc, secretbox or aesgcm
 kube_encryption_algorithm: "aescbc"
+
+# You may want to use ca.pem depending on your situation
+kube_front_proxy_ca: "front-proxy-ca.pem"

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -117,8 +117,13 @@ spec:
     - --feature-gates={{ kube_feature_gates|join(',') }}
 {% endif %}
 {% if kube_version | version_compare('v1.9', '>=') %}
-    - --requestheader-client-ca-file={{ kube_cert_dir }}/front-proxy-ca.pem
+    - --requestheader-client-ca-file={{ kube_cert_dir }}/{{ kube_front_proxy_ca }}
+{# FIXME(mattymo): Vault certs do not work with front-proxy-client #}
+{% if cert_management == "vault" %}
+    - --requestheader-allowed-names=
+{% else %}
     - --requestheader-allowed-names=front-proxy-client
+{% endif %}
     - --requestheader-extra-headers-prefix=X-Remote-Extra-
     - --requestheader-group-headers=X-Remote-Group
     - --requestheader-username-headers=X-Remote-User

roles/kubernetes/node/defaults/main.yml

@@ -128,13 +128,13 @@ vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('')
 
 ## When azure is used, you need to also set the following variables.
 ## see docs/azure.md for details on how to get these values
-#azure_tenant_id:
-#azure_subscription_id:
-#azure_aad_client_id:
-#azure_aad_client_secret:
-#azure_resource_group:
-#azure_location:
-#azure_subnet_name:
-#azure_security_group_name:
-#azure_vnet_name:
-#azure_route_table_name:
+# azure_tenant_id:
+# azure_subscription_id:
+# azure_aad_client_id:
+# azure_aad_client_secret:
+# azure_resource_group:
+# azure_location:
+# azure_subnet_name:
+# azure_security_group_name:
+# azure_vnet_name:
+# azure_route_table_name:

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -13,6 +13,5 @@
 
 - import_tasks: set_resolv_facts.yml
   tags:
-    - bootstrap-os
     - resolvconf
     - facts

roles/kubernetes/secrets/defaults/main.yml

@@ -1,4 +1,3 @@
 ---
 kube_cert_group: kube-cert
-kube_vault_mount_path: kube
-front_proxy_vault_mount_path: front-proxy
+kube_vault_mount_path: "/kube"

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -12,7 +12,6 @@
     - k8s-secrets
     - kube-controller-manager
     - kube-apiserver
-    - bootstrap-os
     - apps
     - network
     - master
@@ -28,7 +27,6 @@
   when: gen_certs|default(false)
   tags:
     - k8s-secrets
-    - bootstrap-os
 
 - name: Gen_certs | write openssl config
   template:

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -127,7 +127,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: front-proxy-client
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
-    issue_cert_mount_path: "{{ front_proxy_vault_mount_path }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_front_proxy_clients_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
   notify: set secret_changed

roles/kubernetes/secrets/tasks/main.yml

@@ -41,7 +41,6 @@
     - k8s-secrets
     - kube-controller-manager
     - kube-apiserver
-    - bootstrap-os
     - apps
     - network
     - master
@@ -57,7 +56,6 @@
   when: gen_certs|default(false) or gen_tokens|default(false)
   tags:
     - k8s-secrets
-    - bootstrap-os
 
 - name: "Get_tokens | Make sure the tokens directory exits (on {{groups['kube-master'][0]}})"
   file:

roles/vault/defaults/main.yml

@@ -16,10 +16,11 @@ vault_cert_dir: "{{ vault_base_dir }}/ssl"
 vault_config_dir: "{{ vault_base_dir }}/config"
 vault_roles_dir: "{{ vault_base_dir }}/roles"
 vault_secrets_dir: "{{ vault_base_dir }}/secrets"
+vault_lib_dir: "/var/lib/vault"
 vault_log_dir: "/var/log/vault"
 
-vault_version: 0.8.1
-vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
+vault_version: 0.10.1
+vault_binary_checksum: 66f0f1b0b221d664dd5913f8697409d7401df4bb2a19c7277e8fbad152063fae
 vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
 vault_download_vars:
   container: "{{ vault_deployment_type != 'host' }}"
@@ -64,10 +65,10 @@ vault_config:
     etcd:
       address: "{{ vault_etcd_url }}"
       ha_enabled: "true"
-      redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}"
-      tls_ca_file: "{{ vault_etcd_cert_dir }}/ca.pem"
-      tls_cert_file: "{{ vault_etcd_cert_dir}}/node-{{ inventory_hostname }}.pem"
-      tls_key_file: "{{ vault_etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem"
+      redirect_addr: "https://{{ inventory_hostname }}:{{ vault_port }}"
+      tls_ca_file: "{{ etcd_cert_dir }}/ca.pem"
+      tls_cert_file: "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem"
+      tls_key_file: "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem"
   cluster_name: "kubernetes-vault"
   default_lease_ttl: "{{ vault_default_lease_ttl }}"
   max_lease_ttl: "{{ vault_max_lease_ttl }}"
@@ -80,6 +81,8 @@ vault_config:
 vault_secret_shares: 1
 vault_secret_threshold: 1
 
+vault_successful_http_codes: ["200", "429", "500", "501", "503"]
+
 vault_ca_options:
   vault:
     common_name: vault
@@ -97,20 +100,29 @@ vault_ca_options:
     format: pem
     ttl: "{{ vault_max_lease_ttl }}"
     exclude_cn_from_sans: true
-  front_proxy:
-    common_name: front-proxy
-    format: pem
-    ttl: "{{ vault_max_lease_ttl }}"
-    exclude_cn_from_sans: true
 
 vault_client_headers:
   Accept: "application/json"
   Content-Type: "application/json"
 
-vault_etcd_cert_dir: /etc/ssl/etcd/ssl
-vault_kube_cert_dir: /etc/kubernetes/ssl
+etcd_cert_dir: /etc/ssl/etcd/ssl
+kube_cert_dir: /etc/kubernetes/ssl
 
 vault_pki_mounts:
+  userpass:
+    name: userpass
+    default_lease_ttl: "{{ vault_default_lease_ttl }}"
+    max_lease_ttl: "{{ vault_max_lease_ttl }}"
+    description: "Userpass"
+    cert_dir: "{{ vault_cert_dir }}"
+    roles:
+      - name: userpass
+        group: userpass
+        password: "{{ lookup('password', inventory_dir + '/credentials/vault/userpass.creds length=15') }}"
+        policy_rules: default
+        role_options:
+          allow_any_name: true
+
   vault:
     name: vault
     default_lease_ttl: "{{ vault_default_lease_ttl }}"
@@ -122,13 +134,14 @@ vault_pki_mounts:
         group: vault
         password: "{{ lookup('password', inventory_dir + '/credentials/vault/vault.creds length=15') }}"
         policy_rules: default
-        role_options: default
+        role_options:
+          allow_any_name: true
   etcd:
     name: etcd
     default_lease_ttl: "{{ vault_default_lease_ttl }}"
     max_lease_ttl: "{{ vault_max_lease_ttl }}"
     description: "Etcd Root CA"
-    cert_dir: "{{ vault_etcd_cert_dir }}"
+    cert_dir: "{{ etcd_cert_dir }}"
     roles:
       - name: etcd
         group: etcd
@@ -143,7 +156,7 @@ vault_pki_mounts:
     default_lease_ttl: "{{ vault_default_lease_ttl }}"
     max_lease_ttl: "{{ vault_max_lease_ttl }}"
     description: "Kubernetes Root CA"
-    cert_dir: "{{ vault_kube_cert_dir }}"
+    cert_dir: "{{ kube_cert_dir }}"
     roles:
       - name: kube-master
         group: kube-master
@@ -153,6 +166,14 @@ vault_pki_mounts:
           allow_any_name: true
           enforce_hostnames: false
           organization: "system:masters"
+      - name: front-proxy-client
+        group: kube-master
+        password: "{{ lookup('password', inventory_dir + '/credentials/vault/kube-proxy.creds length=15') }}"
+        policy_rules: default
+        role_options:
+          allow_any_name: true
+          enforce_hostnames: false
+          organization: "system:front-proxy-client"
       - name: kube-node
         group: k8s-cluster
         password: "{{ lookup('password', inventory_dir + '/credentials/vault/kube-node.creds length=15') }}"
@@ -169,18 +190,3 @@ vault_pki_mounts:
           allow_any_name: true
           enforce_hostnames: false
           organization: "system:node-proxier"
-  front_proxy:
-    name: front-proxy
-    default_lease_ttl: "{{ vault_default_lease_ttl }}"
-    max_lease_ttl: "{{ vault_max_lease_ttl }}"
-    description: "Kubernetes Front Proxy CA"
-    cert_dir: "{{ vault_kube_cert_dir }}"
-    roles:
-      - name: front-proxy-client
-        group: k8s-cluster
-        password: "{{ lookup('password', inventory_dir + '/credentials/vault/front-proxy-client.creds length=15') }}"
-        policy_rules: default
-        role_options:
-          allow_any_name: true
-          enforce_hostnames: false
-          organization: "system:front-proxy"

roles/vault/handlers/main.yml

@@ -0,0 +1,50 @@
+---
+- name: restart vault
+  command: /bin/true
+  notify:
+    - restart vault service
+    - set facts about local Vault health
+    - unseal vault
+
+- name: wait for vault up
+  uri:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: "{{ vault_successful_http_codes | join(',') }}"
+  register: vault_health_check
+  until: vault_health_check|succeeded
+  retries: 10
+  delay: "{{ retry_stagger | random + 3 }}"
+  run_once: yes
+  notify: set facts about local Vault health
+
+- name: wait for vault up nowait
+  uri:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: "{{ vault_successful_http_codes | join(',') }}"
+  register: vault_health_check
+  run_once: yes
+  failed_when: false
+  notify: set facts about local Vault health
+
+- name: set facts about local Vault health
+  set_fact:
+    vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}"
+    vault_cluster_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}"
+
+- name: restart vault service
+  systemd:
+    daemon_reload: true
+    enabled: yes
+    name: vault
+    state: restarted
+
+- name: unseal vault
+  hashivault_unseal:
+    url: "{{ vault_leader_url | default('https://localhost:8200') }}"
+    token: "{{ vault_root_token }}"
+    ca_cert: "{{ vault_cert_dir }}/ca.pem"
+    keys: "{{ item }}"
+  with_items: "{{ vault_unseal_keys|default([]) }}"

roles/vault/tasks/bootstrap/create_mounts.yml

@@ -1,12 +1,13 @@
 ---
 - include_tasks: ../shared/create_mount.yml
   vars:
-    create_mount_path: "{{ item.name }}"
+    create_mount_path: "/{{ item.name }}"
     create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}"
     create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}"
     create_mount_description: "{{ item.description }}"
     create_mount_cert_dir: "{{ item.cert_dir }}"
     create_mount_config_ca_needed: "{{ item.config_ca }}"
   with_items:
+    - "{{ vault_pki_mounts.userpass|combine({'config_ca': not vault_ca_cert_needed}) }}"
     - "{{ vault_pki_mounts.vault|combine({'config_ca': not vault_ca_cert_needed}) }}"
     - "{{ vault_pki_mounts.etcd|combine({'config_ca': not vault_etcd_ca_cert_needed}) }}"

roles/vault/tasks/bootstrap/create_roles.yml

@@ -6,5 +6,5 @@
     create_role_policy_rules: "{{ item.policy_rules }}"
     create_role_password: "{{ item.password }}"
     create_role_options: "{{ item.role_options }}"
-    create_role_mount_path: "{{ mount.name }}"
+    create_role_mount_path: "/{{ mount.name }}"
   with_items: "{{ mount.roles }}"