Merge pull request #37 from devops-rob/db_race_condition

DB race condition

examples/db_cassandra/main.tf

@@ -1,5 +1,5 @@
 provider "vault" {
-  address = "http://localhost:8200"
+  address = var.vault_addr
   token   = var.vault_token
 }
 
@@ -8,6 +8,9 @@ variable "cassandra_path" {
 }
 
 variable "vault_token" {}
+variable "vault_addr" {
+  default = "http://localhost:8200"
+}
 variable "cassandra_username" {}
 variable "cassandra_password" {}
 
@@ -19,6 +22,9 @@ module "cassandra_static" {
     "cassandra"
   ]
 
+  vault_token = var.vault_token
+  vault_addr  = var.vault_addr
+
   cassandra_path = var.cassandra_path
 
   cassandra_allowed_roles = ["*"]

examples/db_mongodb/main.tf

@@ -1,12 +1,16 @@
 provider "vault" {
-  address = "http://localhost:8200"
+  address = var.vault_addr
   token   = var.vault_token
 }
 
 variable "mongodb_path" {
   default = "mongodb"
 }
 
+variable "vault_addr" {
+  default = "http://localhost:8200"
+}
+
 variable "vault_token" {}
 variable "mongodb_user" {}
 variable "mongodb_password" {}
@@ -19,6 +23,9 @@ module "mongodb_static" {
     "mongodb"
   ]
 
+  vault_token = var.vault_token
+  vault_addr  = var.vault_addr
+
   mongodb_path = var.mongodb_path
 
   mongodb_allowed_roles       = ["*"]

examples/db_mysql/main.tf

@@ -1,12 +1,16 @@
 provider "vault" {
-  address = "http://localhost:8200"
+  address = var.vault_addr
   token   = var.vault_token
 }
 
 variable "mysql_path" {
   default = "mysql"
 }
 
+variable "vault_addr" {
+  default = "http://localhost:8200"
+}
+
 variable "vault_token" {}
 variable "mysql_user" {}
 variable "mysql_password" {}
@@ -19,6 +23,9 @@ module "mysql_static" {
     "mysql"
   ]
 
+  vault_token = var.vault_token
+  vault_addr  = var.vault_addr
+
   mysql_path = var.mysql_path
 
   mysql_allowed_roles       = ["*"]

examples/db_postgresql/main.tf

@@ -1,12 +1,16 @@
 provider "vault" {
-  address = "http://localhost:8200"
+  address = var.vault_addr
   token   = var.vault_token
 }
 
 variable "postgresql_path" {
   default = "postgresql"
 }
 
+variable "vault_addr" {
+  default = "http://localhost:8200"
+}
+
 variable "vault_token" {}
 variable "postgresql_username" {}
 variable "postgresql_password" {}
@@ -19,6 +23,9 @@ module "postgresql_static" {
     "postgresql"
   ]
 
+  vault_token = var.vault_token
+  vault_addr  = var.vault_addr
+
   postgresql_path = var.postgresql_path
 
   postgresql_allowed_roles  = ["*"]

main.tf

@@ -138,7 +138,7 @@ resource "vault_database_secret_backend_connection" "cassandra" {
   count   = contains(var.databases, "cassandra") ? 1 : 0
   backend = vault_mount.cassandra[count.index].path
 
-  name                     = "cassandra"
+  name                     = var.cassandra_path
   allowed_roles            = var.cassandra_allowed_roles
   root_rotation_statements = var.cassandra_root_rotation_statements
 
@@ -159,6 +159,15 @@ resource "vault_database_secret_backend_connection" "cassandra" {
 
   verify_connection = var.cassandra_verify_connection
 
+  data = {
+    addr  = var.vault_addr
+    token = var.vault_token
+  }
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
+  }
+
   depends_on = [
     vault_mount.cassandra
   ]
@@ -183,7 +192,7 @@ resource "vault_database_secret_backend_connection" "mongodb" {
   count   = contains(var.databases, "mongodb") ? 1 : 0
   backend = vault_mount.mongodb[count.index].path
 
-  name                     = "mongodb"
+  name                     = var.mongodb_path
   allowed_roles            = var.mongodb_allowed_roles
   root_rotation_statements = var.mongodb_root_rotation_statements
 
@@ -197,6 +206,13 @@ resource "vault_database_secret_backend_connection" "mongodb" {
   data = {
     username = var.mongodb_username
     password = var.mongodb_password
+    addr     = var.vault_addr
+    token    = var.vault_token
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
   }
 
   verify_connection = var.mongodb_verify_connection
@@ -221,7 +237,7 @@ resource "vault_database_secret_backend_connection" "hana" {
   count   = contains(var.databases, "hana") ? 1 : 0
   backend = vault_mount.hana[count.index].path
 
-  name                     = "hana"
+  name                     = var.hana_path
   allowed_roles            = var.hana_allowed_roles
   root_rotation_statements = var.hana_root_rotation_statements
 
@@ -231,6 +247,16 @@ resource "vault_database_secret_backend_connection" "hana" {
     max_idle_connections    = var.hana_max_idle_connections
     max_open_connections    = var.hana_max_open_connections
   }
+
+  data = {
+    addr  = var.vault_addr
+    token = var.vault_token
+  }
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
+  }
+
   verify_connection = var.hana_verify_connection
 }
 
@@ -253,7 +279,7 @@ resource "vault_database_secret_backend_connection" "mssql" {
   count   = contains(var.databases, "mssql") ? 1 : 0
   backend = vault_mount.mssql[count.index].path
 
-  name                     = "mssql"
+  name                     = var.mssql_path
   allowed_roles            = var.mssql_allowed_roles
   root_rotation_statements = var.mssql_root_rotation_statements
 
@@ -263,6 +289,16 @@ resource "vault_database_secret_backend_connection" "mssql" {
     max_idle_connections    = var.mssql_max_idle_connections
     max_open_connections    = var.mssql_max_open_connections
   }
+
+  data = {
+    addr  = var.vault_addr
+    token = var.vault_token
+  }
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
+  }
+
   verify_connection = var.mssql_verify_connection
 }
 
@@ -285,7 +321,7 @@ resource "vault_database_secret_backend_connection" "mysql" {
   count   = contains(var.databases, "mysql") ? 1 : 0
   backend = vault_mount.mysql[count.index].path
 
-  name                     = "mysql"
+  name                     = var.mysql_path
   allowed_roles            = var.mysql_allowed_roles
   root_rotation_statements = var.mysql_root_rotation_statements
 
@@ -300,6 +336,13 @@ resource "vault_database_secret_backend_connection" "mysql" {
   data = {
     username = var.mysql_username
     password = var.mysql_password
+    addr     = var.vault_addr
+    token    = var.vault_token
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
   }
 
   depends_on = [vault_mount.mysql]
@@ -324,7 +367,7 @@ resource "vault_database_secret_backend_connection" "postgresql" {
   count   = contains(var.databases, "postgresql") ? 1 : 0
   backend = vault_mount.postgresql[count.index].path
 
-  name                     = "postgresql"
+  name                     = var.postgresql_path
   allowed_roles            = var.postgresql_allowed_roles
   root_rotation_statements = var.postgresql_root_rotation_statements
 
@@ -339,6 +382,13 @@ resource "vault_database_secret_backend_connection" "postgresql" {
   data = {
     username = var.postgresql_username
     password = var.postgresql_password
+    addr     = var.vault_addr
+    token    = var.vault_token
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
   }
 
   depends_on = [
@@ -365,7 +415,7 @@ resource "vault_database_secret_backend_connection" "oracle" {
   count   = contains(var.databases, "oracle") ? 1 : 0
   backend = vault_mount.oracle[count.index].path
 
-  name                     = "oracle"
+  name                     = var.oracle_path
   allowed_roles            = var.oracle_allowed_roles
   root_rotation_statements = var.oracle_root_rotation_statements
 
@@ -376,6 +426,17 @@ resource "vault_database_secret_backend_connection" "oracle" {
     max_open_connections    = var.oracle_max_open_connections
   }
   verify_connection = var.oracle_verify_connection
+
+  data = {
+    addr  = var.vault_addr
+    token = var.vault_token
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
+  }
+
 }
 
 resource "vault_mount" "elasticsearch" {
@@ -397,7 +458,7 @@ resource "vault_database_secret_backend_connection" "elasticsearch" {
   count   = contains(var.databases, "elasticsearch") ? 1 : 0
   backend = vault_mount.elasticsearch[count.index].path
 
-  name                     = "elasticsearch"
+  name                     = var.elasticsearch_path
   allowed_roles            = var.elasticsearch_allowed_roles
   root_rotation_statements = var.elasticsearch_root_rotation_statements
 
@@ -407,8 +468,18 @@ resource "vault_database_secret_backend_connection" "elasticsearch" {
     password = var.elasticsearch_password
   }
   verify_connection = var.elasticsearch_verify_connection
-}
 
+  data = {
+    addr  = var.vault_addr
+    token = var.vault_token
+  }
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "${path.module}/revoke_lease.sh ${self.data.token} ${self.name} ${self.data.addr}"
+  }
+
+}
 
 resource "vault_database_secret_backend_role" "roles" {
   for_each = {

revoke_lease.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# This script will revoke all leases with the mount/creds prefix
+#
+# This is required due to the way Vault revokes
+# This module creates and manages 4 resources in this order:
+# 1 - Vault mount
+# 2 - Database connection config Which relies on a mount being in place
+# 3 - Vault dynamic role which relies on both the mount and connection being in place to work.
+# 4 - Vault static role which relies on both the mount and connection being in place to work.
+#
+# When terraform destroys these resources, it does so in the reverse order.
+# The Vault mount is responsible for revoking all leases when being destroyed.
+# For this to happen, the Mount relies on the connection config to be in place
+# This is so that Vault can connect to the database and clean up the accounts it has provisioned.
+# This prevents secrets sprawl but causes the destroy command to fail if there are leases.
+# Running this script as a local-exec destroy provisioner on all the database connection config resources in the module
+# prevents the destroy command from failing as all leases are revoked by the time the mount is being destroyed.
+#
+# The script requires:
+# 1 - Vault address
+# 2 - Vault token
+# 3 - Vault mount path
+#
+# Log in to Vault
+vault login -address="${3}" "$1" > /dev/null
+
+# List leases on the mount path and loop over the list with the revoke command until list is empty
+while [[ $(vault list -address="${3}" -format "json" sys/leases/lookup/"${2}"/creds) != "{}" ]]
+do
+    vault lease revoke -address="${3}" -prefix "${2}"/creds
+done

variables.tf

@@ -4,17 +4,17 @@ variable "secrets_engines" {
   description = "A list of secrets engines to enable"
 
   validation {
-    condition     = can(contains(
-    [
-      "aws",
-      "azure",
-      "gcp",
-      "consul",
-      "pki",
-      "transit",
-      "rabbitmq",
-      "ssh",
-      "db"
+    condition = can(contains(
+      [
+        "aws",
+        "azure",
+        "gcp",
+        "consul",
+        "pki",
+        "transit",
+        "rabbitmq",
+        "ssh",
+        "db"
     ], var.secrets_engines))
     error_message = "Invalid secrets engines."
   }
@@ -50,4 +50,16 @@ variable "external_entropy_access" {
   description = "Boolean flag that can be explicitly set to true to enable the secrets engine to access Vault's external entropy source"
 }
 
+variable "vault_token" {
+  type        = string
+  default     = null
+  description = "Vault token to use for authentication."
+}
+
+variable "vault_addr" {
+  type        = string
+  default     = "http://localhost:8200"
+  description = "Vault address."
+}
+