Merge pull request #121 from nimblehq/chore/104-improve-envs-secrets

[#104] Improve the way to manage environment variables and secrets

skeleton/aws/README.md

@@ -63,6 +63,82 @@ _Workspaces can be managed in the terraform cloud or using the CLI._
 
 > 💡 Other variables might change from `staging` to `prod`, such as the DB credentials. Consider reviewing all the available variables and their descriptions.
 
+### Step 4: Environment Variables and Secrets
+
+To provision a new environment variable, it needs to be configured in the Terraform workspace.
+
+> 💡 Editing the environment variables requires planning and applying changes in the Terraform project.
+
+### Non Sensitive Variable
+
+Non-sensitive variables do not require code changes in the `*-infra` project.
+
+Edit the variable named `environment_variables` directly in the Terraform workspace.
+This variable is an object and it can be extended just by editing its content and appending a new item to it.
+
+Example of the `environment_variables` object as displayed in Terraform:
+
+```
+[
+    {
+      name = "AVAILABLE_LOCALES"
+      value = "en,th"
+    },
+    {
+      name = "DEFAULT_LOCALE"
+      value = "th"
+    },
+   {
+     name = "FALLBACK_LOCALES"
+     value = "th"
+   }
+]
+```
+
+> ⚠️ A wrong indentation will break the object.
+> Make sure to carefully apply the right indent when editing this variable.
+
+### Sensitive Variable
+
+When a variable is set to sensitive, its content cannot be read by users once saved.
+So extending an object is not possible for sensitive variables — unless adding a lot of complexity.
+
+The below steps describe how to add a new sensitive environment variable with the name `MY_NEW_VAR`.
+
+First, edit the `*-infra` source code:
+- Declare a new variable in `base/variables.tf` with the name `my_new_var`
+- Edit the `base/main.tf` file, add the name of the variable under the `secrets` section in the `ssm` module:
+  ```terraform
+  module "ssm" {
+    source = "../modules/ssm"
+
+    namespace = var.namespace
+
+    secrets = {
+      secret_key_base = var.secret_key_base,
+      my_new_var      = var.my_new_var
+    }
+  }
+  ```
+
+Then add the variable in the Terraform workspace.
+The variable shall be marked as "sensitive" to ensure its value will not be available within logs.
+
+Once the variable is added and the code pushed, run a Terraform plan.
+The plan results should indicate about the creation of the new variable.
+Apply the plan if it ran successfully.
+
+The new variable `MY_NEW_VAR` will be available in the ECS task definition.
+
+### Update existing variables
+
+- To update an existing variable, edit the variable in the Terraform workspace:
+- If the variable is not sensitive, edit the `environment_variables` object.
+- If the variable is sensitive, edit the variable directly in the Terraform workspace.
+- Once the variable is updated, run a Terraform plan and apply it if it ran successfully.
+
+**Note:** Re-deploying the application is required when updating sensitive variables.
+
 ## License
 
 This project is Copyright (c) 2014 and onwards Nimble. It is free software and may be redistributed under the terms specified in the [LICENSE] file.

skeleton/aws/modules/ecs/main.tf

@@ -3,6 +3,14 @@ data "aws_ecr_repository" "repo" {
 }
 
 locals {
+  # Environment variables from other variables
+  environment_variables = toset([
+    {
+      name  = "AWS_REGION"
+      value = var.region
+    }
+  ])
+
   container_vars = {
     namespace                          = var.namespace
     region                             = var.region
@@ -15,9 +23,12 @@ locals {
     aws_ecr_repository                 = data.aws_ecr_repository.repo.repository_url
     aws_ecr_tag                        = var.ecr_tag
     aws_cloudwatch_log_group_name      = var.aws_cloudwatch_log_group_name
+
+    environment_variables = setunion(local.environment_variables, var.environment_variables)
+    secrets_variables     = var.secrets_variables
   }
 
-  container_definitions = templatefile("${path.module}/service.json.tftpl", merge(local.container_vars, var.aws_parameter_store))
+  container_definitions = templatefile("${path.module}/service.json.tftpl", local.container_vars)
 
   ecs_task_execution_ssm_policy = {
     Version = "2012-10-17",
@@ -27,7 +38,7 @@ locals {
         Action = [
           "ssm:GetParameters"
         ],
-        Resource = "*"
+        Resource = var.secrets_arns
       }
     ]
   }

skeleton/aws/modules/ecs/service.json.tftpl

@@ -20,10 +20,8 @@
         "awslogs-group": "${aws_cloudwatch_log_group_name}"
       }
     },
-    "environment": [
-    ],
-    "secrets": [
-    ],
+    "environment": ${jsonencode(environment_variables)},
+    "secrets": ${jsonencode(secrets_variables)},
     "ulimits": [
       {
         "name": "nofile",

skeleton/aws/modules/ecs/variables.tf

@@ -77,7 +77,20 @@ variable "aws_cloudwatch_log_group_name" {
   type        = string
 }
 
-variable "aws_parameter_store" {
-  description = "AWS parameter store"
-  type        = map(any)
+variable "environment_variables" {
+  description = "List of [{name = \"\", value = \"\"}] pairs of environment variables"
+  type = set(object({
+    name  = string
+    value = string
+  }))
+}
+
+variable "secrets_variables" {
+  description = "List of [{name = \"\", valueFrom = \"\"}] pairs of secret variables"
+  type        = list(any)
+}
+
+variable "secrets_arns" {
+  description = "The ARNs of the SSM Parameter Store parameters"
+  type        = list(string)
 }

skeleton/aws/modules/ssm/main.tf

@@ -1,11 +1,23 @@
-resource "aws_ssm_parameter" "secret_key_base" {
-  name  = "/${var.namespace}/SECRET_KEY_BASE"
+resource "aws_ssm_parameter" "secret_parameters" {
+  for_each = var.secrets
+
+  name  = "/${var.namespace}/${each.key}"
   type  = "String"
-  value = var.secret_key_base
+  value = each.value
 }
 
-resource "aws_ssm_parameter" "database_url" {
-  name  = "/${var.namespace}/DATABASE_URL"
-  type  = "String"
-  value = "postgresql://${var.rds_username}:${var.rds_password}@${var.rds_endpoint}/${var.rds_database_name}"
+locals {
+  # Create a list of parameter store ARNs for granting access to ECS task execution role
+  parameter_store_arns = [for parameter in aws_ssm_parameter.secret_parameters : parameter.arn]
+
+  # Get secret names array
+  secret_names = keys(var.secrets)
+
+  # Create a map {secret_name: secret_arn} using zipmap function for iteration
+  secret_arns = zipmap(local.secret_names, local.parameter_store_arns)
+
+  # Create the formatted secrets for ECS task definition
+  secrets_variables = [for secret_key, secret_arn in local.secrets_name_arn_map :
+    tomap({ "name" = upper(secret_key), "valueFrom" = secret_arn })
+  ]
 }

skeleton/aws/modules/ssm/outputs.tf

@@ -1,8 +1,9 @@
-output "parameter_store" {
-  description = "ARNs of the parameters"
+output "secrets_variables" {
+  description = "The formatted secrets for ECS task definition"
+  value       = local.secrets_variables
+}
 
-  value = {
-    secret_base_ssm_arn  = aws_ssm_parameter.secret_key_base.arn
-    database_url_ssm_arn = aws_ssm_parameter.database_url.arn
-  }
+output "parameter_store_arns" {
+  description = "List of parameter store ARNs for granting access to ECS task execution role"
+  value       = local.parameter_store_arns
 }

skeleton/aws/modules/ssm/variables.tf

@@ -3,27 +3,8 @@ variable "namespace" {
   type        = string
 }
 
-variable "secret_key_base" {
-  description = "The Secret key base for the application"
-  type        = string
-}
-
-variable "rds_username" {
-  description = "The DB username for building DB URL"
-  type        = string
-}
-
-variable "rds_password" {
-  description = "The DB password for building DB URL"
-  type        = string
-}
-
-variable "rds_endpoint" {
-  description = "The DB endpoint for building DB URL"
-  type        = string
-}
-
-variable "rds_database_name" {
-  description = "The DB name for building DB URL"
-  type        = string
+variable "secrets" {
+  description = "Map of secrets to keep in AWS SSM Parameter Store"
+  type        = map(string)
+  default     = {}
 }

src/templates/aws/addons/ecs.ts

@@ -25,7 +25,38 @@ const ecsVariablesContent = dedent`
       deployment_minimum_healthy_percent = number
     })
   }
+
+  variable "environment_variables" {
+    description = "List of [{name = \"\", value = \"\"}] pairs of environment variables"
+    type = set(object({
+      name  = string
+      value = string
+    }))
+    default = [
+      {
+        name  = "AVAILABLE_LOCALES"
+        value = "en"
+      },
+      {
+        name  = "DEFAULT_LOCALE"
+        value = "en"
+      },
+      {
+        name  = "FALLBACK_LOCALES"
+        value = "en"
+      },
+      {
+        name  = "MAILER_DEFAULT_HOST"
+        value = "localhost"
+      },
+      {
+        name  = "MAILER_DEFAULT_PORT"
+        value = "80"
+      },
+    ]
+  }
 \n`;
+
 const ecsModuleContent = dedent`
   module "ecs" {
     source = "./modules/ecs"
@@ -47,7 +78,9 @@ const ecsModuleContent = dedent`
     deployment_minimum_healthy_percent = var.ecs.deployment_minimum_healthy_percent
     container_memory                   = var.ecs.task_container_memory
 
-    aws_parameter_store = module.ssm.parameter_store
+    environment_variables = var.environment_variables
+    secrets_variables     = module.ssm.secrets_variables
+    secrets_arns          = module.ssm.parameter_store_arns
   }
 \n`;
 

src/templates/aws/addons/ssm.ts

@@ -14,12 +14,11 @@ const ssmModuleContent = dedent`
     source = "./modules/ssm"
 
     namespace = var.namespace
-    secret_key_base       = var.secret_key_base
 
-    rds_username      = var.rds_username
-    rds_password      = var.rds_password
-    rds_database_name = var.rds_database_name
-    rds_endpoint      = module.rds.db_endpoint
+    secrets = {
+      database_url = "postgres://\${var.rds_username}:\${var.rds_password}@\${module.rds.db_endpoint}/\${var.rds_database_name}"
+      secret_key_base = var.secret_key_base
+    }
   }
 \n`;