From 8190a82b010d043cde80e953dbda676eb14a4eb5 Mon Sep 17 00:00:00 2001 From: veerapratapg <57515481+veerapratapg@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:58:30 -0600 Subject: [PATCH 1/8] test commit 3 --- tempfile1.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 tempfile1.txt diff --git a/tempfile1.txt b/tempfile1.txt new file mode 100644 index 00000000..ce013625 --- /dev/null +++ b/tempfile1.txt @@ -0,0 +1 @@ +hello From 53a7ed30a04f4d6d9c4c9edcd74cce2a40862eab Mon Sep 17 00:00:00 2001 From: veerapratapg <57515481+veerapratapg@users.noreply.github.com> Date: Wed, 17 Jan 2024 13:11:32 -0600 Subject: [PATCH 2/8] removed test file tempfile1.txt --- tempfile1.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tempfile1.txt diff --git a/tempfile1.txt b/tempfile1.txt deleted file mode 100644 index ce013625..00000000 --- a/tempfile1.txt +++ /dev/null @@ -1 +0,0 @@ -hello From 83aa0bcc8d43fe66e7d8610513e104117b7c42e9 Mon Sep 17 00:00:00 2001 From: veerapratapg <57515481+veerapratapg@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:23:22 -0600 Subject: [PATCH 3/8] Added a module to enable Grafana API Key rotation and made appropriate changes to the eks-monitoring module and examples/existing-cluster-with-base-and-infra directory --- .../main.tf | 29 ++- .../outputs.tf | 27 +++ .../variables.tf | 24 +++ .../add-ons/external-secrets/main.tf | 4 +- .../add-ons/external-secrets/outputs.tf | 14 ++ .../add-ons/external-secrets/variables.tf | 10 + modules/eks-monitoring/main.tf | 12 +- modules/eks-monitoring/outputs.tf | 15 ++ modules/eks-monitoring/variables.tf | 10 + .../grafana-key-rotation/lambda_function.py | 85 ++++++++ modules/grafana-key-rotation/main.tf | 184 ++++++++++++++++++ modules/grafana-key-rotation/outputs.tf | 20 ++ modules/grafana-key-rotation/variables.tf | 65 +++++++ 13 files changed, 487 insertions(+), 12 deletions(-) create mode 100644 modules/grafana-key-rotation/lambda_function.py create mode 100644 modules/grafana-key-rotation/main.tf create mode 100644 modules/grafana-key-rotation/outputs.tf create mode 100644 modules/grafana-key-rotation/variables.tf diff --git a/examples/existing-cluster-with-base-and-infra/main.tf b/examples/existing-cluster-with-base-and-infra/main.tf index eccd94a1..606b2259 100644 --- a/examples/existing-cluster-with-base-and-infra/main.tf +++ b/examples/existing-cluster-with-base-and-infra/main.tf @@ -71,11 +71,13 @@ module "eks_monitoring" { enable_apiserver_monitoring = true # deploys external-secrets in to the cluster - enable_external_secrets = true - grafana_api_key = var.grafana_api_key - target_secret_name = "grafana-admin-credentials" - target_secret_namespace = "grafana-operator" - grafana_url = module.aws_observability_accelerator.managed_grafana_workspace_endpoint + enable_external_secrets = true + grafana_api_key = var.grafana_api_key + target_secret_name = "grafana-admin-credentials" + target_secret_namespace = "grafana-operator" + grafana_url = module.aws_observability_accelerator.managed_grafana_workspace_endpoint + grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval + managed_grafana_workspace_id = var.managed_grafana_workspace_id # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true' enable_dashboards = var.enable_dashboards @@ -99,3 +101,20 @@ module "eks_monitoring" { module.aws_observability_accelerator ] } + +# Enabling Grafana API Key Rotation +module "grafana_key_rotation" { + source = "../../modules/grafana-key-rotation" + # source = "github.com/aws-observability/terraform-aws-observability-accelerator//modules/eks-key-rotation" + + count = var.enable_grafana_key_rotation ? 1 : 0 + + managed_grafana_workspace_id = var.managed_grafana_workspace_id + grafana_api_key_interval = var.grafana_api_key_interval + eventbridge_scheduler_schedule_expression = var.eventbridge_scheduler_schedule_expression + + ssmparameter_name = module.eks_monitoring.ssmparameter_name_eks_monitoring + ssmparameter_arn = module.eks_monitoring.ssmparameter_arn_eks_monitoring + kms_key_arn_ssm = module.eks_monitoring.kms_key_arn_eks_monitoring + +} diff --git a/examples/existing-cluster-with-base-and-infra/outputs.tf b/examples/existing-cluster-with-base-and-infra/outputs.tf index f8b4d584..f6d4821f 100644 --- a/examples/existing-cluster-with-base-and-infra/outputs.tf +++ b/examples/existing-cluster-with-base-and-infra/outputs.tf @@ -12,3 +12,30 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = module.eks_monitoring.eks_cluster_id } + +output "eks_key_rotation_lambda_function_arn" { + description = "ARN of the Lambda function performing Key rotation" + # value = module.grafana_key_rotation.lambda_function_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].lambda_function_arn : null +} + + +output "eks_key_rotation_lambda_function_role_arn" { + description = "ARN of the Lambda function execution role" + # value = module.grafana_key_rotation.lambda_function_role_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].lambda_function_role_arn : null +} + + +output "eks_key_rotation_eventbridge_scheduler_arn" { + description = "ARN of the EventBridge Scheduler invoking Lambda Function for Key rotation" + # value = module.grafana_key_rotation.eventbridge_scheduler_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].eventbridge_scheduler_arn : null +} + + +output "eks_key_rotation_eventbridge_scheduler_role_arn" { + description = "ARN of the IAM Role of EventBridge Scheduler invoking Lambda Function for Key rotation" + # value = module.grafana_key_rotation.eventbridge_scheduler_role_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].eventbridge_scheduler_role_arn : null +} diff --git a/examples/existing-cluster-with-base-and-infra/variables.tf b/examples/existing-cluster-with-base-and-infra/variables.tf index 6df0e2e5..679fe346 100644 --- a/examples/existing-cluster-with-base-and-infra/variables.tf +++ b/examples/existing-cluster-with-base-and-infra/variables.tf @@ -31,3 +31,27 @@ variable "enable_dashboards" { type = bool default = true } + +variable "enable_grafana_key_rotation" { + description = "Enables or disables Grafana API key rotation" + type = bool + default = true +} + +variable "grafana_api_key_interval" { + description = "Number of seconds for secondsToLive value while creating API Key" + type = number + default = 5400 +} + +variable "eventbridge_scheduler_schedule_expression" { + description = "Schedule Expression for EventBridge Scheduler in Grafana API Key Rotation" + type = string + default = "rate(60 minutes)" +} + +variable "grafana_api_key_refresh_interval" { + description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" + type = string + default = "5m" +} diff --git a/modules/eks-monitoring/add-ons/external-secrets/main.tf b/modules/eks-monitoring/add-ons/external-secrets/main.tf index dfd3669b..e1bd7aaf 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/main.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/main.tf @@ -77,7 +77,7 @@ YAML } resource "aws_ssm_parameter" "secret" { - name = "/terraform-accelerator/grafana-api-key" + name = join("", ["/terraform-accelerator/grafana-api-key-", var.managed_grafana_workspace_id]) description = "SSM Secret to store grafana API Key" type = "SecureString" value = jsonencode({ @@ -94,7 +94,7 @@ metadata: name: ${local.name}-sm namespace: ${var.target_secret_namespace} spec: - refreshInterval: 1h + refreshInterval: ${var.grafana_api_key_refresh_interval} secretStoreRef: name: ${local.cluster_secretstore_name} kind: ClusterSecretStore diff --git a/modules/eks-monitoring/add-ons/external-secrets/outputs.tf b/modules/eks-monitoring/add-ons/external-secrets/outputs.tf index e69de29b..d8979a55 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/outputs.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/outputs.tf @@ -0,0 +1,14 @@ +output "ssmparameter_name" { + description = "Name of the SSM Parameter" + value = aws_ssm_parameter.secret.name +} + +output "ssmparameter_arn" { + description = "Name of the SSM Parameter" + value = aws_ssm_parameter.secret.arn +} + +output "kms_key_arn_ssm" { + description = "Name of the SSM Parameter" + value = aws_kms_key.secrets.arn +} diff --git a/modules/eks-monitoring/add-ons/external-secrets/variables.tf b/modules/eks-monitoring/add-ons/external-secrets/variables.tf index c9aa965a..ec99490b 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/variables.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/variables.tf @@ -41,3 +41,13 @@ variable "target_secret_name" { description = "Name to store the secret for Grafana API Key" type = string } + +variable "grafana_api_key_refresh_interval" { + description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" + type = string +} + +variable "managed_grafana_workspace_id" { + description = "Amazon Managed Grafana Workspace ID" + type = string +} diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 0d6f201e..a834a492 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -250,11 +250,13 @@ module "external_secrets" { source = "./add-ons/external-secrets" count = var.enable_external_secrets ? 1 : 0 - enable_external_secrets = var.enable_external_secrets - grafana_api_key = var.grafana_api_key - addon_context = local.context - target_secret_namespace = var.target_secret_namespace - target_secret_name = var.target_secret_name + enable_external_secrets = var.enable_external_secrets + grafana_api_key = var.grafana_api_key + addon_context = local.context + target_secret_namespace = var.target_secret_namespace + target_secret_name = var.target_secret_name + grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval + managed_grafana_workspace_id = var.managed_grafana_workspace_id depends_on = [resource.helm_release.grafana_operator] } diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index c85d3cce..d6b9b7b5 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -12,3 +12,18 @@ output "adot_irsa_arn" { description = "IRSA Arn for ADOT" value = module.helm_addon.irsa_arn } + +output "ssmparameter_name_eks_monitoring" { + description = "Name of the SSM Parameter" + value = module.external_secrets[0].ssmparameter_name +} + +output "ssmparameter_arn_eks_monitoring" { + description = "Name of the SSM Parameter" + value = module.external_secrets[0].ssmparameter_arn +} + +output "kms_key_arn_eks_monitoring" { + description = "Name of the SSM Parameter" + value = module.external_secrets[0].kms_key_arn_ssm +} diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index cb958785..954c2f50 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -575,3 +575,13 @@ variable "kubeproxy_monitoring_config" { # defaults are pre-computed in locals.tf, provide a full definition to override default = null } + +variable "grafana_api_key_refresh_interval" { + description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" + type = string +} + +variable "managed_grafana_workspace_id" { + description = "Amazon Managed Grafana Workspace ID" + type = string +} diff --git a/modules/grafana-key-rotation/lambda_function.py b/modules/grafana-key-rotation/lambda_function.py new file mode 100644 index 00000000..b564db02 --- /dev/null +++ b/modules/grafana-key-rotation/lambda_function.py @@ -0,0 +1,85 @@ +import boto3 +import os +import logging +from datetime import datetime, timedelta + +# Initializing the AWS SDK clients +ssm_client = boto3.client('ssm') +grafana_client = boto3.client('grafana', region_name=os.environ['AWS_REGION']) + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +def lambda_handler(event, context): + try: + # Parse input arguments from the request event + ssm_parameter_name = event['ssmparameter'] + interval_value = int(event['interval']) + workspace_id = event['workspaceid'] + + # Check if the SSM Parameter exists + response = ssm_client.get_parameter(Name=ssm_parameter_name) + parameter_version = response['Parameter']['Version'] + logger.info(f'SSM Parameter exists. Current Parameter Version is : {parameter_version}') + + # Timestamp to be used for unique KeyName while creating Grafana Workspace API Key + now = datetime.now() + + # Create a new API key in Amazon Managed Grafana + response = grafana_client.create_workspace_api_key( + workspaceId=workspace_id, + keyRole='ADMIN', + keyName='observability-accelerator-'+now.strftime("%Y%m%d-%H"), + secondsToLive=interval_value + ) + new_api_key = '{\"GF_SECURITY_ADMIN_APIKEY\":\"'+response['key']+'\"}' + new_api_key_name = response['keyName'] + logger.info(f'New API Key Name : {new_api_key_name}') + + # Updating SSM Parameter value with new Key + response = ssm_client.put_parameter( + Name=ssm_parameter_name, + Value=new_api_key, + Type='SecureString', + Overwrite=True + ) + new_parameter_version = response['Version'] + logger.info(f'API Key updated in SSM parameter successfully. New Parameter Version is : {new_parameter_version}') + + + # Deleting hour-2 API key to deal with the limit of 100 keys per WorkSpace + logger.info(f'Proceeding to delete older API Key') + last_hour_date_time = now - timedelta(hours = 2) + old_key_name = 'observability-accelerator-'+last_hour_date_time.strftime('%Y%m%d-%H') + logger.info(f'Deleting key : {old_key_name}') + grafana_client.delete_workspace_api_key( + keyName=old_key_name, + workspaceId=workspace_id + ) + logger.info(f'Deleted the API Key with name : {old_key_name}') + + return { + 'statusCode': 200, + 'body': 'API key updated successfully and older key has been deleted' + } + + except ssm_client.exceptions.ParameterNotFound: + logger.error('SSM Parameter does not exist') + return { + 'statusCode': 404, + 'body': 'SSM Parameter does not exist' + } + + except grafana_client.exceptions.ResourceNotFoundException: + logger.error('Older API Key does not exist') + return { + 'statusCode': 200, + 'body': 'New API Key added to SSM Parameter value.' + } + + except Exception as e: + logger.error(f'An error occurred: {str(e)}') + return { + 'statusCode': 500, + 'body': 'Lambda execution failed. Please check CloudWatch Logs for additional information on error encountered.' + } \ No newline at end of file diff --git a/modules/grafana-key-rotation/main.tf b/modules/grafana-key-rotation/main.tf new file mode 100644 index 00000000..7ad20598 --- /dev/null +++ b/modules/grafana-key-rotation/main.tf @@ -0,0 +1,184 @@ +data "archive_file" "lambda_function_archive" { + type = "zip" + + source_file = "${path.module}/lambda_function.py" + output_path = "${path.module}/lambda_function.zip" +} + + +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} + +# Unique random string to avoid Resource Already Exists errors. +resource "random_string" "random_string_resource" { + length = 4 + special = false + lower = true + upper = false +} + +# Lambda function resource +resource "aws_lambda_function" "observability_accelerator_lambda" { + function_name = "${var.lambda_function_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" + handler = "lambda_function.lambda_handler" + runtime = "python3.8" + memory_size = 128 + timeout = 180 + filename = data.archive_file.lambda_function_archive.output_path + role = aws_iam_role.lambda_role.arn +} + + +# Lambda Execution IAM Role +resource "aws_iam_role" "lambda_role" { + name = "${var.lambda_execution_role_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" + assume_role_policy = < Date: Tue, 30 Jan 2024 21:24:13 -0600 Subject: [PATCH 4/8] Made the changes requested by @lewinkedrs.\n Additional information can be found in the comments of the Pull Request. --- .../README.md | 10 +++ .../main.tf | 6 +- .../variables.tf | 6 ++ modules/eks-monitoring/README.md | 5 ++ .../add-ons/external-secrets/README.md | 8 +- modules/eks-monitoring/variables.tf | 1 + modules/grafana-key-rotation/README.md | 42 ++++++++++ .../grafana-key-rotation/lambda_function.py | 18 ++--- modules/grafana-key-rotation/main.tf | 81 +++++++++---------- modules/grafana-key-rotation/outputs.tf | 2 +- modules/grafana-key-rotation/variables.tf | 12 ++- 11 files changed, 130 insertions(+), 61 deletions(-) create mode 100644 modules/grafana-key-rotation/README.md diff --git a/examples/existing-cluster-with-base-and-infra/README.md b/examples/existing-cluster-with-base-and-infra/README.md index 3b06c870..90578aa3 100644 --- a/examples/existing-cluster-with-base-and-infra/README.md +++ b/examples/existing-cluster-with-base-and-infra/README.md @@ -37,6 +37,7 @@ View the full documentation for this example [here](https://aws-observability.gi |------|--------|---------| | [aws\_observability\_accelerator](#module\_aws\_observability\_accelerator) | ../../ | n/a | | [eks\_monitoring](#module\_eks\_monitoring) | ../../modules/eks-monitoring | n/a | +| [grafana\_key\_rotation](#module\_grafana\_key\_rotation) | ../../modules/grafana-key-rotation | n/a | ## Resources @@ -52,7 +53,12 @@ View the full documentation for this example [here](https://aws-observability.gi | [aws\_region](#input\_aws\_region) | AWS Region | `string` | n/a | yes | | [eks\_cluster\_id](#input\_eks\_cluster\_id) | Name of the EKS cluster | `string` | `"eks-cluster-with-vpc"` | no | | [enable\_dashboards](#input\_enable\_dashboards) | Enables or disables curated dashboards. Dashboards are managed by the Grafana Operator | `bool` | `true` | no | +| [enable\_grafana\_key\_rotation](#input\_enable\_grafana\_key\_rotation) | Enables or disables Grafana API key rotation | `bool` | `true` | no | +| [eventbridge\_scheduler\_schedule\_expression](#input\_eventbridge\_scheduler\_schedule\_expression) | Schedule Expression for EventBridge Scheduler in Grafana API Key Rotation | `string` | `"rate(60 minutes)"` | no | | [grafana\_api\_key](#input\_grafana\_api\_key) | API key for authorizing the Grafana provider to make changes to Amazon Managed Grafana | `string` | n/a | yes | +| [grafana\_api\_key\_interval](#input\_grafana\_api\_key\_interval) | Number of seconds for secondsToLive value while creating API Key | `number` | `5400` | no | +| [grafana\_api\_key\_refresh\_interval](#input\_grafana\_api\_key\_refresh\_interval) | Refresh Internal to be used by External Secrets for Grafana API Key rotation | `string` | `"5m"` | no | +| [lambda\_runtime\_grafana\_key\_rotation](#input\_lambda\_runtime\_grafana\_key\_rotation) | Python Runtime Identifier for the Lambda Function | `string` | `"python3.12"` | no | | [managed\_grafana\_workspace\_id](#input\_managed\_grafana\_workspace\_id) | Amazon Managed Grafana Workspace ID | `string` | n/a | yes | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Service for Prometheus Workspace ID | `string` | `""` | no | @@ -63,4 +69,8 @@ View the full documentation for this example [here](https://aws-observability.gi | [aws\_region](#output\_aws\_region) | AWS Region | | [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id | | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | +| [eks\_key\_rotation\_eventbridge\_scheduler\_arn](#output\_eks\_key\_rotation\_eventbridge\_scheduler\_arn) | ARN of the EventBridge Scheduler invoking Lambda Function for Key rotation | +| [eks\_key\_rotation\_eventbridge\_scheduler\_role\_arn](#output\_eks\_key\_rotation\_eventbridge\_scheduler\_role\_arn) | ARN of the IAM Role of EventBridge Scheduler invoking Lambda Function for Key rotation | +| [eks\_key\_rotation\_lambda\_function\_arn](#output\_eks\_key\_rotation\_lambda\_function\_arn) | ARN of the Lambda function performing Key rotation | +| [eks\_key\_rotation\_lambda\_function\_role\_arn](#output\_eks\_key\_rotation\_lambda\_function\_role\_arn) | ARN of the Lambda function execution role | diff --git a/examples/existing-cluster-with-base-and-infra/main.tf b/examples/existing-cluster-with-base-and-infra/main.tf index 606b2259..eff35a6f 100644 --- a/examples/existing-cluster-with-base-and-infra/main.tf +++ b/examples/existing-cluster-with-base-and-infra/main.tf @@ -105,16 +105,14 @@ module "eks_monitoring" { # Enabling Grafana API Key Rotation module "grafana_key_rotation" { source = "../../modules/grafana-key-rotation" - # source = "github.com/aws-observability/terraform-aws-observability-accelerator//modules/eks-key-rotation" - - count = var.enable_grafana_key_rotation ? 1 : 0 + count = var.enable_grafana_key_rotation ? 1 : 0 managed_grafana_workspace_id = var.managed_grafana_workspace_id grafana_api_key_interval = var.grafana_api_key_interval eventbridge_scheduler_schedule_expression = var.eventbridge_scheduler_schedule_expression + lambda_runtime_grafana_key_rotation = var.lambda_runtime_grafana_key_rotation ssmparameter_name = module.eks_monitoring.ssmparameter_name_eks_monitoring ssmparameter_arn = module.eks_monitoring.ssmparameter_arn_eks_monitoring kms_key_arn_ssm = module.eks_monitoring.kms_key_arn_eks_monitoring - } diff --git a/examples/existing-cluster-with-base-and-infra/variables.tf b/examples/existing-cluster-with-base-and-infra/variables.tf index 679fe346..a976c8d7 100644 --- a/examples/existing-cluster-with-base-and-infra/variables.tf +++ b/examples/existing-cluster-with-base-and-infra/variables.tf @@ -55,3 +55,9 @@ variable "grafana_api_key_refresh_interval" { type = string default = "5m" } + +variable "lambda_runtime_grafana_key_rotation" { + description = "Python Runtime Identifier for the Lambda Function" + type = string + default = "python3.12" +} diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 7e45b875..9669105b 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -100,6 +100,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [flux\_kustomization\_path](#input\_flux\_kustomization\_path) | Flux Kustomization Path | `string` | `"./artifacts/grafana-operator-manifests/eks/infrastructure"` | no | | [go\_config](#input\_go\_config) | Grafana Operator configuration | 
object({
    create_namespace   = bool
    helm_chart         = string
    helm_name          = string
    k8s_namespace      = string
    helm_release_name  = string
    helm_chart_version = string
  })
| 
{
  "create_namespace": true,
  "helm_chart": "oci://ghcr.io/grafana-operator/helm-charts/grafana-operator",
  "helm_chart_version": "v5.5.2",
  "helm_name": "grafana-operator",
  "helm_release_name": "grafana-operator",
  "k8s_namespace": "grafana-operator"
}
| no | | [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace. Required if `enable_external_secrets = true` | `string` | `""` | no | +| [grafana\_api\_key\_refresh\_interval](#input\_grafana\_api\_key\_refresh\_interval) | Refresh Internal to be used by External Secrets for Grafana API Key rotation | `string` | `"5m"` | no | | [grafana\_cluster\_dashboard\_url](#input\_grafana\_cluster\_dashboard\_url) | Dashboard URL for Cluster Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/cluster.json"` | no | | [grafana\_kubelet\_dashboard\_url](#input\_grafana\_kubelet\_dashboard\_url) | Dashboard URL for Kubelet Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/infrastructure/kubelet.json"` | no | | [grafana\_kubeproxy\_dashboard\_url](#input\_grafana\_kubeproxy\_dashboard\_url) | Dashboard URL for kube-proxy Grafana Dashboard JSON | `string` | `"https://raw.githubusercontent.com/aws-observability/aws-observability-accelerator/v0.2.0/artifacts/grafana-dashboards/eks/kube-proxy/kube-proxy.json"` | no | @@ -118,6 +119,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [ksm\_config](#input\_ksm\_config) | Kube State metrics configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "kube-state-metrics",
  "helm_chart_version": "5.15.2",
  "helm_release_name": "kube-state-metrics",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "kube-system",
  "scrape_interval": "60s",
  "scrape_timeout": "15s"
}
| no | | [kubeproxy\_monitoring\_config](#input\_kubeproxy\_monitoring\_config) | Config object for kube-proxy monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      default = string
    })
  })
| `null` | no | | [logs\_config](#input\_logs\_config) | Configuration object for logs collection | 
object({
    cw_log_retention_days = number
  })
| 
{
  "cw_log_retention_days": 90
}
| no | +| [managed\_grafana\_workspace\_id](#input\_managed\_grafana\_workspace\_id) | Amazon Managed Grafana Workspace ID | `string` | n/a | yes | | [managed\_prometheus\_cross\_account\_role](#input\_managed\_prometheus\_cross\_account\_role) | Amazon Managed Prometheus Workspace's Account Role Arn | `string` | `""` | no | | [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | @@ -137,4 +139,7 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [adot\_irsa\_arn](#output\_adot\_irsa\_arn) | IRSA Arn for ADOT | | [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id | | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | +| [kms\_key\_arn\_eks\_monitoring](#output\_kms\_key\_arn\_eks\_monitoring) | Name of the SSM Parameter | +| [ssmparameter\_arn\_eks\_monitoring](#output\_ssmparameter\_arn\_eks\_monitoring) | Name of the SSM Parameter | +| [ssmparameter\_name\_eks\_monitoring](#output\_ssmparameter\_name\_eks\_monitoring) | Name of the SSM Parameter | diff --git a/modules/eks-monitoring/add-ons/external-secrets/README.md b/modules/eks-monitoring/add-ons/external-secrets/README.md index 62d7a5cd..cefe85d2 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/README.md +++ b/modules/eks-monitoring/add-ons/external-secrets/README.md @@ -44,11 +44,17 @@ This deploys an EKS Cluster with the External Secrets Operator. The cluster is p | [addon\_context](#input\_addon\_context) | Input configuration for the addon | 
object({
    aws_caller_identity_account_id = string
    aws_caller_identity_arn        = string
    aws_eks_cluster_endpoint       = string
    aws_partition_id               = string
    aws_region_name                = string
    eks_cluster_id                 = string
    eks_oidc_issuer_url            = string
    eks_oidc_provider_arn          = string
    irsa_iam_role_path             = string
    irsa_iam_permissions_boundary  = string
    tags                           = map(string)
  })
| n/a | yes | | [enable\_external\_secrets](#input\_enable\_external\_secrets) | Enable external-secrets | `bool` | `true` | no | | [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace | `string` | n/a | yes | +| [grafana\_api\_key\_refresh\_interval](#input\_grafana\_api\_key\_refresh\_interval) | Refresh Internal to be used by External Secrets for Grafana API Key rotation | `string` | n/a | yes | | [helm\_config](#input\_helm\_config) | Helm provider config for external secrets | `any` | `{}` | no | +| [managed\_grafana\_workspace\_id](#input\_managed\_grafana\_workspace\_id) | Amazon Managed Grafana Workspace ID | `string` | n/a | yes | | [target\_secret\_name](#input\_target\_secret\_name) | Name to store the secret for Grafana API Key | `string` | n/a | yes | | [target\_secret\_namespace](#input\_target\_secret\_namespace) | Namespace to store the secret for Grafana API Key | `string` | n/a | yes | ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [kms\_key\_arn\_ssm](#output\_kms\_key\_arn\_ssm) | Name of the SSM Parameter | +| [ssmparameter\_arn](#output\_ssmparameter\_arn) | Name of the SSM Parameter | +| [ssmparameter\_name](#output\_ssmparameter\_name) | Name of the SSM Parameter | diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index 954c2f50..8fe382f2 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -579,6 +579,7 @@ variable "kubeproxy_monitoring_config" { variable "grafana_api_key_refresh_interval" { description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" type = string + default = "5m" } variable "managed_grafana_workspace_id" { diff --git a/modules/grafana-key-rotation/README.md b/modules/grafana-key-rotation/README.md new file mode 100644 index 00000000..9b38b4bc --- /dev/null +++ b/modules/grafana-key-rotation/README.md @@ -0,0 +1,42 @@ +# API Key Rotation for Amazon Managed Grafana Workspace + +- This module automates the rotation of Amazon Managed Grafana Workspace API Keys. +- When created, it generates an API Key for Amazon Managed Grafana Workspace on a schedule and updates the SSM parameter value to use this new key; this would then be consumed by the External Secrets Operator in `eks-monitoring` module deployed to EKS Cluster which retrieves and Syncs the Grafana API keys from AWS SSM Parameter Store. +- This module/feature is enabled by default and can be disabled setting the value of `enable_grafana_key_rotation` variable to `false` in the `variables.tf` file of `existing-cluster-with-base-and-infra` example. + +## Resources created through the module +The following AWS resources are created through this module to implement API key rotation : +- Lambda function + - To create a new Grafana API Key + - Update the SSM Parameter Value + - Delete older API key. +- Lambda Execution IAM Role + - To provide permissions related to grafana, SSM and CloudWatch logs to Lambda function. + - The permissions of the role are restricted to the specific Grafana workspace and SSM parameter that are in scope of this solution. +- EventBridge Scheduler + - To invoke the Lambda function on a cron-based schedule. +- EventBridge Scheduler IAM Role + - To provide permissions to the EventBridge Schedule to invoke the Lambda function. + - The permissions of the role are restricted to the specific Lambda function created in this solution. + + +## Configuration Options + +### Through the `grafana-key-rotation` module +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| lambda_function_name | Name of the Lambda function that creates the API Key and updates the SSM Parameter | `string` | `observability-accelerator-lambda` | No | +| lambda_execution_role_name | Name of Lambda Execution IAM Role | `string` | "`observability-accelerator-lambdaRole`" | No | +| lambda_execution_role_policy_name | Name of the Lambda Execution Role Policy | `string` | "`observability-accelerator-lambda-Policy`" | No | +| eventbridge_scheduler_name | Name of the EventBridge Scheduler that triggers the Lambda function | `string` | "`observability-accelerator-EBridge`" | No | +| eventbridge_scheduler_role_name | Name of the IAM role for EventBridge | `string` | "`observability-accelerator-EBridgeRole`" | No | +| eventbridge_scheduler_role_policy_name | Name of the IAM policy for EventBridge Role | `string` | "`observability-accelerator-EBridge-Policy`" | No | +| grafana_api_key_interval | Interval to be used while creating Grafana API Key | `number` | "`5400`" | No | +| eventbridge_scheduler_schedule_expression | Schedule Expression for EventBridge Scheduler in Grafana API Key Rotation | `string` | "`rate(60 minutes)`" | No | +| lambda_runtime_grafana_key_rotation | "Python Runtime Identifier for the Lambda Function" | `string` | "`python3.12`" | No | + +### Through the `existing-cluster-with-base-and-infra` example +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| enable_grafana_key_rotation | Enables or disables Grafana API key rotation | `bool` | "`true`" | No | +| grafana_api_key_refresh_interval | Refresh Internal to be used by External Secrets Operator of eks-monitoring module, for Grafana API Key rotation | `string` | "`5m`" | No | diff --git a/modules/grafana-key-rotation/lambda_function.py b/modules/grafana-key-rotation/lambda_function.py index b564db02..c43c9434 100644 --- a/modules/grafana-key-rotation/lambda_function.py +++ b/modules/grafana-key-rotation/lambda_function.py @@ -22,9 +22,9 @@ def lambda_handler(event, context): parameter_version = response['Parameter']['Version'] logger.info(f'SSM Parameter exists. Current Parameter Version is : {parameter_version}') - # Timestamp to be used for unique KeyName while creating Grafana Workspace API Key + # Timestamp to be used for unique KeyName while creating Grafana Workspace API Key now = datetime.now() - + # Create a new API key in Amazon Managed Grafana response = grafana_client.create_workspace_api_key( workspaceId=workspace_id, @@ -34,8 +34,8 @@ def lambda_handler(event, context): ) new_api_key = '{\"GF_SECURITY_ADMIN_APIKEY\":\"'+response['key']+'\"}' new_api_key_name = response['keyName'] - logger.info(f'New API Key Name : {new_api_key_name}') - + logger.info(f'New API Key Name : {new_api_key_name}') + # Updating SSM Parameter value with new Key response = ssm_client.put_parameter( Name=ssm_parameter_name, @@ -44,14 +44,14 @@ def lambda_handler(event, context): Overwrite=True ) new_parameter_version = response['Version'] - logger.info(f'API Key updated in SSM parameter successfully. New Parameter Version is : {new_parameter_version}') + logger.info(f'API Key updated in SSM parameter successfully. New Parameter Version is : {new_parameter_version}') + - # Deleting hour-2 API key to deal with the limit of 100 keys per WorkSpace logger.info(f'Proceeding to delete older API Key') - last_hour_date_time = now - timedelta(hours = 2) + last_hour_date_time = now - timedelta(hours = 2) old_key_name = 'observability-accelerator-'+last_hour_date_time.strftime('%Y%m%d-%H') - logger.info(f'Deleting key : {old_key_name}') + logger.info(f'Deleting key : {old_key_name}') grafana_client.delete_workspace_api_key( keyName=old_key_name, workspaceId=workspace_id @@ -82,4 +82,4 @@ def lambda_handler(event, context): return { 'statusCode': 500, 'body': 'Lambda execution failed. Please check CloudWatch Logs for additional information on error encountered.' - } \ No newline at end of file + } diff --git a/modules/grafana-key-rotation/main.tf b/modules/grafana-key-rotation/main.tf index 7ad20598..feeff7d3 100644 --- a/modules/grafana-key-rotation/main.tf +++ b/modules/grafana-key-rotation/main.tf @@ -1,7 +1,7 @@ data "archive_file" "lambda_function_archive" { type = "zip" - source_file = "${path.module}/lambda_function.py" + source_file = "${path.module}/lambda_function.py" output_path = "${path.module}/lambda_function.zip" } @@ -10,29 +10,29 @@ data "aws_region" "current" {} data "aws_caller_identity" "current" {} -# Unique random string to avoid Resource Already Exists errors. +# Unique random string to avoid Resource Already Exists errors. resource "random_string" "random_string_resource" { - length = 4 + length = 4 special = false - lower = true - upper = false + lower = true + upper = false } -# Lambda function resource +# Lambda function resource resource "aws_lambda_function" "observability_accelerator_lambda" { function_name = "${var.lambda_function_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" handler = "lambda_function.lambda_handler" - runtime = "python3.8" + runtime = var.lambda_runtime_grafana_key_rotation memory_size = 128 - timeout = 180 + timeout = 180 filename = data.archive_file.lambda_function_archive.output_path - role = aws_iam_role.lambda_role.arn + role = aws_iam_role.lambda_role.arn } -# Lambda Execution IAM Role +# Lambda Execution IAM Role resource "aws_iam_role" "lambda_role" { - name = "${var.lambda_execution_role_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" + name = "${var.lambda_execution_role_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" assume_role_policy = < Date: Fri, 2 Feb 2024 16:24:32 +0000 Subject: [PATCH 5/8] Apply pre-commit --- examples/existing-cluster-with-base-and-infra/main.tf | 10 +++++----- modules/eks-monitoring/README.md | 4 ++-- modules/eks-monitoring/outputs.tf | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/existing-cluster-with-base-and-infra/main.tf b/examples/existing-cluster-with-base-and-infra/main.tf index 74d64460..cd6a2f00 100644 --- a/examples/existing-cluster-with-base-and-infra/main.tf +++ b/examples/existing-cluster-with-base-and-infra/main.tf @@ -53,11 +53,11 @@ module "eks_monitoring" { enable_apiserver_monitoring = true # deploys external-secrets in to the cluster - enable_external_secrets = true - grafana_api_key = var.grafana_api_key - target_secret_name = "grafana-admin-credentials" - target_secret_namespace = "grafana-operator" - grafana_url = "https://${data.aws_grafana_workspace.this.endpoint}" + enable_external_secrets = true + grafana_api_key = var.grafana_api_key + target_secret_name = "grafana-admin-credentials" + target_secret_namespace = "grafana-operator" + grafana_url = "https://${data.aws_grafana_workspace.this.endpoint}" grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval managed_grafana_workspace_id = var.managed_grafana_workspace_id diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index f8a67f13..61a4242e 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -146,9 +146,9 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id | | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | | [kms\_key\_arn\_eks\_monitoring](#output\_kms\_key\_arn\_eks\_monitoring) | Name of the SSM Parameter | -| [ssmparameter\_arn\_eks\_monitoring](#output\_ssmparameter\_arn\_eks\_monitoring) | Name of the SSM Parameter | -| [ssmparameter\_name\_eks\_monitoring](#output\_ssmparameter\_name\_eks\_monitoring) | Name of the SSM Parameter | | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus workspace region | +| [ssmparameter\_arn\_eks\_monitoring](#output\_ssmparameter\_arn\_eks\_monitoring) | Name of the SSM Parameter | +| [ssmparameter\_name\_eks\_monitoring](#output\_ssmparameter\_name\_eks\_monitoring) | Name of the SSM Parameter | diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index a24d0aea..b7a01f16 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -27,7 +27,7 @@ output "kms_key_arn_eks_monitoring" { description = "Name of the SSM Parameter" value = module.external_secrets[0].kms_key_arn_ssm } - + output "managed_prometheus_workspace_endpoint" { description = "Amazon Managed Prometheus workspace endpoint" value = local.managed_prometheus_workspace_endpoint From 22de53f55900bc8216832f3df5ed0e1a977b5dea Mon Sep 17 00:00:00 2001 From: veerapratapg <57515481+veerapratapg@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:14:50 -0600 Subject: [PATCH 6/8] Made the changes requested by @bonclay7.\n Additional information can be found in the comments of the Pull Request. --- .../main.tf | 17 ++++++++--------- modules/eks-monitoring/README.md | 7 +++---- .../add-ons/external-secrets/README.md | 1 - .../add-ons/external-secrets/main.tf | 2 +- .../add-ons/external-secrets/variables.tf | 5 ----- modules/eks-monitoring/main.tf | 1 - modules/eks-monitoring/outputs.tf | 8 ++++---- modules/eks-monitoring/variables.tf | 5 ----- 8 files changed, 16 insertions(+), 30 deletions(-) diff --git a/examples/existing-cluster-with-base-and-infra/main.tf b/examples/existing-cluster-with-base-and-infra/main.tf index 74d64460..27ac54b7 100644 --- a/examples/existing-cluster-with-base-and-infra/main.tf +++ b/examples/existing-cluster-with-base-and-infra/main.tf @@ -53,13 +53,12 @@ module "eks_monitoring" { enable_apiserver_monitoring = true # deploys external-secrets in to the cluster - enable_external_secrets = true - grafana_api_key = var.grafana_api_key - target_secret_name = "grafana-admin-credentials" - target_secret_namespace = "grafana-operator" - grafana_url = "https://${data.aws_grafana_workspace.this.endpoint}" + enable_external_secrets = true + grafana_api_key = var.grafana_api_key + target_secret_name = "grafana-admin-credentials" + target_secret_namespace = "grafana-operator" + grafana_url = "https://${data.aws_grafana_workspace.this.endpoint}" grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval - managed_grafana_workspace_id = var.managed_grafana_workspace_id # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true' enable_dashboards = var.enable_dashboards @@ -92,7 +91,7 @@ module "grafana_key_rotation" { eventbridge_scheduler_schedule_expression = var.eventbridge_scheduler_schedule_expression lambda_runtime_grafana_key_rotation = var.lambda_runtime_grafana_key_rotation - ssmparameter_name = module.eks_monitoring.ssmparameter_name_eks_monitoring - ssmparameter_arn = module.eks_monitoring.ssmparameter_arn_eks_monitoring - kms_key_arn_ssm = module.eks_monitoring.kms_key_arn_eks_monitoring + ssmparameter_name = module.eks_monitoring.ssmparameter_name + ssmparameter_arn = module.eks_monitoring.ssmparameter_arn + kms_key_arn_ssm = module.eks_monitoring.kms_key_arn } diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index f8a67f13..94c42fe8 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -124,7 +124,6 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [ksm\_config](#input\_ksm\_config) | Kube State metrics configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "kube-state-metrics",
  "helm_chart_version": "5.15.2",
  "helm_release_name": "kube-state-metrics",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "kube-system",
  "scrape_interval": "60s",
  "scrape_timeout": "15s"
}
| no | | [kubeproxy\_monitoring\_config](#input\_kubeproxy\_monitoring\_config) | Config object for kube-proxy monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      default = string
    })
  })
| `null` | no | | [logs\_config](#input\_logs\_config) | Configuration object for logs collection | 
object({
    cw_log_retention_days = number
  })
| 
{
  "cw_log_retention_days": 90
}
| no | -| [managed\_grafana\_workspace\_id](#input\_managed\_grafana\_workspace\_id) | Amazon Managed Grafana Workspace ID | `string` | n/a | yes | | [managed\_prometheus\_cross\_account\_role](#input\_managed\_prometheus\_cross\_account\_role) | Amazon Managed Prometheus Workspace's Account Role Arn | `string` | `""` | no | | [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | @@ -145,10 +144,10 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [adot\_irsa\_arn](#output\_adot\_irsa\_arn) | IRSA Arn for ADOT | | [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id | | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | -| [kms\_key\_arn\_eks\_monitoring](#output\_kms\_key\_arn\_eks\_monitoring) | Name of the SSM Parameter | -| [ssmparameter\_arn\_eks\_monitoring](#output\_ssmparameter\_arn\_eks\_monitoring) | Name of the SSM Parameter | -| [ssmparameter\_name\_eks\_monitoring](#output\_ssmparameter\_name\_eks\_monitoring) | Name of the SSM Parameter | +| [kms\_key\_arn](#output\_kms\_key\_arn) | Name of the SSM Parameter | | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus workspace region | +| [ssmparameter\_arn](#output\_ssmparameter\_arn) | Name of the SSM Parameter | +| [ssmparameter\_name](#output\_ssmparameter\_name) | Name of the SSM Parameter | diff --git a/modules/eks-monitoring/add-ons/external-secrets/README.md b/modules/eks-monitoring/add-ons/external-secrets/README.md index cefe85d2..41ef95ba 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/README.md +++ b/modules/eks-monitoring/add-ons/external-secrets/README.md @@ -46,7 +46,6 @@ This deploys an EKS Cluster with the External Secrets Operator. The cluster is p | [grafana\_api\_key](#input\_grafana\_api\_key) | Grafana API key for the Amazon Managed Grafana workspace | `string` | n/a | yes | | [grafana\_api\_key\_refresh\_interval](#input\_grafana\_api\_key\_refresh\_interval) | Refresh Internal to be used by External Secrets for Grafana API Key rotation | `string` | n/a | yes | | [helm\_config](#input\_helm\_config) | Helm provider config for external secrets | `any` | `{}` | no | -| [managed\_grafana\_workspace\_id](#input\_managed\_grafana\_workspace\_id) | Amazon Managed Grafana Workspace ID | `string` | n/a | yes | | [target\_secret\_name](#input\_target\_secret\_name) | Name to store the secret for Grafana API Key | `string` | n/a | yes | | [target\_secret\_namespace](#input\_target\_secret\_namespace) | Namespace to store the secret for Grafana API Key | `string` | n/a | yes | diff --git a/modules/eks-monitoring/add-ons/external-secrets/main.tf b/modules/eks-monitoring/add-ons/external-secrets/main.tf index e1bd7aaf..f326af29 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/main.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/main.tf @@ -77,7 +77,7 @@ YAML } resource "aws_ssm_parameter" "secret" { - name = join("", ["/terraform-accelerator/grafana-api-key-", var.managed_grafana_workspace_id]) + name = "/terraform-accelerator/grafana-api-key" description = "SSM Secret to store grafana API Key" type = "SecureString" value = jsonencode({ diff --git a/modules/eks-monitoring/add-ons/external-secrets/variables.tf b/modules/eks-monitoring/add-ons/external-secrets/variables.tf index ec99490b..ea28035d 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/variables.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/variables.tf @@ -46,8 +46,3 @@ variable "grafana_api_key_refresh_interval" { description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" type = string } - -variable "managed_grafana_workspace_id" { - description = "Amazon Managed Grafana Workspace ID" - type = string -} diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 10c1c360..99bc07f4 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -267,7 +267,6 @@ module "external_secrets" { target_secret_namespace = var.target_secret_namespace target_secret_name = var.target_secret_name grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval - managed_grafana_workspace_id = var.managed_grafana_workspace_id depends_on = [resource.helm_release.grafana_operator] } diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index a24d0aea..2a2873ea 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -13,21 +13,21 @@ output "adot_irsa_arn" { value = module.helm_addon.irsa_arn } -output "ssmparameter_name_eks_monitoring" { +output "ssmparameter_name" { description = "Name of the SSM Parameter" value = module.external_secrets[0].ssmparameter_name } -output "ssmparameter_arn_eks_monitoring" { +output "ssmparameter_arn" { description = "Name of the SSM Parameter" value = module.external_secrets[0].ssmparameter_arn } -output "kms_key_arn_eks_monitoring" { +output "kms_key_arn" { description = "Name of the SSM Parameter" value = module.external_secrets[0].kms_key_arn_ssm } - + output "managed_prometheus_workspace_endpoint" { description = "Amazon Managed Prometheus workspace endpoint" value = local.managed_prometheus_workspace_endpoint diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index c1f4faab..b996e704 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -613,8 +613,3 @@ variable "grafana_api_key_refresh_interval" { type = string default = "5m" } - -variable "managed_grafana_workspace_id" { - description = "Amazon Managed Grafana Workspace ID" - type = string -} From 5f5ecc9e942ad7c2a13556a0a6c0cf9038d80d61 Mon Sep 17 00:00:00 2001 From: veerapratapg <57515481+veerapratapg@users.noreply.github.com> Date: Mon, 5 Feb 2024 23:57:03 -0600 Subject: [PATCH 7/8] Made the changes requested by @bonclay7.\n Additional information can be found in the comments of the Pull Request. --- modules/eks-monitoring/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/eks-monitoring/README.md b/modules/eks-monitoring/README.md index 7d02b5eb..94c42fe8 100644 --- a/modules/eks-monitoring/README.md +++ b/modules/eks-monitoring/README.md @@ -124,7 +124,6 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [ksm\_config](#input\_ksm\_config) | Kube State metrics configuration | 
object({
    create_namespace   = bool
    k8s_namespace      = string
    helm_chart_name    = string
    helm_chart_version = string
    helm_release_name  = string
    helm_repo_url      = string
    helm_settings      = map(string)
    helm_values        = map(any)

    scrape_interval = string
    scrape_timeout  = string
  })
| 
{
  "create_namespace": true,
  "helm_chart_name": "kube-state-metrics",
  "helm_chart_version": "5.15.2",
  "helm_release_name": "kube-state-metrics",
  "helm_repo_url": "https://prometheus-community.github.io/helm-charts",
  "helm_settings": {},
  "helm_values": {},
  "k8s_namespace": "kube-system",
  "scrape_interval": "60s",
  "scrape_timeout": "15s"
}
| no | | [kubeproxy\_monitoring\_config](#input\_kubeproxy\_monitoring\_config) | Config object for kube-proxy monitoring | 
object({
    flux_gitrepository_name   = string
    flux_gitrepository_url    = string
    flux_gitrepository_branch = string
    flux_kustomization_name   = string
    flux_kustomization_path   = string

    dashboards = object({
      default = string
    })
  })
| `null` | no | | [logs\_config](#input\_logs\_config) | Configuration object for logs collection | 
object({
    cw_log_retention_days = number
  })
| 
{
  "cw_log_retention_days": 90
}
| no | -| [managed\_grafana\_workspace\_id](#input\_managed\_grafana\_workspace\_id) | Amazon Managed Grafana Workspace ID | `string` | n/a | yes | | [managed\_prometheus\_cross\_account\_role](#input\_managed\_prometheus\_cross\_account\_role) | Amazon Managed Prometheus Workspace's Account Role Arn | `string` | `""` | no | | [managed\_prometheus\_workspace\_endpoint](#input\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus Workspace Endpoint | `string` | `""` | no | | [managed\_prometheus\_workspace\_id](#input\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus Workspace ID | `string` | `null` | no | @@ -145,10 +144,10 @@ See examples using this Terraform modules in the **Amazon EKS** section of [this | [adot\_irsa\_arn](#output\_adot\_irsa\_arn) | IRSA Arn for ADOT | | [eks\_cluster\_id](#output\_eks\_cluster\_id) | EKS Cluster Id | | [eks\_cluster\_version](#output\_eks\_cluster\_version) | EKS Cluster version | -| [kms\_key\_arn\_eks\_monitoring](#output\_kms\_key\_arn\_eks\_monitoring) | Name of the SSM Parameter | +| [kms\_key\_arn](#output\_kms\_key\_arn) | Name of the SSM Parameter | | [managed\_prometheus\_workspace\_endpoint](#output\_managed\_prometheus\_workspace\_endpoint) | Amazon Managed Prometheus workspace endpoint | | [managed\_prometheus\_workspace\_id](#output\_managed\_prometheus\_workspace\_id) | Amazon Managed Prometheus workspace ID | | [managed\_prometheus\_workspace\_region](#output\_managed\_prometheus\_workspace\_region) | Amazon Managed Prometheus workspace region | -| [ssmparameter\_arn\_eks\_monitoring](#output\_ssmparameter\_arn\_eks\_monitoring) | Name of the SSM Parameter | -| [ssmparameter\_name\_eks\_monitoring](#output\_ssmparameter\_name\_eks\_monitoring) | Name of the SSM Parameter | - \ No newline at end of file +| [ssmparameter\_arn](#output\_ssmparameter\_arn) | Name of the SSM Parameter | +| [ssmparameter\_name](#output\_ssmparameter\_name) | Name of the SSM Parameter | + From 102e3f31f9551ca231f2374905706cf3a39175dc Mon Sep 17 00:00:00 2001 From: veerapratapg <57515481+veerapratapg@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:22:02 -0500 Subject: [PATCH 8/8] Adding Unit Tests as per my discussing with bonclay7@. Additional information can be found in the comments of the Pull Request. --- .../main.tf | 19 ++ .../outputs.tf | 28 +++ .../add-ons/external-secrets/main.tf | 2 +- .../add-ons/external-secrets/outputs.tf | 16 ++ .../add-ons/external-secrets/variables.tf | 5 + modules/eks-monitoring/main.tf | 1 + modules/eks-monitoring/outputs.tf | 15 ++ modules/eks-monitoring/variables.tf | 5 + modules/grafana-key-rotation/README.md | 42 ++++ modules/grafana-key-rotation/main.tf | 195 ++++++++++++++++++ modules/grafana-key-rotation/outputs.tf | 20 ++ .../src/lambda_function.py | 80 +++++++ .../tests/unit/src/README.md | 25 +++ .../tests/unit/src/test_lambda_function.py | 138 +++++++++++++ modules/grafana-key-rotation/variables.tf | 70 +++++++ 15 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 modules/grafana-key-rotation/README.md create mode 100644 modules/grafana-key-rotation/main.tf create mode 100644 modules/grafana-key-rotation/outputs.tf create mode 100644 modules/grafana-key-rotation/src/lambda_function.py create mode 100644 modules/grafana-key-rotation/tests/unit/src/README.md create mode 100644 modules/grafana-key-rotation/tests/unit/src/test_lambda_function.py create mode 100644 modules/grafana-key-rotation/variables.tf diff --git a/examples/existing-cluster-with-base-and-infra/main.tf b/examples/existing-cluster-with-base-and-infra/main.tf index a00df02f..7345b95e 100644 --- a/examples/existing-cluster-with-base-and-infra/main.tf +++ b/examples/existing-cluster-with-base-and-infra/main.tf @@ -58,6 +58,7 @@ module "eks_monitoring" { target_secret_name = "grafana-admin-credentials" target_secret_namespace = "grafana-operator" grafana_url = "https://${data.aws_grafana_workspace.this.endpoint}" + grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true' enable_dashboards = var.enable_dashboards @@ -79,3 +80,21 @@ module "eks_monitoring" { tags = local.tags } + +# Enabling Key Rotation for API Keys of Grafana Workspace +module "grafana_key_rotation" { + source = "../../modules/grafana-key-rotation" + # source = "github.com/aws-observability/terraform-aws-observability-accelerator//modules/eks-key-rotation" + + count = var.enable_grafana_key_rotation ? 1 : 0 + + managed_grafana_workspace_id = var.managed_grafana_workspace_id + grafana_api_key_interval = var.grafana_api_key_interval + eventbridge_scheduler_schedule_expression = var.eventbridge_scheduler_schedule_expression + lambda_runtime_grafana_key_rotation = var.lambda_runtime_grafana_key_rotation + + ssmparameter_name = module.eks_monitoring.ssmparameter_name_eks_monitoring + ssmparameter_arn = module.eks_monitoring.ssmparameter_arn_eks_monitoring + kms_key_arn_ssm = module.eks_monitoring.kms_key_arn_eks_monitoring + +} \ No newline at end of file diff --git a/examples/existing-cluster-with-base-and-infra/outputs.tf b/examples/existing-cluster-with-base-and-infra/outputs.tf index e14427e3..12bc0d07 100644 --- a/examples/existing-cluster-with-base-and-infra/outputs.tf +++ b/examples/existing-cluster-with-base-and-infra/outputs.tf @@ -22,3 +22,31 @@ output "eks_cluster_id" { description = "EKS Cluster Id" value = module.eks_monitoring.eks_cluster_id } + +output "eks_key_rotation_lambda_function_arn" { + description = "ARN of the Lambda function performing Key rotation" + # value = module.grafana_key_rotation.lambda_function_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].lambda_function_arn : null +} + + +output "eks_key_rotation_lambda_function_role_arn" { + description = "ARN of the Lambda function execution role" + # value = module.grafana_key_rotation.lambda_function_role_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].lambda_function_role_arn : null +} + + +output "eks_key_rotation_eventbridge_scheduler_arn" { + description = "ARN of the EventBridge Scheduler invoking Lambda Function for Key rotation" + # value = module.grafana_key_rotation.eventbridge_scheduler_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].eventbridge_scheduler_arn : null +} + + +output "eks_key_rotation_eventbridge_scheduler_role_arn" { + description = "ARN of the IAM Role of EventBridge Scheduler invoking Lambda Function for Key rotation" + # value = module.grafana_key_rotation.eventbridge_scheduler_role_arn + value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].eventbridge_scheduler_role_arn : null +} + diff --git a/modules/eks-monitoring/add-ons/external-secrets/main.tf b/modules/eks-monitoring/add-ons/external-secrets/main.tf index 0f9fca43..c398f664 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/main.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/main.tf @@ -97,7 +97,7 @@ metadata: name: ${local.name}-sm namespace: ${var.target_secret_namespace} spec: - refreshInterval: 1h + refreshInterval: ${var.grafana_api_key_refresh_interval} secretStoreRef: name: ${local.cluster_secretstore_name} kind: ClusterSecretStore diff --git a/modules/eks-monitoring/add-ons/external-secrets/outputs.tf b/modules/eks-monitoring/add-ons/external-secrets/outputs.tf index e69de29b..f8a263f8 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/outputs.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/outputs.tf @@ -0,0 +1,16 @@ +output "ssmparameter_name" { + description = "Name of the SSM Parameter" + value = aws_ssm_parameter.secret.name +} + +output "ssmparameter_arn" { + description = "Name of the SSM Parameter" + value = aws_ssm_parameter.secret.arn +} + +output "kms_key_arn_ssm" { + description = "Name of the SSM Parameter" + value = aws_kms_key.secrets.arn +} + + diff --git a/modules/eks-monitoring/add-ons/external-secrets/variables.tf b/modules/eks-monitoring/add-ons/external-secrets/variables.tf index c9aa965a..6246054a 100644 --- a/modules/eks-monitoring/add-ons/external-secrets/variables.tf +++ b/modules/eks-monitoring/add-ons/external-secrets/variables.tf @@ -41,3 +41,8 @@ variable "target_secret_name" { description = "Name to store the secret for Grafana API Key" type = string } + +variable "grafana_api_key_refresh_interval" { + description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" + type = string +} \ No newline at end of file diff --git a/modules/eks-monitoring/main.tf b/modules/eks-monitoring/main.tf index 5b788b4a..ab5f1157 100644 --- a/modules/eks-monitoring/main.tf +++ b/modules/eks-monitoring/main.tf @@ -270,6 +270,7 @@ module "external_secrets" { addon_context = local.context target_secret_namespace = var.target_secret_namespace target_secret_name = var.target_secret_name + grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval depends_on = [resource.helm_release.grafana_operator] } diff --git a/modules/eks-monitoring/outputs.tf b/modules/eks-monitoring/outputs.tf index 38027fae..bcf2c933 100644 --- a/modules/eks-monitoring/outputs.tf +++ b/modules/eks-monitoring/outputs.tf @@ -27,3 +27,18 @@ output "managed_prometheus_workspace_region" { description = "Amazon Managed Prometheus workspace region" value = local.managed_prometheus_workspace_region } + +output "ssmparameter_name_eks_monitoring" { + description = "Name of the SSM Parameter" + value = module.external_secrets[0].ssmparameter_name +} + +output "ssmparameter_arn_eks_monitoring" { + description = "Name of the SSM Parameter" + value = module.external_secrets[0].ssmparameter_arn +} + +output "kms_key_arn_eks_monitoring" { + description = "Name of the SSM Parameter" + value = module.external_secrets[0].kms_key_arn_ssm +} diff --git a/modules/eks-monitoring/variables.tf b/modules/eks-monitoring/variables.tf index e1e58da1..81fd829c 100644 --- a/modules/eks-monitoring/variables.tf +++ b/modules/eks-monitoring/variables.tf @@ -565,3 +565,8 @@ variable "kubeproxy_monitoring_config" { # defaults are pre-computed in locals.tf, provide a full definition to override default = null } + +variable "grafana_api_key_refresh_interval" { + description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation" + type = string +} \ No newline at end of file diff --git a/modules/grafana-key-rotation/README.md b/modules/grafana-key-rotation/README.md new file mode 100644 index 00000000..9b38b4bc --- /dev/null +++ b/modules/grafana-key-rotation/README.md @@ -0,0 +1,42 @@ +# API Key Rotation for Amazon Managed Grafana Workspace + +- This module automates the rotation of Amazon Managed Grafana Workspace API Keys. +- When created, it generates an API Key for Amazon Managed Grafana Workspace on a schedule and updates the SSM parameter value to use this new key; this would then be consumed by the External Secrets Operator in `eks-monitoring` module deployed to EKS Cluster which retrieves and Syncs the Grafana API keys from AWS SSM Parameter Store. +- This module/feature is enabled by default and can be disabled setting the value of `enable_grafana_key_rotation` variable to `false` in the `variables.tf` file of `existing-cluster-with-base-and-infra` example. + +## Resources created through the module +The following AWS resources are created through this module to implement API key rotation : +- Lambda function + - To create a new Grafana API Key + - Update the SSM Parameter Value + - Delete older API key. +- Lambda Execution IAM Role + - To provide permissions related to grafana, SSM and CloudWatch logs to Lambda function. + - The permissions of the role are restricted to the specific Grafana workspace and SSM parameter that are in scope of this solution. +- EventBridge Scheduler + - To invoke the Lambda function on a cron-based schedule. +- EventBridge Scheduler IAM Role + - To provide permissions to the EventBridge Schedule to invoke the Lambda function. + - The permissions of the role are restricted to the specific Lambda function created in this solution. + + +## Configuration Options + +### Through the `grafana-key-rotation` module +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| lambda_function_name | Name of the Lambda function that creates the API Key and updates the SSM Parameter | `string` | `observability-accelerator-lambda` | No | +| lambda_execution_role_name | Name of Lambda Execution IAM Role | `string` | "`observability-accelerator-lambdaRole`" | No | +| lambda_execution_role_policy_name | Name of the Lambda Execution Role Policy | `string` | "`observability-accelerator-lambda-Policy`" | No | +| eventbridge_scheduler_name | Name of the EventBridge Scheduler that triggers the Lambda function | `string` | "`observability-accelerator-EBridge`" | No | +| eventbridge_scheduler_role_name | Name of the IAM role for EventBridge | `string` | "`observability-accelerator-EBridgeRole`" | No | +| eventbridge_scheduler_role_policy_name | Name of the IAM policy for EventBridge Role | `string` | "`observability-accelerator-EBridge-Policy`" | No | +| grafana_api_key_interval | Interval to be used while creating Grafana API Key | `number` | "`5400`" | No | +| eventbridge_scheduler_schedule_expression | Schedule Expression for EventBridge Scheduler in Grafana API Key Rotation | `string` | "`rate(60 minutes)`" | No | +| lambda_runtime_grafana_key_rotation | "Python Runtime Identifier for the Lambda Function" | `string` | "`python3.12`" | No | + +### Through the `existing-cluster-with-base-and-infra` example +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| enable_grafana_key_rotation | Enables or disables Grafana API key rotation | `bool` | "`true`" | No | +| grafana_api_key_refresh_interval | Refresh Internal to be used by External Secrets Operator of eks-monitoring module, for Grafana API Key rotation | `string` | "`5m`" | No | diff --git a/modules/grafana-key-rotation/main.tf b/modules/grafana-key-rotation/main.tf new file mode 100644 index 00000000..596cd4ea --- /dev/null +++ b/modules/grafana-key-rotation/main.tf @@ -0,0 +1,195 @@ +# Removed provider as you can't have a provider defined here when you want to use count for this module in the root module because, a provider makes this a legacy module.. +# provider "aws" { +# region = "us-east-1" # Change to your desired region +# } + +# provider "archive" { +# } + +data "archive_file" "lambda_function_archive" { + type = "zip" + + source_file = "${path.module}/src/lambda_function.py" + output_path = "${path.module}/src/lambda_function.zip" +} + + +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} + +# Unique random string to avoid Resource Already Exists errors. +resource "random_string" "random_string_resource" { + length = 4 + special = false + lower = true + upper = false +} + +# Lambda function resource +resource "aws_lambda_function" "observability_accelerator_lambda" { + function_name = "${var.lambda_function_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" + handler = "lambda_function.lambda_handler" + # runtime = "python3.8" + runtime = "${var.lambda_runtime_grafana_key_rotation}" + memory_size = 128 + timeout = 180 + # filename = "../../modules/eks-key-rotation/lambda_function.zip" + filename = data.archive_file.lambda_function_archive.output_path + role = aws_iam_role.lambda_role.arn +} + + +# Lambda Execution IAM Role +resource "aws_iam_role" "lambda_role" { + name = "${var.lambda_execution_role_name}-${var.managed_grafana_workspace_id}-${random_string.random_string_resource.id}" + assume_role_policy = <