API Key rotation for Grafana Workspace

examples/existing-cluster-with-base-and-infra/main.tf

@@ -71,11 +71,13 @@ module "eks_monitoring" {
   enable_apiserver_monitoring = true
 
   # deploys external-secrets in to the cluster
-  enable_external_secrets = true
-  grafana_api_key         = var.grafana_api_key
-  target_secret_name      = "grafana-admin-credentials"
-  target_secret_namespace = "grafana-operator"
-  grafana_url             = module.aws_observability_accelerator.managed_grafana_workspace_endpoint
+  enable_external_secrets          = true
+  grafana_api_key                  = var.grafana_api_key
+  target_secret_name               = "grafana-admin-credentials"
+  target_secret_namespace          = "grafana-operator"
+  grafana_url                      = module.aws_observability_accelerator.managed_grafana_workspace_endpoint
+  grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval
+  managed_grafana_workspace_id     = var.managed_grafana_workspace_id
 
   # control the publishing of dashboards by specifying the boolean value for the variable 'enable_dashboards', default is 'true'
   enable_dashboards = var.enable_dashboards
@@ -99,3 +101,20 @@ module "eks_monitoring" {
     module.aws_observability_accelerator
   ]
 }
+
+# Enabling Grafana API Key Rotation
+module "grafana_key_rotation" {
+  source = "../../modules/grafana-key-rotation"
+  # source = "github.com/aws-observability/terraform-aws-observability-accelerator//modules/eks-key-rotation"
+
+  count = var.enable_grafana_key_rotation ? 1 : 0
+
+  managed_grafana_workspace_id              = var.managed_grafana_workspace_id
+  grafana_api_key_interval                  = var.grafana_api_key_interval
+  eventbridge_scheduler_schedule_expression = var.eventbridge_scheduler_schedule_expression
+
+  ssmparameter_name = module.eks_monitoring.ssmparameter_name_eks_monitoring
+  ssmparameter_arn  = module.eks_monitoring.ssmparameter_arn_eks_monitoring
+  kms_key_arn_ssm   = module.eks_monitoring.kms_key_arn_eks_monitoring
+
+}

examples/existing-cluster-with-base-and-infra/outputs.tf

@@ -12,3 +12,30 @@ output "eks_cluster_id" {
   description = "EKS Cluster Id"
   value       = module.eks_monitoring.eks_cluster_id
 }
+
+output "eks_key_rotation_lambda_function_arn" {
+  description = "ARN of the Lambda function performing Key rotation"
+  # value       = module.grafana_key_rotation.lambda_function_arn
+  value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].lambda_function_arn : null
+}
+
+
+output "eks_key_rotation_lambda_function_role_arn" {
+  description = "ARN of the Lambda function execution role"
+  # value       = module.grafana_key_rotation.lambda_function_role_arn
+  value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].lambda_function_role_arn : null
+}
+
+
+output "eks_key_rotation_eventbridge_scheduler_arn" {
+  description = "ARN of the EventBridge Scheduler invoking Lambda Function for Key rotation"
+  # value       = module.grafana_key_rotation.eventbridge_scheduler_arn
+  value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].eventbridge_scheduler_arn : null
+}
+
+
+output "eks_key_rotation_eventbridge_scheduler_role_arn" {
+  description = "ARN of the IAM Role of EventBridge Scheduler invoking Lambda Function for Key rotation"
+  # value       = module.grafana_key_rotation.eventbridge_scheduler_role_arn
+  value = var.enable_grafana_key_rotation ? module.grafana_key_rotation[0].eventbridge_scheduler_role_arn : null
+}

examples/existing-cluster-with-base-and-infra/variables.tf

@@ -31,3 +31,27 @@ variable "enable_dashboards" {
   type        = bool
   default     = true
 }
+
+variable "enable_grafana_key_rotation" {
+  description = "Enables or disables Grafana API key rotation"
+  type        = bool
+  default     = true
+}
+
+variable "grafana_api_key_interval" {
+  description = "Number of seconds for secondsToLive value while creating API Key"
+  type        = number
+  default     = 5400
+}
+
+variable "eventbridge_scheduler_schedule_expression" {
+  description = "Schedule Expression for EventBridge Scheduler in Grafana API Key Rotation"
+  type        = string
+  default     = "rate(60 minutes)"
+}
+
+variable "grafana_api_key_refresh_interval" {
+  description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation"
+  type        = string
+  default     = "5m"
+}

modules/eks-monitoring/add-ons/external-secrets/main.tf

@@ -77,7 +77,7 @@ YAML
 }
 
 resource "aws_ssm_parameter" "secret" {
-  name        = "/terraform-accelerator/grafana-api-key"
+  name        = join("", ["/terraform-accelerator/grafana-api-key-", var.managed_grafana_workspace_id])
   description = "SSM Secret to store grafana API Key"
   type        = "SecureString"
   value = jsonencode({
@@ -94,7 +94,7 @@ metadata:
   name: ${local.name}-sm
   namespace: ${var.target_secret_namespace}
 spec:
-  refreshInterval: 1h
+  refreshInterval: ${var.grafana_api_key_refresh_interval}
   secretStoreRef:
     name: ${local.cluster_secretstore_name}
     kind: ClusterSecretStore

modules/eks-monitoring/add-ons/external-secrets/outputs.tf

@@ -0,0 +1,14 @@
+output "ssmparameter_name" {
+  description = "Name of the SSM Parameter"
+  value       = aws_ssm_parameter.secret.name
+}
+
+output "ssmparameter_arn" {
+  description = "Name of the SSM Parameter"
+  value       = aws_ssm_parameter.secret.arn
+}
+
+output "kms_key_arn_ssm" {
+  description = "Name of the SSM Parameter"
+  value       = aws_kms_key.secrets.arn
+}

modules/eks-monitoring/add-ons/external-secrets/variables.tf

@@ -41,3 +41,13 @@ variable "target_secret_name" {
   description = "Name to store the secret for Grafana API Key"
   type        = string
 }
+
+variable "grafana_api_key_refresh_interval" {
+  description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation"
+  type        = string
+}
+
+variable "managed_grafana_workspace_id" {
+  description = "Amazon Managed Grafana Workspace ID"
+  type        = string
+}

modules/eks-monitoring/main.tf

@@ -250,11 +250,13 @@ module "external_secrets" {
   source = "./add-ons/external-secrets"
   count  = var.enable_external_secrets ? 1 : 0
 
-  enable_external_secrets = var.enable_external_secrets
-  grafana_api_key         = var.grafana_api_key
-  addon_context           = local.context
-  target_secret_namespace = var.target_secret_namespace
-  target_secret_name      = var.target_secret_name
+  enable_external_secrets          = var.enable_external_secrets
+  grafana_api_key                  = var.grafana_api_key
+  addon_context                    = local.context
+  target_secret_namespace          = var.target_secret_namespace
+  target_secret_name               = var.target_secret_name
+  grafana_api_key_refresh_interval = var.grafana_api_key_refresh_interval
+  managed_grafana_workspace_id     = var.managed_grafana_workspace_id
 
   depends_on = [resource.helm_release.grafana_operator]
 }

modules/eks-monitoring/outputs.tf

@@ -12,3 +12,18 @@ output "adot_irsa_arn" {
   description = "IRSA Arn for ADOT"
   value       = module.helm_addon.irsa_arn
 }
+
+output "ssmparameter_name_eks_monitoring" {
+  description = "Name of the SSM Parameter"
+  value       = module.external_secrets[0].ssmparameter_name
+}
+
+output "ssmparameter_arn_eks_monitoring" {
+  description = "Name of the SSM Parameter"
+  value       = module.external_secrets[0].ssmparameter_arn
+}
+
+output "kms_key_arn_eks_monitoring" {
+  description = "Name of the SSM Parameter"
+  value       = module.external_secrets[0].kms_key_arn_ssm
+}

modules/eks-monitoring/variables.tf

@@ -575,3 +575,13 @@ variable "kubeproxy_monitoring_config" {
   # defaults are pre-computed in locals.tf, provide a full definition to override
   default = null
 }
+
+variable "grafana_api_key_refresh_interval" {
+  description = "Refresh Internal to be used by External Secrets for Grafana API Key rotation"
+  type        = string
+}
+
+variable "managed_grafana_workspace_id" {
+  description = "Amazon Managed Grafana Workspace ID"
+  type        = string
+}

modules/grafana-key-rotation/lambda_function.py

@@ -0,0 +1,85 @@
+import boto3
+import os
+import logging
+from datetime import datetime, timedelta
+
+# Initializing the AWS SDK clients
+ssm_client = boto3.client('ssm')
+grafana_client = boto3.client('grafana', region_name=os.environ['AWS_REGION'])
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+def lambda_handler(event, context):
+    try:
+        # Parse input arguments from the request event
+        ssm_parameter_name = event['ssmparameter']
+        interval_value = int(event['interval'])
+        workspace_id = event['workspaceid']
+
+        # Check if the SSM Parameter exists
+        response = ssm_client.get_parameter(Name=ssm_parameter_name)
+        parameter_version = response['Parameter']['Version']
+        logger.info(f'SSM Parameter exists. Current Parameter Version is : {parameter_version}')
+
+        # Timestamp to be used for unique KeyName while creating Grafana Workspace API Key 
+        now = datetime.now()
+
+        # Create a new API key in Amazon Managed Grafana
+        response = grafana_client.create_workspace_api_key(
+            workspaceId=workspace_id,
+            keyRole='ADMIN',
+            keyName='observability-accelerator-'+now.strftime("%Y%m%d-%H"),
+            secondsToLive=interval_value
+        )
+        new_api_key = '{\"GF_SECURITY_ADMIN_APIKEY\":\"'+response['key']+'\"}'
+        new_api_key_name = response['keyName']
+        logger.info(f'New API Key Name : {new_api_key_name}') 
+
+        # Updating SSM Parameter value with new Key
+        response = ssm_client.put_parameter(
+            Name=ssm_parameter_name,
+            Value=new_api_key,
+            Type='SecureString',
+            Overwrite=True
+        )
+        new_parameter_version = response['Version']
+        logger.info(f'API Key updated in SSM parameter successfully. New Parameter Version is : {new_parameter_version}') 
+
+
+        # Deleting hour-2 API key to deal with the limit of 100 keys per WorkSpace
+        logger.info(f'Proceeding to delete older API Key')
+        last_hour_date_time = now - timedelta(hours = 2) 
+        old_key_name = 'observability-accelerator-'+last_hour_date_time.strftime('%Y%m%d-%H')
+        logger.info(f'Deleting key : {old_key_name}') 
+        grafana_client.delete_workspace_api_key(
+            keyName=old_key_name,
+            workspaceId=workspace_id
+            )
+        logger.info(f'Deleted the API Key with name :  {old_key_name}')
+
+        return {
+            'statusCode': 200,
+            'body': 'API key updated successfully and older key has been deleted'
+        }
+
+    except ssm_client.exceptions.ParameterNotFound:
+        logger.error('SSM Parameter does not exist')
+        return {
+            'statusCode': 404,
+            'body': 'SSM Parameter does not exist'
+        }
+
+    except grafana_client.exceptions.ResourceNotFoundException:
+        logger.error('Older API Key does not exist')
+        return {
+            'statusCode': 200,
+            'body': 'New API Key added to SSM Parameter value.'
+        }
+
+    except Exception as e:
+        logger.error(f'An error occurred: {str(e)}')
+        return {
+            'statusCode': 500,
+            'body': 'Lambda execution failed. Please check CloudWatch Logs for additional information on error encountered.'
+        }