Merge pull request #45 from bhayden53/lambda/refreshCacheLogging

storage gateway refresh cache event logging + bugfixes

lambda/blackboard/scrape_batch.py

@@ -94,8 +94,12 @@ def lambda_handler(event, context):
     with open(filename, "rb") as f:
         s3.upload_fileobj(f, os.environ["BUCKET"], "blackboard/blackboardAWS.snapshot")
 
-    response = gateway.refresh_cache(FileShareARN=os.environ["FILESHARE"], FolderList=["/blackboard/"], Recursive=True)
+    try:
+        response = gateway.refresh_cache(FileShareARN=os.environ["FILESHARE"], FolderList=["/blackboard/"], Recursive=True)
+        print(response)
+    except Exception as exc:
+        print(str(exc))
 
-    print(response)
+
 
     return None

lambda/refreshCacheLogs/refresh_cache_logs.py

@@ -0,0 +1,2 @@
+def lambda_handler(event, context):
+    print(event)

terraform/batch-outlier.tf

@@ -9,7 +9,7 @@ resource "aws_batch_job_queue" "batch_outlier_queue" {
 }
 
 resource "aws_batch_compute_environment" "calcloud_outlier" {
-  compute_environment_name = "calcloud-hst-outlier${local.environment}-${random_string.env_name.result}"
+  compute_environment_name_prefix = "calcloud-hst-outlier${local.environment}-"
   type = "MANAGED"
   service_role = data.aws_ssm_parameter.batch_service_role.value
 
@@ -31,6 +31,7 @@ resource "aws_batch_compute_environment" "calcloud_outlier" {
 
     launch_template {
       launch_template_id = aws_launch_template.hstdp.id
+      version = "$Latest"
     }
   }
   lifecycle { 

terraform/batch.tf

@@ -2,16 +2,6 @@ provider "aws" {
   region  = var.region
 }
 
-resource "random_string" "env_name" {
-  # see https://github.com/hashicorp/terraform-provider-aws/pull/2347#issuecomment-345292890
-  # regarding the need for a random string in the compute-env name to avoid
-  # issues recreating them. They must be created before the old
-  # one can be destroyed, so they need unique names.
-  length = 5
-  special = false
-  upper = false
-}
-
 data "template_file" "userdata" {
   template = file("${path.module}/user_data.sh")
   vars = {
@@ -37,7 +27,7 @@ resource "aws_launch_template" "hstdp" {
 
   ebs {
     delete_on_termination = "true"
-    encrypted             = "false"
+    encrypted             = "true"
     iops                  = 0
     volume_size           = 150
     volume_type           = "gp2"
@@ -78,7 +68,7 @@ resource "aws_batch_job_queue" "batch_queue" {
 }
 
 resource "aws_batch_compute_environment" "calcloud" {
-  compute_environment_name  = "calcloud-hst${local.environment}-${random_string.env_name.result}"
+  compute_environment_name_prefix = "calcloud-hst${local.environment}-"
   type = "MANAGED"
   service_role = data.aws_ssm_parameter.batch_service_role.value
 
@@ -97,6 +87,7 @@ resource "aws_batch_compute_environment" "calcloud" {
 
     launch_template {
       launch_template_id = aws_launch_template.hstdp.id
+      version = "$Latest"
     }
   }
   lifecycle { 
@@ -151,6 +142,13 @@ resource "aws_s3_bucket" "calcloud" {
     "CALCLOUD" = "calcloud-processing${local.environment}"
     "Name"     = "calcloud-processing${local.environment}"
   }
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        sse_algorithm     = "AES256"
+      }
+    }
+  }
 }
 
 resource "aws_s3_bucket_public_access_block" "s3_public_block" {
@@ -161,3 +159,32 @@ resource "aws_s3_bucket_public_access_block" "s3_public_block" {
   restrict_public_buckets = true
   ignore_public_acls=true
 }
+
+# ssl requests policy
+resource "aws_s3_bucket_policy" "ssl_only_processing" {
+  bucket = aws_s3_bucket.calcloud.id
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression's result to valid JSON syntax.
+  policy = jsonencode({
+    Id = "SSLPolicy",
+    Version = "2012-10-17",
+    Statement = [
+        {
+            Sid = "AllowSSLRequestsOnly",
+            Action = "s3:*",
+            Effect = "Deny",
+            Principal = "*",
+            Resource = [
+                aws_s3_bucket.calcloud.arn,
+                "${aws_s3_bucket.calcloud.arn}/*"
+            ],
+            Condition = {
+                Bool = {
+                     "aws:SecureTransport" = "false"
+                }
+            }
+        }
+    ]
+  })
+}

terraform/lambda_common.tf

@@ -4,9 +4,45 @@ resource "aws_s3_bucket" "calcloud_lambda_envs" {
   tags = {
     "Name"     = "calcloud-lambda-envs${local.environment}"
   }
+  server_side_encryption_configuration {
+    rule {
+      apply_server_side_encryption_by_default {
+        sse_algorithm     = "AES256"
+      }
+    }
+  }
   force_destroy = true
 }
 
+# ssl requests policy
+resource "aws_s3_bucket_policy" "ssl_only_lambda_envs" {
+  bucket = aws_s3_bucket.calcloud_lambda_envs.id
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression's result to valid JSON syntax.
+  policy = jsonencode({
+    Id = "SSLPolicy",
+    Version = "2012-10-17",
+    Statement = [
+        {
+            Sid = "AllowSSLRequestsOnly",
+            Action = "s3:*",
+            Effect = "Deny",
+            Principal = "*",
+            Resource = [
+                aws_s3_bucket.calcloud_lambda_envs.arn,
+                "${aws_s3_bucket.calcloud_lambda_envs.arn}/*"
+            ],
+            Condition = {
+                Bool = {
+                     "aws:SecureTransport" = "false"
+                }
+            }
+        }
+    ]
+  })
+}
+
 # locks down the lambda env bucket
 resource "aws_s3_bucket_public_access_block" "s3_lambda_public_block" {
   bucket = aws_s3_bucket.calcloud_lambda_envs.id

terraform/lambda_refresh_cache_logging.tf

@@ -0,0 +1,80 @@
+module "calcloud_lambda_refreshCache" {
+  source = "terraform-aws-modules/lambda/aws"
+
+  function_name = "calcloud-fileshare-refresh_cache${local.environment}"
+  description   = "listens for refresh cache operations and logs them"
+  # the path is relative to the path inside the lambda env, not in the local filesystem. 
+  handler       = "refresh_cache_logs.lambda_handler"
+  runtime       = "python3.6"
+  publish       = false
+  timeout       = 900
+
+  source_path = [
+    {
+      # this is the lambda itself. The code in path will be placed directly into the lambda execution path
+      path = "${path.module}/../lambda/refreshCacheLogs"
+      pip_requirements = false
+    },
+    {
+      # calcloud for the package. We don't need to install boto3 and whatnot so we leave out the pip requirements
+      # in the zip it will be installed into a directory called calcloud
+      path = "${path.module}/../calcloud"
+      prefix_in_zip = "calcloud"
+      pip_requirements = false
+    }
+  ]
+
+  store_on_s3 = true
+  s3_bucket   = aws_s3_bucket.calcloud_lambda_envs.id
+
+  # ensures that terraform doesn't try to mess with IAM
+  create_role = false
+  attach_cloudwatch_logs_policy = false
+  attach_dead_letter_policy = false
+  attach_network_policy = false
+  attach_tracing_policy = false
+  attach_async_event_policy = false
+  # existing role for the lambda
+  # will need to parametrize when ITSD takes over role creation. 
+  # for now this role was created by hand in the console, it is not terraform managed
+  lambda_role = data.aws_ssm_parameter.lambda_cloudwatch_role.value
+
+#   environment_variables = {
+#     JOBQUEUES="${aws_batch_job_queue.batch_queue.name},${aws_batch_job_queue.batch_outlier_queue.name}"
+#   }
+
+  tags = {
+    Name = "calcloud-fileshare-refresh_caches${local.environment}"
+  }
+}
+
+# the event rule for this lambda/cloudwatch interaction is AWS failure events
+resource "aws_cloudwatch_event_rule" "refresh_cache" {
+  name = "capture-refresh-cache-operations"
+  description = "capture file share refresh cache operations to track and evaluate them in log stream"
+
+  event_pattern = <<EOF
+{
+  "source": [
+    "aws.storagegateway"
+  ],
+  "detail-type": [
+    "Storage Gateway Refresh Cache Event"
+  ]
+}
+EOF
+}
+
+resource "aws_cloudwatch_event_target" "refresh_cache" {
+  rule      = aws_cloudwatch_event_rule.refresh_cache.name
+  target_id = "lambda"
+  arn       = module.calcloud_lambda_refreshCache.this_lambda_function_arn
+}
+
+resource "aws_lambda_permission" "allow_lambda_exec_refreshCache" {
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = module.calcloud_lambda_refreshCache.this_lambda_function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.refresh_cache.arn
+}

terraform/parameters.tf

@@ -42,6 +42,10 @@ data aws_ssm_parameter lambda_delete_role {
   name = "/iam/roles/calcloud_lambda_delete"
 }
 
+data aws_ssm_parameter lambda_cloudwatch_role {
+  name = "/iam/roles/calcloud_lambda_cloudWatchLogs"
+}
+
 data aws_ssm_parameter file_share_arn {
   name = "/gateway/fileshare"
 }