Merge pull request #75 from bhayden53/master

CRDS config for using the official S3 bucket
spacetelescope · Mar 29, 2021 · b64b608 · b64b608
2 parents 0d3a1fb + a79700a
commit b64b608
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 8 deletions.
diff --git a/calcloud/plan.py b/calcloud/plan.py
@@ -107,11 +107,11 @@ def _get_resources(ipppssoot, output_bucket, input_path):
     instr = hst.get_instrument(ipppssoot)
     job_name = ipppssoot
     input_path = input_path
-    crds_config = "caldp-config-offsite"
+    crds_config = "caldp-config-aws"
     # invoke calcloud-ai lambda
     predictions = invoke_lambda_predict(ipppssoot, output_bucket)
     initial_bin = predictions["memBin"]  # 0
-    kill_time = predictions["clockTime"] * 3  # 48 * 60 * 60
+    kill_time = min(max(predictions["clockTime"] * 5, 600), 48 * 60 * 60)  # between 10 mins and 2 days
 
     return JobResources(ipppssoot, instr, job_name, s3_output_uri, input_path, crds_config, initial_bin, kill_time)
 

diff --git a/terraform/batch.tf b/terraform/batch.tf
@@ -165,7 +165,9 @@ resource "aws_batch_job_definition" "calcloud_2g" {
     "environment": [
       {"name": "AWSDPVER", "value": "${var.awsdpver}"},
       {"name": "AWSYSVER", "value": "${var.awsysver}"},
-      {"name": "CSYS_VER", "value": "${var.csys_ver}"}
+      {"name": "CSYS_VER", "value": "${var.csys_ver}"},
+      {"name": "CRDSBUCKET", "value": "${local.crds_bucket}"},
+      {"name": "CRDS_CONTEXT", "value": "${local.crds_context}"}
     ],
     "image": "${aws_ecr_repository.caldp_ecr.repository_url}:${data.aws_ecr_image.caldp_latest.image_tag}",
     "jobRoleArn": "${data.aws_ssm_parameter.batch_job_role.value}",
@@ -199,7 +201,9 @@ resource "aws_batch_job_definition" "calcloud_8g" {
     "environment": [
       {"name": "AWSDPVER", "value": "${var.awsdpver}"},
       {"name": "AWSYSVER", "value": "${var.awsysver}"},
-      {"name": "CSYS_VER", "value": "${var.csys_ver}"}
+      {"name": "CSYS_VER", "value": "${var.csys_ver}"},
+      {"name": "CRDSBUCKET", "value": "${local.crds_bucket}"},
+      {"name": "CRDS_CONTEXT", "value": "${local.crds_context}"}
     ],
     "image": "${aws_ecr_repository.caldp_ecr.repository_url}:${data.aws_ecr_image.caldp_latest.image_tag}",
     "jobRoleArn": "${data.aws_ssm_parameter.batch_job_role.value}",
@@ -233,7 +237,9 @@ resource "aws_batch_job_definition" "calcloud_16g" {
     "environment": [
       {"name": "AWSDPVER", "value": "${var.awsdpver}"},
       {"name": "AWSYSVER", "value": "${var.awsysver}"},
-      {"name": "CSYS_VER", "value": "${var.csys_ver}"}
+      {"name": "CSYS_VER", "value": "${var.csys_ver}"},
+      {"name": "CRDSBUCKET", "value": "${local.crds_bucket}"},
+      {"name": "CRDS_CONTEXT", "value": "${local.crds_context}"}
     ],
     "image": "${aws_ecr_repository.caldp_ecr.repository_url}:${data.aws_ecr_image.caldp_latest.image_tag}",
     "jobRoleArn": "${data.aws_ssm_parameter.batch_job_role.value}",
@@ -267,7 +273,9 @@ resource "aws_batch_job_definition" "calcloud_64g" {
     "environment": [
       {"name": "AWSDPVER", "value": "${var.awsdpver}"},
       {"name": "AWSYSVER", "value": "${var.awsysver}"},
-      {"name": "CSYS_VER", "value": "${var.csys_ver}"}
+      {"name": "CSYS_VER", "value": "${var.csys_ver}"},
+      {"name": "CRDSBUCKET", "value": "${local.crds_bucket}"},
+      {"name": "CRDS_CONTEXT", "value": "${local.crds_context}"}
     ],
     "image": "${aws_ecr_repository.caldp_ecr.repository_url}:${data.aws_ecr_image.caldp_latest.image_tag}",
     "jobRoleArn": "${data.aws_ssm_parameter.batch_job_role.value}",

diff --git a/terraform/deploy.sh b/terraform/deploy.sh
@@ -5,8 +5,8 @@
 # variables that will likely be changed frequently
 CALCLOUD_VER="0.3.1"
 CALDP_VER="0.2.1"
-CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20201208_DRZ_final"
-CSYS_VER="CALDP_20201208"
+CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20210323_CAL_final"
+CSYS_VER="CALDP_20210323"
 # this is the tag that the image will have in AWS ECR
 CALDP_IMAGE_TAG="latest"
 

diff --git a/terraform/locals.tf b/terraform/locals.tf
@@ -13,4 +13,14 @@ locals {
 
        ecr_address = format("%v.dkr.ecr.%v.amazonaws.com", data.aws_caller_identity.this.account_id, data.aws_region.current.name)
        ecr_predict_lambda_image   = format("%v/%v:model", local.ecr_address, aws_ecr_repository.caldp_ecr.name)
+
+       # because we cannot reference ssm params in variables, we have to set the crds bucket here by looking up the desired bucket through a string
+       # set in var.crds_bucket. We then use this map to convert that string to the correct ssm param here
+       crds_bucket = {
+              "test" : data.aws_ssm_parameter.crds_test.value,
+              "ops" : data.aws_ssm_parameter.crds_ops.value,
+       }[lookup(var.crds_bucket, local.environment, "test")]
+
+       # code is cleaner to put this in locals
+       crds_context = lookup(var.crds_context, local.environment, var.crds_context["-sb"])
 }
diff --git a/terraform/parameters.tf b/terraform/parameters.tf
@@ -69,3 +69,11 @@ data aws_ssm_parameter file_share_arn {
 data aws_ssm_parameter lambda_rescue_role {
   name = "/iam/roles/calcloud_lambda_rescue"
 }
+
+data aws_ssm_parameter crds_ops {
+  name = "/s3/external/crds-ops"
+}
+
+data aws_ssm_parameter crds_test {
+  name = "/s3/external/crds-test"
+}
diff --git a/terraform/variables.tf b/terraform/variables.tf
@@ -92,3 +92,24 @@ variable ce_max_vcpu {
     "-ops" = 128
   }
 }
+
+variable crds_context {
+  description = "the crds context to be the default for the environment"
+  type = map(string)
+  default = {
+    "-sb" = "hst_0866.pmap"
+    "-dev" = "hst_0866.pmap"
+    "-test" = "hst_0866.pmap"
+    "-ops" = "hst_0869.pmap"
+  }
+}
+
+variable crds_bucket {
+  type = map(string)
+  default = {
+    "-sb" = "test"
+    "-dev" = "test"
+    "-test" = "test"
+    "-ops" = "ops"
+  }
+}