Merge pull request #143 from spacetelescope/hotfix/v0.4.26

Hotfix/v0.4.26

.gitignore

@@ -18,5 +18,6 @@ terraform/*.out
 terraform/local.tfvars
 terraform/.terraform.lock.hcl
 terraform/builds/
+scripts/*.json
 
 *.DS_Store

scripts/check_batch_jobs.py

@@ -0,0 +1,32 @@
+#! /usr/bin/python3
+
+# the deploy_ami_rotate.sh script calls this script, 
+# and stops ami rotation before calling terraform
+# if this script exits non-zero
+
+import os
+import json
+import sys
+
+statuses = ['RUNNING','SUBMITTED','PENDING','RUNNABLE','STARTING']
+
+# clean up before running
+cmd = 'rm ./*.json'
+os.system(cmd)
+
+# dump the jobQueues to json
+cmd = "awsudo $ADMIN_ARN aws batch describe-job-queues > queues.json"
+os.system(cmd)
+
+with open('./queues.json', 'r') as f:
+    queues = json.load(f)
+
+for queue in queues['jobQueues']:
+    name = queue['jobQueueName']
+    for status in statuses:
+        cmd = f"awsudo $ADMIN_ARN aws batch list-jobs --job-queue {name} --job-status {status} > {name}_{status}.json"
+        os.system(cmd)
+        with open(f"{name}_{status}.json", 'r') as f:
+            jobs = json.load(f)
+            if len(jobs['jobSummaryList']) > 0:
+                sys.exit(1)    

terraform/deploy.sh

@@ -3,7 +3,7 @@
 # ADMIN_ARN is set in the ci node env and should not be included in this deploy script
 
 # variables that will likely be changed frequently
-CALCLOUD_VER="0.4.25"
+CALCLOUD_VER="0.4.26"
 CALDP_VER="0.2.13"
 CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20210721_CAL_final"
 

terraform/deploy_ami_rotate.sh

@@ -2,10 +2,18 @@
 
 # ADMIN_ARN is set in the ci node env and should not be included in this deploy script
 
-# variables that will likely be changed frequently
-CALCLOUD_VER="0.4.25"
-CALDP_VER="0.2.13"
-CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20210721_CAL_final"
+# get the versions from ssm params
+calcloud_ver_response=`awsudo $ADMIN_ARN aws ssm get-parameter --name "/tf/env/awsysver" | grep "Value"`
+CALCLOUD_VER=${calcloud_ver_response##*:}
+CALCLOUD_VER=`echo $CALCLOUD_VER | tr -d '",'`
+
+caldp_ver_response=`awsudo $ADMIN_ARN aws ssm get-parameter --name "/tf/env/awsdpver" | grep "Value"`
+CALDP_VER=${caldp_ver_response##*:}
+CALDP_VER=`echo $CALDP_VER | tr -d '",'`
+
+csys_ver_response=`awsudo $ADMIN_ARN aws ssm get-parameter --name "/tf/env/csys_ver" | grep "Value"`
+CSYS_VER=${csys_ver_response##*:}
+CSYS_VER=`echo $CSYS_VER | tr -d '",'`
 
 # these variables are overrides for developers that allow the deploy script to build from local calcloud/caldp source
 # i.e. CALCLOUD_BUILD_DIR="$HOME/deployer/calcloud"
@@ -15,12 +23,6 @@ CALCLOUD_BUILD_DIR=${CALCLOUD_BUILD_DIR:-""}
 CALDP_BUILD_DIR=${CALDP_BUILD_DIR:-""}
 aws_env=${aws_env:-""}
 
-# turn CAL_BASE_IMAGE into CSYS_VER by splitting at the :, splitting again by underscore and keeping the
-# first two fields, and then converting to lowercase
-CSYS_VER=${CAL_BASE_IMAGE##*:}
-CSYS_VER=`echo $CSYS_VER | cut -f1,2 -d'_'` #split by underscores, keep the first two
-CSYS_VER=`echo $CSYS_VER | awk '{print tolower($0)}'`
-
 # variables that will be changed less-frequently
 TMP_INSTALL_DIR="/tmp/calcloud_install"
 
@@ -34,21 +36,28 @@ then
     cd $TMP_INSTALL_DIR
     git clone https://github.com/spacetelescope/calcloud.git
     cd calcloud && git fetch --all --tags && git checkout tags/v${CALCLOUD_VER} && cd ..
+    git_exit_status=$?
+    if [[ $git_exit_status -ne 0 ]]; then
+        # try without the v
+        cd calcloud && git fetch --all --tags && git checkout tags/${CALCLOUD_VER} && cd ..
+        git_exit_status=$?
+    fi
+    if [[ $git_exit_status -ne 0 ]]; then
+        echo "could not checkout ${CALCLOUD_VER}; exiting"
+        exit 1
+    fi
 fi
 
-# setting up the caldp source dir if it needs downloaded
-# equivalent to "if len($var) == 0"
-if [ -z "${CALDP_BUILD_DIR}"]
-then
-    mkdir -p $TMP_INSTALL_DIR
-    CALDP_BUILD_DIR="${TMP_INSTALL_DIR}/caldp"
-    cd $TMP_INSTALL_DIR
-    # caldp source download/unpack
-    # github's tarballs don't work with pip install, so we have to clone and checkout the tag
-    git clone https://github.com/spacetelescope/caldp.git
-    cd caldp && git fetch --all --tags && git checkout tags/v${CALDP_VER} && cd ..
+# check for Batch jobs and exit if any exist that are running or should be soon
+cd ${CALCLOUD_BUILD_DIR}/scripts
+./check_batch_jobs.py 
+batch_jobs=$?
+if [[ $batch_jobs -ne 0 ]]; then
+    echo "there are running or submitted batch jobs; cannot rotate ami"
+    exit 1
 fi
 
+
 # get a couple of things from AWS ssm
 # the env, i.e. sb,dev,test,prod
 if [ -z "${aws_env}" ]
@@ -86,3 +95,6 @@ awsudo $ADMIN_ARN terraform plan -var "environment=${aws_env}" -out ami_rotate.o
     -var "awsysver=${CALCLOUD_VER}" -var "awsdpver=${CALDP_VER}" -var "csys_ver=${CSYS_VER}" -var "environment=${aws_env}"
 
 awsudo $ADMIN_ARN terraform apply "ami_rotate.out"
+
+cd $HOME
+rm -rf $TMP_INSTALL_DIR

terraform/parameters.tf

@@ -104,4 +104,22 @@ data aws_ssm_parameter model_bucket {
 
 data aws_ssm_parameter batch_exec {
   name = "/iam/roles/batch_exec"
+}
+
+resource "aws_ssm_parameter" "awsysver" {
+  name  = "/tf/env/awsysver"
+  type  = "String"
+  value = "${var.awsysver}"
+}
+
+resource "aws_ssm_parameter" "awsdpver" {
+  name  = "/tf/env/awsdpver"
+  type  = "String"
+  value = "${var.awsdpver}"
+}
+
+resource "aws_ssm_parameter" "csys_ver" {
+  name  = "/tf/env/csys_ver"
+  type  = "String"
+  value = "${var.csys_ver}"
 }