diff --git a/.gitignore b/.gitignore index 2009134e..3ba109db 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,6 @@ terraform/*.out terraform/local.tfvars terraform/.terraform.lock.hcl terraform/builds/ +scripts/*.json *.DS_Store diff --git a/scripts/check_batch_jobs.py b/scripts/check_batch_jobs.py new file mode 100755 index 00000000..63b04fc3 --- /dev/null +++ b/scripts/check_batch_jobs.py @@ -0,0 +1,32 @@ +#! /usr/bin/python3 + +# the deploy_ami_rotate.sh script calls this script, +# and stops ami rotation before calling terraform +# if this script exits non-zero + +import os +import json +import sys + +statuses = ['RUNNING','SUBMITTED','PENDING','RUNNABLE','STARTING'] + +# clean up before running +cmd = 'rm ./*.json' +os.system(cmd) + +# dump the jobQueues to json +cmd = "awsudo $ADMIN_ARN aws batch describe-job-queues > queues.json" +os.system(cmd) + +with open('./queues.json', 'r') as f: + queues = json.load(f) + +for queue in queues['jobQueues']: + name = queue['jobQueueName'] + for status in statuses: + cmd = f"awsudo $ADMIN_ARN aws batch list-jobs --job-queue {name} --job-status {status} > {name}_{status}.json" + os.system(cmd) + with open(f"{name}_{status}.json", 'r') as f: + jobs = json.load(f) + if len(jobs['jobSummaryList']) > 0: + sys.exit(1) \ No newline at end of file diff --git a/terraform/deploy.sh b/terraform/deploy.sh index 79516eee..8f624fac 100755 --- a/terraform/deploy.sh +++ b/terraform/deploy.sh @@ -3,7 +3,7 @@ # ADMIN_ARN is set in the ci node env and should not be included in this deploy script # variables that will likely be changed frequently -CALCLOUD_VER="0.4.25" +CALCLOUD_VER="0.4.26" CALDP_VER="0.2.13" CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20210721_CAL_final" diff --git a/terraform/deploy_ami_rotate.sh b/terraform/deploy_ami_rotate.sh index 6fdaf85a..e8bffe85 100755 --- a/terraform/deploy_ami_rotate.sh +++ b/terraform/deploy_ami_rotate.sh @@ -2,10 +2,18 @@ # ADMIN_ARN is set in the ci node env and should not be included in this deploy script -# variables that will likely be changed frequently -CALCLOUD_VER="0.4.25" -CALDP_VER="0.2.13" -CAL_BASE_IMAGE="stsci/hst-pipeline:CALDP_20210721_CAL_final" +# get the versions from ssm params +calcloud_ver_response=`awsudo $ADMIN_ARN aws ssm get-parameter --name "/tf/env/awsysver" | grep "Value"` +CALCLOUD_VER=${calcloud_ver_response##*:} +CALCLOUD_VER=`echo $CALCLOUD_VER | tr -d '",'` + +caldp_ver_response=`awsudo $ADMIN_ARN aws ssm get-parameter --name "/tf/env/awsdpver" | grep "Value"` +CALDP_VER=${caldp_ver_response##*:} +CALDP_VER=`echo $CALDP_VER | tr -d '",'` + +csys_ver_response=`awsudo $ADMIN_ARN aws ssm get-parameter --name "/tf/env/csys_ver" | grep "Value"` +CSYS_VER=${csys_ver_response##*:} +CSYS_VER=`echo $CSYS_VER | tr -d '",'` # these variables are overrides for developers that allow the deploy script to build from local calcloud/caldp source # i.e. CALCLOUD_BUILD_DIR="$HOME/deployer/calcloud" @@ -15,12 +23,6 @@ CALCLOUD_BUILD_DIR=${CALCLOUD_BUILD_DIR:-""} CALDP_BUILD_DIR=${CALDP_BUILD_DIR:-""} aws_env=${aws_env:-""} -# turn CAL_BASE_IMAGE into CSYS_VER by splitting at the :, splitting again by underscore and keeping the -# first two fields, and then converting to lowercase -CSYS_VER=${CAL_BASE_IMAGE##*:} -CSYS_VER=`echo $CSYS_VER | cut -f1,2 -d'_'` #split by underscores, keep the first two -CSYS_VER=`echo $CSYS_VER | awk '{print tolower($0)}'` - # variables that will be changed less-frequently TMP_INSTALL_DIR="/tmp/calcloud_install" @@ -34,21 +36,28 @@ then cd $TMP_INSTALL_DIR git clone https://github.com/spacetelescope/calcloud.git cd calcloud && git fetch --all --tags && git checkout tags/v${CALCLOUD_VER} && cd .. + git_exit_status=$? + if [[ $git_exit_status -ne 0 ]]; then + # try without the v + cd calcloud && git fetch --all --tags && git checkout tags/${CALCLOUD_VER} && cd .. + git_exit_status=$? + fi + if [[ $git_exit_status -ne 0 ]]; then + echo "could not checkout ${CALCLOUD_VER}; exiting" + exit 1 + fi fi -# setting up the caldp source dir if it needs downloaded -# equivalent to "if len($var) == 0" -if [ -z "${CALDP_BUILD_DIR}"] -then - mkdir -p $TMP_INSTALL_DIR - CALDP_BUILD_DIR="${TMP_INSTALL_DIR}/caldp" - cd $TMP_INSTALL_DIR - # caldp source download/unpack - # github's tarballs don't work with pip install, so we have to clone and checkout the tag - git clone https://github.com/spacetelescope/caldp.git - cd caldp && git fetch --all --tags && git checkout tags/v${CALDP_VER} && cd .. +# check for Batch jobs and exit if any exist that are running or should be soon +cd ${CALCLOUD_BUILD_DIR}/scripts +./check_batch_jobs.py +batch_jobs=$? +if [[ $batch_jobs -ne 0 ]]; then + echo "there are running or submitted batch jobs; cannot rotate ami" + exit 1 fi + # get a couple of things from AWS ssm # the env, i.e. sb,dev,test,prod if [ -z "${aws_env}" ] @@ -86,3 +95,6 @@ awsudo $ADMIN_ARN terraform plan -var "environment=${aws_env}" -out ami_rotate.o -var "awsysver=${CALCLOUD_VER}" -var "awsdpver=${CALDP_VER}" -var "csys_ver=${CSYS_VER}" -var "environment=${aws_env}" awsudo $ADMIN_ARN terraform apply "ami_rotate.out" + +cd $HOME +rm -rf $TMP_INSTALL_DIR diff --git a/terraform/parameters.tf b/terraform/parameters.tf index cf5be3bb..ca7647eb 100644 --- a/terraform/parameters.tf +++ b/terraform/parameters.tf @@ -104,4 +104,22 @@ data aws_ssm_parameter model_bucket { data aws_ssm_parameter batch_exec { name = "/iam/roles/batch_exec" +} + +resource "aws_ssm_parameter" "awsysver" { + name = "/tf/env/awsysver" + type = "String" + value = "${var.awsysver}" +} + +resource "aws_ssm_parameter" "awsdpver" { + name = "/tf/env/awsdpver" + type = "String" + value = "${var.awsdpver}" +} + +resource "aws_ssm_parameter" "csys_ver" { + name = "/tf/env/csys_ver" + type = "String" + value = "${var.csys_ver}" } \ No newline at end of file