From 4e77a3e8132feee2429800d2040a5ba26230f625 Mon Sep 17 00:00:00 2001 From: Soo Lee Date: Wed, 25 Aug 2021 09:45:47 -0400 Subject: [PATCH 1/6] Update aws_run_workflow_generic.sh --- awsf3/aws_run_workflow_generic.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/awsf3/aws_run_workflow_generic.sh b/awsf3/aws_run_workflow_generic.sh index 0ebb718c6..b04e3a4e0 100755 --- a/awsf3/aws_run_workflow_generic.sh +++ b/awsf3/aws_run_workflow_generic.sh @@ -47,6 +47,7 @@ export LOGFILE2=$LOCAL_OUTDIR/$JOBID.log export STATUS=0 export ERRFILE=$LOCAL_OUTDIR/$JOBID.error # if this is found on s3, that means something went wrong. export INSTANCE_REGION=$(ec2metadata --availability-zone | sed 's/[a-z]$//') +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity| grep Account | sed 's/[^0-9]//g') # function that executes a command and collecting log @@ -159,6 +160,12 @@ if [ ! -z $ACCESS_KEY -a ! -z $SECRET_KEY -a ! -z $REGION ]; then echo -ne "$ACCESS_KEY\n$SECRET_KEY\n$REGION\njson" | aws configure --profile user1 fi +### log into ECR if necessary +exl echo +exl echo "## Logging into ECR" +exlo docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com; +send_log + # send log before starting docker exl echo exl echo "## Running dockerized awsf scripts" From 7e2249ae74f4ce864ba6f41d90dd812b2614601d Mon Sep 17 00:00:00 2001 From: Soo Lee Date: Wed, 25 Aug 2021 10:37:58 -0400 Subject: [PATCH 2/6] Update run.sh --- awsf3-docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awsf3-docker/run.sh b/awsf3-docker/run.sh index 0789770b2..8060e0e3a 100755 --- a/awsf3-docker/run.sh +++ b/awsf3-docker/run.sh @@ -54,7 +54,7 @@ export LOCAL_WF_TMPDIR_CWL=$MOUNT_DIR_PREFIX$LOCAL_WF_TMPDIR # function that executes a command and collecting log exl(){ $@ >> $LOGFILE 2>> $LOGFILE; handle_error $?; } ## usage: exl command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. -exlj(){ $@ >> $LOGJSONFILE 2>> $LOGFILE; $ERRCODE=$?; cat $LOGJSONFILE >> $LOGFILE; handle_error $ERRCODE; } ## usage: exl command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. This one separates stdout to json as well. +exlj(){ $@ >> $LOGJSONFILE 2>> $LOGFILE; ERRCODE=$?; cat $LOGJSONFILE >> $LOGFILE; handle_error $ERRCODE; } ## usage: exl command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. This one separates stdout to json as well. exle(){ $@ >> /dev/null 2>> $LOGFILE; handle_error $?; } ## usage: exle command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. This one eats stdout. Useful for downloading/uploading files to/from s3, because it writes progress to stdout. exlo(){ $@ 2>> /dev/null >> $LOGFILE; handle_error $?; } ## usage: exlo command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. This one eats stderr. Useful for hiding long errors or credentials. From 238dfa8ceaff3488a15d1fb72b37949b2f2c5d15 Mon Sep 17 00:00:00 2001 From: Soo Lee Date: Wed, 25 Aug 2021 13:45:10 -0400 Subject: [PATCH 3/6] Update aws_run_workflow_generic.sh --- awsf3/aws_run_workflow_generic.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awsf3/aws_run_workflow_generic.sh b/awsf3/aws_run_workflow_generic.sh index b04e3a4e0..8c04c7e99 100755 --- a/awsf3/aws_run_workflow_generic.sh +++ b/awsf3/aws_run_workflow_generic.sh @@ -172,6 +172,8 @@ exl echo "## Running dockerized awsf scripts" send_log # run dockerized awsf scripts +exl docker pull $AWSF_IMAGE +send_log docker run --privileged --net host -v /home/ubuntu/:/home/ubuntu/:rw -v /mnt/:/mnt/:rw $AWSF_IMAGE run.sh -i $JOBID -l $LOGBUCKET -f $EBS_DEVICE -S $STATUS $SINGULARITY_OPTION_TO_PASS handle_error $? From 431801a29fa14f926135e1216d1ce8dfb89a0cfd Mon Sep 17 00:00:00 2001 From: Soo Lee Date: Wed, 25 Aug 2021 13:59:16 -0400 Subject: [PATCH 4/6] Update aws_run_workflow_generic.sh --- awsf3/aws_run_workflow_generic.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/awsf3/aws_run_workflow_generic.sh b/awsf3/aws_run_workflow_generic.sh index 8c04c7e99..35685cba1 100755 --- a/awsf3/aws_run_workflow_generic.sh +++ b/awsf3/aws_run_workflow_generic.sh @@ -163,6 +163,7 @@ fi ### log into ECR if necessary exl echo exl echo "## Logging into ECR" +exl echo "Logging into ECR $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com..." exlo docker login --username AWS --password $(aws ecr get-login-password --region $INSTANCE_REGION) $AWS_ACCOUNT_ID.dkr.ecr.$INSTANCE_REGION.amazonaws.com; send_log From cd1bbf1d82ebf4285ba6fd27dc0c15e10491cd6a Mon Sep 17 00:00:00 2001 From: Soo Lee Date: Wed, 25 Aug 2021 14:30:17 -0400 Subject: [PATCH 5/6] Update aws_run_workflow_generic.sh --- awsf3/aws_run_workflow_generic.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/awsf3/aws_run_workflow_generic.sh b/awsf3/aws_run_workflow_generic.sh index 35685cba1..c6d980912 100755 --- a/awsf3/aws_run_workflow_generic.sh +++ b/awsf3/aws_run_workflow_generic.sh @@ -52,6 +52,7 @@ export AWS_ACCOUNT_ID=$(aws sts get-caller-identity| grep Account | sed 's/[^0-9 # function that executes a command and collecting log exl(){ $@ >> $LOGFILE 2>> $LOGFILE; handle_error $?; } ## usage: exl command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. +exlo(){ $@ 2>> /dev/null >> $LOGFILE; handle_error $?; } ## usage: exlo command ## ERRCODE has the error code for the command. if something is wrong, send error to s3. This one eats stderr. Useful for hiding long errors or credentials. # function that sends log to s3 (it requires LOGBUCKET to be defined, which is done by sourcing $ENV_FILE.) send_log(){ aws s3 cp $LOGFILE s3://$LOGBUCKET &>/dev/null; } ## usage: send_log (no argument) From c3e319ad0e18acbdc3e3b93567709f48cbdc8ecb Mon Sep 17 00:00:00 2001 From: Soo Lee Date: Wed, 25 Aug 2021 15:15:44 -0400 Subject: [PATCH 6/6] Update _version.py --- tibanna/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tibanna/_version.py b/tibanna/_version.py index 4a14e0297..957e0ea30 100755 --- a/tibanna/_version.py +++ b/tibanna/_version.py @@ -1,4 +1,4 @@ """Version information.""" # The following line *must* be the last in the module, exactly as formatted: -__version__ = "1.4.1" +__version__ = "1.5.0"