From 433eab43a6969d525a914155f0b0ceafb663d228 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Sat, 18 Nov 2023 12:06:12 -0600 Subject: [PATCH 1/8] Adapt for slurm --- scripts/submit_csv.sh | 65 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/scripts/submit_csv.sh b/scripts/submit_csv.sh index e5c69d7..84764f6 100755 --- a/scripts/submit_csv.sh +++ b/scripts/submit_csv.sh @@ -28,12 +28,32 @@ shift TARGET=${1:-2} shift +# environment variable to indicate whether the job is running on condor or slurm +SYSTEM=${SYSTEM:-condor} + # process csv file into jobs -CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET}) +if [ -n "${CSV_FILE:-}" ]; then + # allow to set custom csv file for job instead of fetching from web archive + CSV_FILE=$(realpath -e ${CSV_FILE}) +else + CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET}) +fi # create command line -EXECUTABLE="./scripts/run.sh" -ARGUMENTS="${TYPE} EVGEN/\$(file).\$(ext) \$(nevents) \$(ichunk)" +EXECUTABLE="$PWD/scripts/run.sh" +if [ ${SYSTEM} = "condor" ]; then + ARGUMENTS="${TYPE} EVGEN/\$(file).\$(ext) \$(nevents) \$(ichunk)" +elif [ ${SYSTEM} = "slurm" ]; then + ARGUMENTS="${TYPE}" + # FIXME: This is not ideal. It prevents from submitting multiple jobs with different JUG_XL_TAG simultaneously. + cd scripts + wget --output-document install.sh https://get.epic-eic.org + sed -i 's/nightly/${JUG_XL_TAG}/g' install.sh + bash install.sh + cd .. +else + echo "Enter a valid SYSTEM value (condor or slurm)" +fi # construct environment file ENVIRONMENT=environment-$(date --iso-8601=minutes).sh @@ -54,8 +74,12 @@ REQUIREMENTS="" # construct input files INPUT_FILES=${ENVIRONMENT} +# calculate number of jobs being submitted +NJOBS=$( wc -l ${CSV_FILE} | awk '{print $1}' ) + # construct submission file SUBMIT_FILE=$(basename ${CSV_FILE} .csv).submit + sed " s|%EXECUTABLE%|${EXECUTABLE}|g; s|%ARGUMENTS%|${ARGUMENTS}|g; @@ -65,9 +89,44 @@ sed " s|%INPUT_FILES%|${INPUT_FILES}|g; s|%REQUIREMENTS%|${REQUIREMENTS}|g; s|%CSV_FILE%|${CSV_FILE}|g; + s|%ACCOUNT%|${ACCOUNT:-rrg-wdconinc}|g; + s|%CAMPAIGN_LOG%|${CAMPAIGN_LOG:-$PWD}|g; + s|%TARGET%|$TARGET|g; + s|%NJOBS%|${NJOBS}|g; " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE} # submit job + +echo "Submitting ${NJOBS} to a ${SYSTEM} system" + +if [ ${SYSTEM} = "condor" ]; then + condor_submit -verbose -file ${SUBMIT_FILE} + + # create log dir + if [ $? -eq 0 ] ; then + for i in `condor_q | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do + mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/CONDOR/osg_$i/ + done + fi +elif [ ${SYSTEM} = "slurm" ]; then + sbatch ${SUBMIT_FILE} + + # create log dir + if [ $? -eq 0 ] ; then + for i in `squeue -u ${USER} | tail -n1 | awk -F" |_" '{print($1)}' | cut -d. -f1` ; do + mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/SLURM/slurm_$i/ + done + fi +else + echo "Enter a valid SYSTEM value (condor or slurm)" +fi + + + + + + + condor_submit -verbose -file ${SUBMIT_FILE} # create log dir From 63c7697678259ec36dec874f26bf4ef553dfe8ca Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Sat, 18 Nov 2023 12:07:22 -0600 Subject: [PATCH 2/8] Create narval_csv.submit.in --- templates/narval_csv.submit.in | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 templates/narval_csv.submit.in diff --git a/templates/narval_csv.submit.in b/templates/narval_csv.submit.in new file mode 100644 index 0000000..8eaf72c --- /dev/null +++ b/templates/narval_csv.submit.in @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --account=%ACCOUNT% +#SBATCH --array=1-%NJOBS% +#SBATCH --ntasks=1 # number of MPI processes +#SBATCH --mem-per-cpu=3G # memory; default unit is megabytes +#SBATCH --time=0-%TARGET%:00 # time (DD-HH:MM) +#SBATCH --output=%CAMPAIGN_LOG%/LOG/SLURM/slurm_%A/slurm_%A_%a.out # standard output log +#SBATCH --error=%CAMPAIGN_LOG%/LOG/SLURM/slurm_%A/slurm_%A_%a.err # standard error log + +input_string=$(sed "${SLURM_ARRAY_TASK_ID}q;d" %CSV_FILE%) +read -r file ext nevents ichunk <<< $(echo "$input_string" | awk -F',' '{print $1, $2, $3, $4}') + +echo $file +echo $ext +echo $nevents +echo $ichunk + +cat << EOF | $(dirname %EXECUTABLE%)/eic-shell + +%EXECUTABLE% %ARGUMENTS% $file $ext $nevents $ichunk + +EOF From c27c221feb8a512259c14de110ab37885dd1d802 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Sat, 18 Nov 2023 12:08:22 -0600 Subject: [PATCH 3/8] Adapt for narval --- templates/narval_csv.sh.in | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 templates/narval_csv.sh.in diff --git a/templates/narval_csv.sh.in b/templates/narval_csv.sh.in new file mode 100644 index 0000000..310b593 --- /dev/null +++ b/templates/narval_csv.sh.in @@ -0,0 +1,8 @@ +export S3_ACCESS_KEY=%S3_ACCESS_KEY% +export S3_SECRET_KEY=%S3_SECRET_KEY% +export S3RW_ACCESS_KEY=%S3RW_ACCESS_KEY% +export S3RW_SECRET_KEY=%S3RW_SECRET_KEY% +export DETECTOR_VERSION=%DETECTOR_VERSION% +export DETECTOR_CONFIG=%DETECTOR_CONFIG% +export EBEAM=%EBEAM% +export PBEAM=%PBEAM% From 3bdc1f27d36345c7b6ea751cf0ccab45ae36e9b6 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Sat, 18 Nov 2023 13:48:57 -0600 Subject: [PATCH 4/8] Update submit_csv.sh --- scripts/submit_csv.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/scripts/submit_csv.sh b/scripts/submit_csv.sh index 84764f6..ab83b8b 100755 --- a/scripts/submit_csv.sh +++ b/scripts/submit_csv.sh @@ -120,18 +120,3 @@ elif [ ${SYSTEM} = "slurm" ]; then else echo "Enter a valid SYSTEM value (condor or slurm)" fi - - - - - - - -condor_submit -verbose -file ${SUBMIT_FILE} - -# create log dir -if [ $? -eq 0 ] ; then - for i in `condor_q | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do - mkdir -p LOG/CONDOR/osg_$i/ - done -fi From f7729effbc706083c246d9be2b930fd2a3eb8e9e Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Mon, 20 Nov 2023 23:25:03 -0600 Subject: [PATCH 5/8] Strip leading and trailing white spaces before parsing --- scripts/submit_csv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/submit_csv.sh b/scripts/submit_csv.sh index ab83b8b..fff4f9b 100755 --- a/scripts/submit_csv.sh +++ b/scripts/submit_csv.sh @@ -113,7 +113,7 @@ elif [ ${SYSTEM} = "slurm" ]; then # create log dir if [ $? -eq 0 ] ; then - for i in `squeue -u ${USER} | tail -n1 | awk -F" |_" '{print($1)}' | cut -d. -f1` ; do + for i in `squeue -u ${USER} | tail -n1 | awk '{$1=$1; print}' | awk -F" |_" '{print($1)}' | cut -d. -f1` ; do mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/SLURM/slurm_$i/ done fi From 522f3cf8df3dd7071473d53787fbca9e339ba93c Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Tue, 21 Nov 2023 13:04:29 -0600 Subject: [PATCH 6/8] Need to append extension with filename --- templates/narval_csv.submit.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/narval_csv.submit.in b/templates/narval_csv.submit.in index 8eaf72c..e55933e 100644 --- a/templates/narval_csv.submit.in +++ b/templates/narval_csv.submit.in @@ -17,6 +17,6 @@ echo $ichunk cat << EOF | $(dirname %EXECUTABLE%)/eic-shell -%EXECUTABLE% %ARGUMENTS% $file $ext $nevents $ichunk +%EXECUTABLE% %ARGUMENTS% ${file}.${ext} ${nevents} ${ichunk} EOF From f6693e2ead504174d3842444f1e7e61c300bbc4a Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Tue, 21 Nov 2023 13:07:26 -0600 Subject: [PATCH 7/8] Needs EVGEN in directory path --- templates/narval_csv.submit.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/narval_csv.submit.in b/templates/narval_csv.submit.in index e55933e..7352b6a 100644 --- a/templates/narval_csv.submit.in +++ b/templates/narval_csv.submit.in @@ -17,6 +17,6 @@ echo $ichunk cat << EOF | $(dirname %EXECUTABLE%)/eic-shell -%EXECUTABLE% %ARGUMENTS% ${file}.${ext} ${nevents} ${ichunk} +%EXECUTABLE% %ARGUMENTS% EVGEN/${file}.${ext} ${nevents} ${ichunk} EOF From 84f708a150c5c2e042cd74cd645d385b2f123cf9 Mon Sep 17 00:00:00 2001 From: Sakib Rahman Date: Tue, 21 Nov 2023 13:22:30 -0600 Subject: [PATCH 8/8] Crash if account name not specified --- scripts/submit_csv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/submit_csv.sh b/scripts/submit_csv.sh index fff4f9b..7960a65 100755 --- a/scripts/submit_csv.sh +++ b/scripts/submit_csv.sh @@ -89,7 +89,7 @@ sed " s|%INPUT_FILES%|${INPUT_FILES}|g; s|%REQUIREMENTS%|${REQUIREMENTS}|g; s|%CSV_FILE%|${CSV_FILE}|g; - s|%ACCOUNT%|${ACCOUNT:-rrg-wdconinc}|g; + s|%ACCOUNT%|${ACCOUNT:?Define ACCOUNT with the slurm account}|g; s|%CAMPAIGN_LOG%|${CAMPAIGN_LOG:-$PWD}|g; s|%TARGET%|$TARGET|g; s|%NJOBS%|${NJOBS}|g;