diff --git a/scripts/submit_csv.sh b/scripts/submit_csv.sh index e5c69d7..7960a65 100755 --- a/scripts/submit_csv.sh +++ b/scripts/submit_csv.sh @@ -28,12 +28,32 @@ shift TARGET=${1:-2} shift +# environment variable to indicate whether the job is running on condor or slurm +SYSTEM=${SYSTEM:-condor} + # process csv file into jobs -CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET}) +if [ -n "${CSV_FILE:-}" ]; then + # allow to set custom csv file for job instead of fetching from web archive + CSV_FILE=$(realpath -e ${CSV_FILE}) +else + CSV_FILE=$($(dirname $0)/csv_to_chunks.sh ${FILE} ${TARGET}) +fi # create command line -EXECUTABLE="./scripts/run.sh" -ARGUMENTS="${TYPE} EVGEN/\$(file).\$(ext) \$(nevents) \$(ichunk)" +EXECUTABLE="$PWD/scripts/run.sh" +if [ ${SYSTEM} = "condor" ]; then + ARGUMENTS="${TYPE} EVGEN/\$(file).\$(ext) \$(nevents) \$(ichunk)" +elif [ ${SYSTEM} = "slurm" ]; then + ARGUMENTS="${TYPE}" + # FIXME: This is not ideal. It prevents from submitting multiple jobs with different JUG_XL_TAG simultaneously. + cd scripts + wget --output-document install.sh https://get.epic-eic.org + sed -i 's/nightly/${JUG_XL_TAG}/g' install.sh + bash install.sh + cd .. +else + echo "Enter a valid SYSTEM value (condor or slurm)" +fi # construct environment file ENVIRONMENT=environment-$(date --iso-8601=minutes).sh @@ -54,8 +74,12 @@ REQUIREMENTS="" # construct input files INPUT_FILES=${ENVIRONMENT} +# calculate number of jobs being submitted +NJOBS=$( wc -l ${CSV_FILE} | awk '{print $1}' ) + # construct submission file SUBMIT_FILE=$(basename ${CSV_FILE} .csv).submit + sed " s|%EXECUTABLE%|${EXECUTABLE}|g; s|%ARGUMENTS%|${ARGUMENTS}|g; @@ -65,14 +89,34 @@ sed " s|%INPUT_FILES%|${INPUT_FILES}|g; s|%REQUIREMENTS%|${REQUIREMENTS}|g; s|%CSV_FILE%|${CSV_FILE}|g; + s|%ACCOUNT%|${ACCOUNT:?Define ACCOUNT with the slurm account}|g; + s|%CAMPAIGN_LOG%|${CAMPAIGN_LOG:-$PWD}|g; + s|%TARGET%|$TARGET|g; + s|%NJOBS%|${NJOBS}|g; " templates/${TEMPLATE}.submit.in > ${SUBMIT_FILE} # submit job -condor_submit -verbose -file ${SUBMIT_FILE} -# create log dir -if [ $? -eq 0 ] ; then - for i in `condor_q | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do - mkdir -p LOG/CONDOR/osg_$i/ - done +echo "Submitting ${NJOBS} to a ${SYSTEM} system" + +if [ ${SYSTEM} = "condor" ]; then + condor_submit -verbose -file ${SUBMIT_FILE} + + # create log dir + if [ $? -eq 0 ] ; then + for i in `condor_q | grep ^${USER} | tail -n1 | awk '{print($NF)}' | cut -d. -f1` ; do + mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/CONDOR/osg_$i/ + done + fi +elif [ ${SYSTEM} = "slurm" ]; then + sbatch ${SUBMIT_FILE} + + # create log dir + if [ $? -eq 0 ] ; then + for i in `squeue -u ${USER} | tail -n1 | awk '{$1=$1; print}' | awk -F" |_" '{print($1)}' | cut -d. -f1` ; do + mkdir -p ${CAMPAIGN_LOG:-$PWD}/LOG/SLURM/slurm_$i/ + done + fi +else + echo "Enter a valid SYSTEM value (condor or slurm)" fi diff --git a/templates/narval_csv.sh.in b/templates/narval_csv.sh.in new file mode 100644 index 0000000..310b593 --- /dev/null +++ b/templates/narval_csv.sh.in @@ -0,0 +1,8 @@ +export S3_ACCESS_KEY=%S3_ACCESS_KEY% +export S3_SECRET_KEY=%S3_SECRET_KEY% +export S3RW_ACCESS_KEY=%S3RW_ACCESS_KEY% +export S3RW_SECRET_KEY=%S3RW_SECRET_KEY% +export DETECTOR_VERSION=%DETECTOR_VERSION% +export DETECTOR_CONFIG=%DETECTOR_CONFIG% +export EBEAM=%EBEAM% +export PBEAM=%PBEAM% diff --git a/templates/narval_csv.submit.in b/templates/narval_csv.submit.in new file mode 100644 index 0000000..7352b6a --- /dev/null +++ b/templates/narval_csv.submit.in @@ -0,0 +1,22 @@ +#!/bin/bash +#SBATCH --account=%ACCOUNT% +#SBATCH --array=1-%NJOBS% +#SBATCH --ntasks=1 # number of MPI processes +#SBATCH --mem-per-cpu=3G # memory; default unit is megabytes +#SBATCH --time=0-%TARGET%:00 # time (DD-HH:MM) +#SBATCH --output=%CAMPAIGN_LOG%/LOG/SLURM/slurm_%A/slurm_%A_%a.out # standard output log +#SBATCH --error=%CAMPAIGN_LOG%/LOG/SLURM/slurm_%A/slurm_%A_%a.err # standard error log + +input_string=$(sed "${SLURM_ARRAY_TASK_ID}q;d" %CSV_FILE%) +read -r file ext nevents ichunk <<< $(echo "$input_string" | awk -F',' '{print $1, $2, $3, $4}') + +echo $file +echo $ext +echo $nevents +echo $ichunk + +cat << EOF | $(dirname %EXECUTABLE%)/eic-shell + +%EXECUTABLE% %ARGUMENTS% EVGEN/${file}.${ext} ${nevents} ${ichunk} + +EOF