From c4c2fe76c11204ae8fada4a3328181d7ea0b3927 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Tue, 8 Oct 2019 17:53:07 -0400 Subject: [PATCH 01/23] Implement a uniform run control object --- HEN_HOUSE/egs++/egs_application.cpp | 80 +++++++++--- HEN_HOUSE/egs++/egs_application.h | 18 +++ HEN_HOUSE/egs++/egs_run_control.cpp | 189 +++++++++++++++++++++++++++- HEN_HOUSE/egs++/egs_run_control.h | 107 ++++++++++++++-- 4 files changed, 371 insertions(+), 23 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_application.cpp b/HEN_HOUSE/egs++/egs_application.cpp index c4159c3de..c172f0567 100644 --- a/HEN_HOUSE/egs++/egs_application.cpp +++ b/HEN_HOUSE/egs++/egs_application.cpp @@ -73,6 +73,9 @@ using namespace std; #include #endif +// #define MAXIMUM_JOB_NUMBER 8192 // GPSC1: 256 nodes with 16 cores (32 threads) +#define MAXIMUM_JOB_NUMBER 1024 + static char __egs_app_msg1[] = "EGS_Application::EGS_Application(int,char**):"; static char __egs_app_msg2[] = "EGS_Application::initSimulation():"; static char __egs_app_msg3[] = "EGS_Application::runSimulation():"; @@ -389,22 +392,30 @@ EGS_Application::EGS_Application(int argc, char **argv) : input(0), geometry(0), i_parallel = 0; } } - else if (have_np || have_ip) - egsWarning("%s\n to specify a parallel run you need both," - " the -P and -j command line options\n",__egs_app_msg1); + else if (have_np && !have_ip) { // user wants to reset n_parallel + // and combine parallel jobs + n_parallel = ::strtol(npar.c_str(),0,10); + simple_run = true; + } + else if (n_parallel && !have_ip) { // user wants to combine + // parallel jobs + simple_run = true; + } + else if (!have_np && have_ip) { + egsWarning("\n%s\n to specify a parallel run you need both," + " the -P and -j command line options\n\n",__egs_app_msg1); + } // // *** see if user wants simple job control // - { - for (int j=1; jreportResults(); + } + outputResults(); + return 0; +} + int EGS_Application::combineResults() { egsInformation( "\n Suming the following .egsdat files:\n" @@ -571,8 +615,16 @@ int EGS_Application::combineResults() { EGS_I64 last_ncase = 0; int ndat = 0; bool ok = true; - for (int j=1; j<500; j++) { - sprintf(buf,"%s_w%d.egsdat",output_file.c_str(),j); + /* + If trying to combine results and n_parallel set to 0, + use a hard-coded value for number of jobs.This is possible + if -P njobs was not passed as argument. + */ + if (!n_parallel) { + n_parallel = MAXIMUM_JOB_NUMBER; + } + for (int j=first_parallel; j < first_parallel + n_parallel; j++) { + sprintf(buf,"%s_w%d.egsdat",final_output_file.c_str(),j); string dfile = egsJoinPath(app_dir,buf); ifstream data(dfile.c_str()); if (data) { diff --git a/HEN_HOUSE/egs++/egs_application.h b/HEN_HOUSE/egs++/egs_application.h index 527513719..52ec3ad6a 100644 --- a/HEN_HOUSE/egs++/egs_application.h +++ b/HEN_HOUSE/egs++/egs_application.h @@ -419,6 +419,24 @@ class EGS_EXPORT EGS_Application { */ virtual int combineResults(); + /*! \brief Combine intermediate results from parallel runs. + + Calls combineResults, followed by the output of intermediate + results. Currently used by the uniform RCO while a watcher job + is waiting for all jobs to complete. + */ + virtual int combinePartialResults(); + + /*! \brief Counts how many *.egsdat files in app folder. + + Used by the uniform RCO to estimate how many parallel runs + completed. This RCO initially deletes existing *.egsdat files + to avoid counting files from previous runs. It is an estimate + since some jobs might have failed. + */ + int howManyJobsDone(); + + /*! \brief Output intermediate results. This function stores the state of the application to a data diff --git a/HEN_HOUSE/egs++/egs_run_control.cpp b/HEN_HOUSE/egs++/egs_run_control.cpp index 16ac25b48..8725c358b 100644 --- a/HEN_HOUSE/egs++/egs_run_control.cpp +++ b/HEN_HOUSE/egs++/egs_run_control.cpp @@ -25,6 +25,7 @@ # # Contributors: Frederic Tessier # Hubert Ho +# Ernesto Mainegra-Hing # ############################################################################### */ @@ -44,6 +45,7 @@ #include #include #include + using namespace std; vector rc_libs; @@ -51,7 +53,8 @@ static int n_run_controls = 0; EGS_RunControl::EGS_RunControl(EGS_Application *a) : geomErrorCount(0), geomErrorMax(0), app(a), input(0), ncase(0), ndone(0), maxt(-1), accu(-1), - nbatch(10), restart(0), nchunk(1), cpu_time(0), previous_cpu_time(0) { + nbatch(10), restart(0), nchunk(1), cpu_time(0), previous_cpu_time(0), + rco_type(simple) { n_run_controls++; if (!app) egsFatal("EGS_RunControl::EGS_RunControl: it is not allowed\n" " to construct a run control object on a NULL application\n"); @@ -76,6 +79,9 @@ EGS_RunControl::EGS_RunControl(EGS_Application *a) : geomErrorCount(0), "'number of histories' input\n"); } ncase = EGS_I64(ncase_double); + if (app->getNparallel()) { + ncase /= app->getNparallel(); + } err = input->getInput("nbatch",nbatch); if (err) { nbatch = 10; @@ -92,6 +98,7 @@ EGS_RunControl::EGS_RunControl(EGS_Application *a) : geomErrorCount(0), if (err) { geomErrorMax = 0; } + vector ctype; ctype.push_back("first"); ctype.push_back("restart"); @@ -234,6 +241,102 @@ bool EGS_RunControl::finishBatch() { return true; } +EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : + EGS_RunControl(a), njob(0), npar(app->getNparallel()), + ipar(app->getIparallel()), ifirst(app->getFirstParallel()), + milliseconds(1000), check_intervals(5), check_egsdat(false), + watcher_job(false) { + + rco_type = uniform; + +// Not needed: Done now in the base EGS_RunControl constructor +// if (npar){ +// ncase /= npar; +// } + + if (input) { + + /*Change waiting time to check for parallel run completion*/ + int dummy; + int err = input->getInput("interval wait time", dummy); + if (!err) { + milliseconds = dummy; + } + + /*Change how many times to check for parallel run completion*/ + err = input->getInput("number of intervals", dummy); + if (!err) { + check_intervals = dummy; + } + + /* Define watcher jobs to check for parallel run completion*/ + vector w_jobs; + err = input->getInput("watcher jobs", w_jobs); + if (!err) { + for (int i = 0; i < w_jobs.size(); i++) { + if (ipar == w_jobs[i]) { + watcher_job = true; + break; + } + } + } + else { // use defaults + /* last job is watcher job */ + if (ipar == ifirst + npar - 1) { + watcher_job = true; + } + else { + watcher_job = false; + } + } + + + /* Request checking parallel run completion */ + vector check_options; + check_options.push_back("no"); + check_options.push_back("yes"); + int ichk = input->getInput("check jobs completed",check_options,0); + if (ichk != 0) { + check_egsdat = true; // false by default + } + + } +} + +int EGS_UniformRunControl::startSimulation() { + + egsInformation("\n\n-> RCO is of type %d\n", rco_type); + if (watcher_job) { + egsInformation(" I am a watcher job! \n"); + if (check_egsdat) { + egsInformation("And will check for parallel run completion every %d s for %d s!\n", + milliseconds/1000, check_intervals*milliseconds/1000); + } + else { + egsInformation("\n\n"); + } + } + + /* Check run completion based on *egsdat files requires erasing + existing files from previous runs. + */ + if (check_egsdat) { + char buf[512]; + sprintf(buf,"%s_w%d.egsdat",app->getFinalOutputFile().c_str(), ipar); + string datFile = egsJoinPath(app->getAppDir(),buf); + if (remove(datFile.c_str()) == 0) { + egsWarning("EGS_UniformRunControl: %s deleted\n", + datFile.c_str()); + } + else { + egsWarning("EGS_UniformRunControl: %s not found!\n", + datFile.c_str()); + } + } + + return EGS_RunControl::startSimulation(); +} + #ifdef WIN32 #include @@ -242,6 +345,7 @@ bool EGS_RunControl::finishBatch() { #include #include #include + #include #define OPEN_FILE _open #define CLOSE_FILE _close @@ -421,6 +525,9 @@ EGS_JCFControl::EGS_JCFControl(EGS_Application *a, int Nbuf) : last_sum2(0), last_count(0), njob(0), npar(app->getNparallel()), ipar(app->getIparallel()), ifirst(app->getFirstParallel()), first_time(true), removed_jcf(false), nbuf(Nbuf), p(new EGS_FileLocking) { + + rco_type = balanced; + if (input) { int err = input->getInput("nchunk",nchunk); if (err) { @@ -660,6 +767,27 @@ EGS_I64 EGS_JCFControl::getNextChunk() { return nrun; } +/*! \brief Suspend execution for a given time (in ms) + + Called from the uniform RCO to wait for all jobs to + finish. Time is set by default to 1s, but user can + change it using the input key + + interval wait time = time in ms + + in the run control input block. + + +void rco_sleep(const int& mscnds); +*/ +void rco_sleep(const int &mscnds) { +#ifdef WIN32 + Sleep(mscnds); +#else + usleep(mscnds * 1000); +#endif +} + int EGS_RunControl::finishSimulation() { cpu_time = timer.time(); egsInformation("\n\nFinished simulation\n\n"); @@ -683,6 +811,43 @@ int EGS_RunControl::finishSimulation() { // all_steps); egsInformation("%-40s","Number of all electron steps:"); egsInformation("%-14g\n",all_steps); + + int n_par = app->getNparallel(), + i_par = app->getIparallel(), + i_first = app->getFirstParallel(); + /* If parallel run and last job, trigger the app combineResults method */ + return (n_par && i_par == i_first + n_par - 1) ? 1 : 0; +} + +int EGS_UniformRunControl::finishSimulation() { + int err = EGS_RunControl::finishSimulation(); + if (err < 0) { + return err; + } + /* Check and wait for all jobs to finish */ + if (watcher_job) { + int interval = 0, njobs_done = 0, njobs_done_old= 0; + while (interval < check_intervals) { + rco_sleep(milliseconds); + if (check_egsdat) { + njobs_done = app->howManyJobsDone(); + //egsInformation("\n-> Finished %d jobs...\n",njobs_done); + if (njobs_done == npar - 1) { + watcher_job=false;//don't enter this after all jobs done! + break; + } + // Only combine if new jobs finished + if (njobs_done_old < njobs_done) { + egsInformation("=> Combining %d jobs ...\n",njobs_done); + app->combinePartialResults(); + } + njobs_done_old = njobs_done; + } + interval++; + } + return 1; + } + /*I am not a watcher job, do not combine results yet!*/ return 0; } @@ -749,6 +914,9 @@ EGS_RunControl *EGS_RunControl::getRunControlObject(EGS_Application *a) { if (inp) { irc = inp->getInputItem("run control"); } + /* If no input file, defaults to simple RCO for single runs and + to JCF RCO for parallel runs. + */ if (!irc) { /* egsWarning("EGS_RunControl::getRunControlObject(): " @@ -810,7 +978,24 @@ EGS_RunControl *EGS_RunControl::getRunControlObject(EGS_Application *a) { } else { if (a->getNparallel() > 0) { - result = new EGS_JCFControl(a); + vector allowed_types; + allowed_types.push_back("simple"); + allowed_types.push_back("uniform"); + allowed_types.push_back("balanced"); + int rco_t = irc->getInput("rco type",allowed_types,2); + switch (rco_t) { + case 0: + result = new EGS_RunControl(a); + break; + case 1: + result = new EGS_UniformRunControl(a); + break; + case 2: + result = new EGS_JCFControl(a); + break; + default: + result = new EGS_JCFControl(a); + } } else { result = new EGS_RunControl(a); diff --git a/HEN_HOUSE/egs++/egs_run_control.h b/HEN_HOUSE/egs++/egs_run_control.h index 9ee204d8f..c9b3fba83 100644 --- a/HEN_HOUSE/egs++/egs_run_control.h +++ b/HEN_HOUSE/egs++/egs_run_control.h @@ -61,7 +61,7 @@ class EGS_Input;

Terminology - Simulations are split into 'chunks'. For simple simulations - (no parallel runs, etc.) there is a single simulatioin chunk + (no parallel runs, etc.) there is a single simulation chunk with the number of histories specified in the input file. - Each simulation chunk is split into 'batches'. The batches are not required for statistical analysis (by using the provided @@ -72,7 +72,7 @@ class EGS_Input; completion of a batch and the current results can be stored into a data file. By default there are 10 batches per simulation chunk -

Two RCO's are provided with egspp: +

Three RCO's are provided with egspp: - A 'simple' RCO implemented in EGS_RunControl. This RCO is used by default for single job control. This RCO provides the ability to run simulations with a user specified number of particles and up to a user specified @@ -81,9 +81,14 @@ class EGS_Input; simulations or to simply analyze the results of a previous run or to combine the results of previously performed parallel runs. - A \link EGS_JCFControl 'job control file' (JCF) RCO \endlink. - This RCO is used by default for parallel - runs. It has all the functionality of the 'simple' RCO plus additional - methods to contyrol parallel execution via a 'job control file'. + This RCO is used by default for parallel runs. It has all the functionality + of the 'simple' RCO plus additional methods to control parallel execution + via a 'job control file'. + - A \link EGS_UniformRunControl uniform RCO \endlink. + This RCO distributes the histories uniformly among all jobs. + It can be an alternative to the JCF RCO when a JCF cannot be used. This can + happen when jobs do not start sequentially. In such cases the JCF may not be + available if job number 1 is not started first. */ class EGS_EXPORT EGS_RunControl { @@ -161,7 +166,7 @@ class EGS_EXPORT EGS_RunControl { return getNcase() - ndone; }; - /*! Finish the simulation. + /*! \brief Finish the simulation. This function is called from within the finishSimulation() method of EGS_Application and should return 1 of the follwoing 3 exit codes: @@ -200,16 +205,26 @@ class EGS_EXPORT EGS_RunControl { virtual EGS_I64 getNdone() const { return ndone; }; + virtual void setNdone(EGS_I64 Ndone) { ndone = Ndone; }; + virtual void incrementNdone() { ++ndone; }; + virtual EGS_Float getCPUTime() const { return cpu_time+previous_cpu_time; }; + /*! \brief Define RCO types */ + enum RCOType { + simple, //!< single job or multiple independent jobs + uniform, //!< parallel jobs with same numbe of histories + balanced //!< parallel jobs with balanced load via JCF + }; + static EGS_RunControl *getRunControlObject(EGS_Application *); int geomErrorCount, geomErrorMax; @@ -230,6 +245,8 @@ class EGS_EXPORT EGS_RunControl { // =3 => combine parallel run int nchunk; // number of simulation "chunks" + RCOType rco_type; //!< RCO type to use + EGS_Timer timer; EGS_Float cpu_time; EGS_Float previous_cpu_time; @@ -248,7 +265,7 @@ class EGS_FileLocking; parallel job and contains information such as the number of particles remaining to be simulated, number of jobs running, combined result of all parallel jobs, etc. This RCO objects requires that file locking - works on the file system containing the \c EGS_HOME directory becuase + works on the file system containing the \c EGS_HOME directory because the JCF is locked prior to being modified by one of the jobs to prevent multiply jobs modifying the file at the same time. For more details see PIRS-877. @@ -301,5 +318,81 @@ class EGS_EXPORT EGS_JCFControl : public EGS_RunControl { }; +/*! \brief A job control object for homogeneous computing environments (HCE). + + \ingroup egspp_main + + The uniform RCO is used for controlling parallel job execution + in computing environments (CE) with identical hardware, software + and communication layer (aka homogeneous CE): + + - Number of histories 'ncase' split equally among all jobs. + + - Assume last job finishes last and then cycles 5 times by default + ('check_intervals' variable) for a period of time defined to + be 1 s by default ('milliseconds' variable). Defaults can be changed + via the 'run control' input block using: + + interval wait time = time in ms (default 1 s) + number of intervals = an_integer_value (default 5) + + - The last job combines the parallel runs by default. Since the last job + could finish before some of the other jobs, users can set another + job or several jobs to be 'watcher' jobs. In principle it is enough to + define one 'watcher' job that waits long enough for all jobs to complete. + To change the default, use the following key: + + watcher jobs = job_i,..., job_j + + - If requested, a run-completion check can be made every cycle + by checking that the number of *.egsdat files equals the number + of parallel jobs submitted. This could speed things up by not having + to wait for all checking cycles. However, it could also be the case + that some jobs might have failed, in which case, after the checking + cycles complete, only the available *.egsdat files will be combined. + + This option can be set via the 'run control' input block using: + + check jobs completed = yes|no # default is 'no' + + When this option is enabled, each job erases at the beginning of the run + its corresponding *.egsdat file if it exists. + +*/ + +class EGS_EXPORT EGS_UniformRunControl : public EGS_RunControl { + +public: + + EGS_UniformRunControl(EGS_Application *app); + ~EGS_UniformRunControl() {}; + + int startSimulation(); + + /*! \brief Uses 'watcher' jobs to determine if the simulation has finished. + + If the current job is a 'watcher' job, it waits for some time before issuing + the signal to recombine all available parallel jobs. These 'watcher' jobs + can also produce intermediate results while waiting. If all jobs complete while + waiting, the 'watcher' job combines all results and exits. + + */ + int finishSimulation(); + +protected: + + int milliseconds; // time interval for checking + // if all jobs finished (default 1000 ms) + + int check_intervals;// Number of intervals to check + // if all jobs done (default 5) + + int njob; + int npar; + int ipar; + int ifirst; + bool check_egsdat;// If true, and a 'watcher' job, produce intermediate results + bool watcher_job; // If true, job is a 'watcher' +}; #endif From 5cbc8fc8b6d3a7f59e4b58484a1d0a0fa1e0c839 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 23 Mar 2020 16:00:15 -0400 Subject: [PATCH 02/23] Increase maximum number of jobs to 8192 --- HEN_HOUSE/egs++/egs_application.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_application.cpp b/HEN_HOUSE/egs++/egs_application.cpp index c172f0567..ff4449940 100644 --- a/HEN_HOUSE/egs++/egs_application.cpp +++ b/HEN_HOUSE/egs++/egs_application.cpp @@ -73,8 +73,8 @@ using namespace std; #include #endif -// #define MAXIMUM_JOB_NUMBER 8192 // GPSC1: 256 nodes with 16 cores (32 threads) -#define MAXIMUM_JOB_NUMBER 1024 +#define MAXIMUM_JOB_NUMBER 8192 // GPSC1: 256 nodes with 16 cores (32 threads) +//#define MAXIMUM_JOB_NUMBER 1024 static char __egs_app_msg1[] = "EGS_Application::EGS_Application(int,char**):"; static char __egs_app_msg2[] = "EGS_Application::initSimulation():"; From 3901664c2f71c9af692cfb131306c013233cb6df Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 23 Mar 2020 16:02:54 -0400 Subject: [PATCH 03/23] Add environment variable for density corrections In the EGSnrc shell additions, set a new environment variable dcpath to the density correction path $HEN_HOUSE/pegs4/density_corrections. --- HEN_HOUSE/scripts/egsnrc_bashrc_additions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HEN_HOUSE/scripts/egsnrc_bashrc_additions b/HEN_HOUSE/scripts/egsnrc_bashrc_additions index d24f53630..5ef7bdb2a 100644 --- a/HEN_HOUSE/scripts/egsnrc_bashrc_additions +++ b/HEN_HOUSE/scripts/egsnrc_bashrc_additions @@ -105,6 +105,8 @@ fi # the following is for testing the distribution. alias test_distribution='$HEN_HOUSE/scripts/test_egsnrcmp_distribution' +export dcpath=$HEN_HOUSE/pegs4/density_corrections + export HEN_HOUSE PATH # Now check for local system wide sh additions From 3f2f1b82a13291479946313aefe3b74d1ae57b71 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Tue, 24 Mar 2020 12:27:34 -0400 Subject: [PATCH 04/23] Change the urco default to check finished jobs Previously, the uniform run control object was set to not check for finished jobs, but that defeats the purpose! Also, clarify the watcher job information messages. --- HEN_HOUSE/egs++/egs_run_control.cpp | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_run_control.cpp b/HEN_HOUSE/egs++/egs_run_control.cpp index 8725c358b..50eabd5ec 100644 --- a/HEN_HOUSE/egs++/egs_run_control.cpp +++ b/HEN_HOUSE/egs++/egs_run_control.cpp @@ -244,7 +244,7 @@ bool EGS_RunControl::finishBatch() { EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : EGS_RunControl(a), njob(0), npar(app->getNparallel()), ipar(app->getIparallel()), ifirst(app->getFirstParallel()), - milliseconds(1000), check_intervals(5), check_egsdat(false), + milliseconds(1000), check_intervals(5), check_egsdat(true), watcher_job(false) { rco_type = uniform; @@ -293,11 +293,11 @@ EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : /* Request checking parallel run completion */ vector check_options; - check_options.push_back("no"); check_options.push_back("yes"); + check_options.push_back("no"); int ichk = input->getInput("check jobs completed",check_options,0); if (ichk != 0) { - check_egsdat = true; // false by default + check_egsdat = false; // true by default } } @@ -305,15 +305,17 @@ EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : int EGS_UniformRunControl::startSimulation() { - egsInformation("\n\n-> RCO is of type %d\n", rco_type); + egsInformation("\n\n-> Uniform run control object (URCO)\n"); if (watcher_job) { - egsInformation(" I am a watcher job! \n"); + //egsInformation( " Watcher job: remains running after completion\n"); if (check_egsdat) { - egsInformation("And will check for parallel run completion every %d s for %d s!\n", - milliseconds/1000, check_intervals*milliseconds/1000); + egsInformation( + " Watcher job: remains running after completion checking\n" + " for other jobs finishing every %d s for %d s!\n", + milliseconds/1000, check_intervals*milliseconds/1000); } else { - egsInformation("\n\n"); + egsInformation(" Option to check for finishing jobs is OFF!\n\n"); } } @@ -328,10 +330,6 @@ int EGS_UniformRunControl::startSimulation() { egsWarning("EGS_UniformRunControl: %s deleted\n", datFile.c_str()); } - else { - egsWarning("EGS_UniformRunControl: %s not found!\n", - datFile.c_str()); - } } return EGS_RunControl::startSimulation(); From dc4e55fdc4ce0cbe499016a8fd8792efd8983547 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Tue, 24 Mar 2020 14:07:40 -0400 Subject: [PATCH 05/23] Improve the description of run control objects Add a proper describeRCO method in EGS_RunControl, and call describeRCO from EGS_Application::describeSimulation. --- HEN_HOUSE/egs++/egs_application.cpp | 4 +++ HEN_HOUSE/egs++/egs_run_control.cpp | 49 +++++++++++++++++++++-------- HEN_HOUSE/egs++/egs_run_control.h | 3 ++ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_application.cpp b/HEN_HOUSE/egs++/egs_application.cpp index ff4449940..3268758aa 100644 --- a/HEN_HOUSE/egs++/egs_application.cpp +++ b/HEN_HOUSE/egs++/egs_application.cpp @@ -803,6 +803,10 @@ void EGS_Application::describeSimulation() { rndm->describeRNG(); egsInformation("\n\n"); } + if (run) { + run->describeRCO(); + egsInformation("\n\n"); + } if (a_objects_list.size() > 0) { egsInformation("The following ausgab objects are included in the simulation\n"); egsInformation("===========================================================\n\n"); diff --git a/HEN_HOUSE/egs++/egs_run_control.cpp b/HEN_HOUSE/egs++/egs_run_control.cpp index 50eabd5ec..ce27cd792 100644 --- a/HEN_HOUSE/egs++/egs_run_control.cpp +++ b/HEN_HOUSE/egs++/egs_run_control.cpp @@ -120,6 +120,23 @@ EGS_RunControl::~EGS_RunControl() { } } +void EGS_RunControl::describeRCO() { + egsInformation( + "Run Control Object (RCO):\n" + "=========================\n"); + switch (rco_type) { + case simple: + egsInformation(" type = simple\n"); + break; + case balanced: + egsInformation(" type = balanced (JCF)\n"); + break; + case uniform: + egsInformation(" type = uniform\n"); + break; + } +} + bool EGS_RunControl::storeState(ostream &data) { if (!egsStoreI64(data,ndone)) { return false; @@ -305,19 +322,6 @@ EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : int EGS_UniformRunControl::startSimulation() { - egsInformation("\n\n-> Uniform run control object (URCO)\n"); - if (watcher_job) { - //egsInformation( " Watcher job: remains running after completion\n"); - if (check_egsdat) { - egsInformation( - " Watcher job: remains running after completion checking\n" - " for other jobs finishing every %d s for %d s!\n", - milliseconds/1000, check_intervals*milliseconds/1000); - } - else { - egsInformation(" Option to check for finishing jobs is OFF!\n\n"); - } - } /* Check run completion based on *egsdat files requires erasing existing files from previous runs. @@ -335,6 +339,25 @@ int EGS_UniformRunControl::startSimulation() { return EGS_RunControl::startSimulation(); } +void EGS_UniformRunControl::describeRCO() { + + EGS_RunControl::describeRCO(); + + if (watcher_job) { + if (check_egsdat) { + egsInformation( + " Watcher job: remains running after completion checking\n" + " for other jobs finishing every %d s for %d s!\n", + milliseconds/1000, check_intervals*milliseconds/1000); + } + else { + egsInformation( + " Option to check for finishing jobs is OFF!\n\n"); + } + } + +} + #ifdef WIN32 #include diff --git a/HEN_HOUSE/egs++/egs_run_control.h b/HEN_HOUSE/egs++/egs_run_control.h index c9b3fba83..3067d649a 100644 --- a/HEN_HOUSE/egs++/egs_run_control.h +++ b/HEN_HOUSE/egs++/egs_run_control.h @@ -194,6 +194,7 @@ class EGS_EXPORT EGS_RunControl { */ virtual bool finishBatch(); + virtual void describeRCO(); virtual bool storeState(ostream &data); virtual bool setState(istream &data); virtual bool addState(istream &data); @@ -367,6 +368,8 @@ class EGS_EXPORT EGS_UniformRunControl : public EGS_RunControl { EGS_UniformRunControl(EGS_Application *app); ~EGS_UniformRunControl() {}; + void describeRCO(); + int startSimulation(); /*! \brief Uses 'watcher' jobs to determine if the simulation has finished. From 2d72f39d90f38909ee3365d2f702241be2d6ad9a Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Tue, 24 Mar 2020 14:17:06 -0400 Subject: [PATCH 06/23] Add script to submit jobs on Sun Grid Engine Add a script to spawn Sun Grid Engine (SGE) jobs on each slot (machine), which in turn submits EGSnrc jobs in each slot. By default 16 jobs per slot are submitted. The script relies on xargs, a command on Unix-like systems to build and execute commands from standard input. --- HEN_HOUSE/scripts/egs-jobsub-xargs | 418 +++++++++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100755 HEN_HOUSE/scripts/egs-jobsub-xargs diff --git a/HEN_HOUSE/scripts/egs-jobsub-xargs b/HEN_HOUSE/scripts/egs-jobsub-xargs new file mode 100755 index 000000000..6d4aa6c4b --- /dev/null +++ b/HEN_HOUSE/scripts/egs-jobsub-xargs @@ -0,0 +1,418 @@ +#! /bin/bash +############################################################################### +# +# Script to submit jobs to the Grid Engine job scheduler using xargs +# Copyright (C) 2018 National Research Council Canada +# +# This file is part of EGSnrc. +# +# EGSnrc is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# EGSnrc is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with EGSnrc. If not, see . +# +############################################################################### +# +# Author: Ernesto Mainegra-Hing, 2018 +# +# Contributors: Iwan Kawrakow +# Dave Rogers +# Frederic Tessier +# Blake Walters +# +############################################################################### +# +# Check enough arguments provided +# +my_name=`echo "$0" |sed 's,.*[\\/],,'` +if test $# -lt 3; then + cat >&2 <&2 <&2 <&2 <&2 + if { mkdir "$egs_home"; status=$?; (exit $status); }; then + : + else + echo "Failed." >&2 + exit 1 + fi + +fi + +if test ! -d "$egs_home/$app"; then + + echo "The directory '$egs_home/$app' dose not exist. Creating it." >&2 + if { mkdir "$egs_home/$app"; status=$?; (exit $status); }; then + : + else + echo "Failed." >&2 + exit 1 + fi + +fi + +executable="${egs_home}bin/$my_machine/$app" +# +# Check if executable exists +# +if test ! -x $executable; then + executable="${hen_house}bin/$my_machine/$app" + if test ! -x $executable; then + echo "No $app executable on your area or on HEN_HOUSE" >&2 + exit 1 + fi +fi +# +# Check if running pegslessly +# +if test "$pegs" = pegsless; then + command="$executable -b" +else + command="$executable -p $pegs -b" +fi + +# +# Add input file to command +# +if test "x$inp" != x; then + command="$command -i $inp" +fi + +# +# Add EGSnrc environment to command +# +command="$command -e $egs_home -H $hen_house" + +# +# Add rest of arguments to command +# +if test "x$other_args" != x; then + command="$command -i $other_args" +fi + +# +# NOT NEEDED! This goes in the extra arguments +# +# Check if no JCF desired +# +#if test "x$simple" = xyes; then +# command="$command -s" +#fi + +# Remove previous egsdat files +if test "x$rmegsdat" = xyes; then + rm -f ${egs_home}${app}/${inpf}_w*.egsdat +fi + +if test $start_job -ge 0; then + if test $stop_job -gt $start_job; then + if test $n_parallel -eq 0; then + n_parallel=`expr $stop_job - $start_job` + fi + else + stop_job=`expr $start_job + $n_parallel` + fi +fi + +inpf=`echo $inp |sed 's/.egsinp//'` +the_email=dummy@nowhere.org +# +# Check if running multiple jobs +# +if test $n_parallel -gt 0; then + command="$command -P $n_parallel" + job=$start_job + first_job=`expr $start_job + 1` + # Submit multiple jobs + while true; do + job=`expr $job + 1` + # Exit loop condition + if test $job -gt $stop_job; then break; fi + # Only send email about last job + jobs_left=`expr $stop_job - $job` + if test $jobs_left -le $cpus; then the_email=$email; fi + #if test $job -eq $stop_job; then the_email=$email; fi + the_command="$command -j $job -f $first_job" + # Set chunk of tasks to run per node (per slot) + top_job=`expr $job + $cpus - 1` + if test $top_job -gt $stop_job; then top_job=$stop_job; fi + name=${inpf}_w${job}-${top_job} + if test "x$testing" = xyes; then + cat > ${name}.sh << EOF +#!/bin/bash +#$ -S /bin/bash +#$ -N $name # Modify job name as needed. +#$ -M $the_email # NOTE: Replace with appropriate email address +##$ -P $project +#$ -m bea +#$ -j y +#$ -o ${egs_home}${app}/$name.out +#$ -pe dev ${slot} +#$ -l res_cpus=${cpus} +#$ -l res_mem=${ram} +#$ -l res_tmpfs=${tmpfs} +#$ -l res_image=${image} +#$ -l h_rt=${cpu_t} +# Export these environmental variables +#$ -v EGS_CONFIG=$egs_configuration +#$ -v HEN_HOUSE=$hen_house +#$ -v EGS_HOME=$egs_home +#$ -v TZ=America/Toronto +# Call your program +for i in \$(seq $job $top_job);do + echo \$i; +done | xargs -P${cpus} -n1 -I{} sh -c '$command -j \$1 -f $first_job' - {} +EOF + else + cat << EOF | jobsub - +#!/bin/bash +#$ -S /bin/bash +#$ -N $name # Modify job name as needed. +#$ -M $the_email # NOTE: Replace with appropriate email address +##$ -P $project +#$ -m bea +#$ -j y +#$ -o ${egs_home}${app}/$name.out +#$ -pe dev ${slot} +#$ -l res_cpus=${cpus} +#$ -l res_mem=${ram} +#$ -l res_tmpfs=${tmpfs} +#$ -l res_image=${image} +#$ -l h_rt=${cpu_t} +# Export these environmental variables +#$ -v EGS_CONFIG=$egs_configuration +#$ -v HEN_HOUSE=$hen_house +#$ -v EGS_HOME=$egs_home +#$ -v TZ=America/Toronto +# Call your program +for i in \$(seq $job $top_job);do + echo \$i; +done | xargs -P${cpus} -n1 -I{} sh -c '$command -j \$1 -f $first_job' - {} +EOF + fi + job=$top_job + done + +# +# Submitting one job +# +else + name=$inpf + if test "x$testing" = xyes; then + echo "Job $name: Executing $command using jobsub" + echo "qsub options: CPU time=$cpu_t email=$email output=${egs_home}${app}/$name.out" + else + cat << EOF | jobsub - +#! /bin/bash +# +## hello.job +# +#$ -N $name # Modify job name as needed. +# +#$ -M $email # NOTE: Replace with appropriate email address +#$ -P $project +#$ -m bea +# +#$ -j y +#$ -o ${egs_home}/${app}$name.out +# +#$ -pe dev ${slot} +#$ -l res_cpus=${cpus} +#$ -l res_mem=${ram} +#$ -l res_tmpfs=${tmpfs} +# +#$ -l res_image=${image} +# +#$ -l h_rt=${cpu_t} +# +# Export these environmental variables +#$ -v EGS_CONFIG=$egs_configuration +#$ -v HEN_HOUSE=$hen_house +#$ -v EGS_HOME=$egs_home +#$ -v TZ=America/Toronto + + # Call your program + +$command +EOF + fi +fi + +exit 0 From ff7a28935bc0f7201eb6de4f09fe2bd023b88895 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Fri, 21 Aug 2020 16:16:50 -0400 Subject: [PATCH 07/23] Add sleep option to stagger SGE job submission --- HEN_HOUSE/scripts/egs-jobsub-xargs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HEN_HOUSE/scripts/egs-jobsub-xargs b/HEN_HOUSE/scripts/egs-jobsub-xargs index 6d4aa6c4b..b5d3f7d40 100755 --- a/HEN_HOUSE/scripts/egs-jobsub-xargs +++ b/HEN_HOUSE/scripts/egs-jobsub-xargs @@ -65,6 +65,7 @@ egs_configuration="$EGS_CONFIG" n_parallel=0 other_args= testing=no +sleep_t=0.1 ######################### # NRC specific settings # @@ -106,6 +107,7 @@ while test $# -gt 0; do start=*) start_job=`echo $1 | sed 's/start=//'`;; stop=*) stop_job=`echo $1 | sed 's/stop=//'`;; cpu_t=*) cpu_t=`echo $1 | sed 's/cpu_t=//'`;; + sleep_t=*) sleep_t=`echo $1 | sed 's/sleep_t=//'`;; cpus=*) cpus=`echo $1 | sed 's/cpus=//'`;; slot=*) slot=`echo $1 | sed 's/slot=//'`;; ram=*) ram=`echo $1 | sed 's/ram=//'`;; @@ -368,6 +370,7 @@ done | xargs -P${cpus} -n1 -I{} sh -c '$command -j \$1 -f $first_job' - {} EOF fi job=$top_job + sleep $sleep_t done # From 3a222f8a0deb98e0a31495ce8f1ba8ad412cb3a7 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Fri, 21 Aug 2020 16:17:49 -0400 Subject: [PATCH 08/23] Improve output messages for the spherical geometry --- HEN_HOUSE/utils/geomsph.mortran | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/HEN_HOUSE/utils/geomsph.mortran b/HEN_HOUSE/utils/geomsph.mortran index dd152fb7e..bc7a4efc1 100644 --- a/HEN_HOUSE/utils/geomsph.mortran +++ b/HEN_HOUSE/utils/geomsph.mortran @@ -232,12 +232,9 @@ DO I = 1, NVALUE(NUM_CONES)[ STOP; ] ] -OUTPUT NCONE;(/'NUMBER OF ANGLES TO BE INPUT NCONE = ',I5); +OUTPUT NCONE;(/'Conical dimension [NCONE] : ',I5,' cones in the problem'); NC = NCONE + 1;"number of conical sections = total number of cones + 1" - - - IF(NC = 1)["PURE SPHERICAL GEOMETRY" ALPHA(1)=180.0; "NONE CONES" NPLAN1=1;NPLAN2=2; @@ -378,7 +375,7 @@ ELSEIF(NC > 1)[ $SKIP-LINE; ] OUTPUT NC;(/'NUMBER OF CONICAL REGIONS NC = ',I5); -OUTPUT NPLAN1;(/'REGION WHERE 90 deg IS UPPER CONE :',I5); +OUTPUT NPLAN1;(/'REGION WHERE 90 deg IS UPPER CONE :',I5,/); IVAL=IVAL+1; @@ -403,7 +400,9 @@ DO I = 1, NVALUE(NUM_RADII)[ STOP; ] ] -OUTPUT NR;(I6,' spheres in the problem ...'); +IF (NR > 0)[ + OUTPUT NR;(/'Spherical dimension: ',I6,' spheres in the problem ...'); +] IVAL = IVAL + 1; NUM_RSPH = IVAL; @@ -415,7 +414,7 @@ DEFAULT(IVAL)=1.0; $GET_INPUT(NUM_RSPH); IF (NR = 0) [ NR = NVALUE(NUM_RSPH); - OUTPUT NR;(I6,' spheres in the problem ...'); + OUTPUT NR;(/'Spherical dimension: ',I6,' spheres in the problem ...'); ] @@ -458,6 +457,7 @@ DO IX=1,I [ ] OUTPUT IX,RSPH(IX);(' RING RADIUS #',I6,':',T60,F12.6,' cms'); ] +OUTPUT; (/); NREG=NR*NC+1; @@ -472,14 +472,14 @@ VALUE_MAX(IVAL)=999999; DEFAULT(IVAL)=1; $GET_INPUT(IVAL); numcavreg = NVALUE(IVAL); -OUTPUT numcavreg;(' number of cavity regions: ', I4 ); +OUTPUT numcavreg;(/' number of cavity regions: ', I4 ); DO IX=1,numcavreg [ cavreg(IX) = VALUE(IVAL,IX); ] "CHECK THAT THE NUMBER OF CAVITY ZONES DOES NOT EXCEED ITS BOUNDS" IF(numcavreg.LE.0)["no cavity defined, just dose in all the regions desired" DO I=1,NREG[cavreg(I)=0;] - OUTPUT; (' **** no cavity regions defined ****'); + OUTPUT; (' **** no cavity regions defined ****'/); ] ELSEIF(numcavreg.GE.NREG)[ "IF IT DOES, REVERT TO THE STANDARD CHAMBER CONFIGURATION" @@ -589,12 +589,12 @@ IF (VALUE(NUM_MEDNUM,1) >= 0) [ NREGHI=VALUE(NUM_NREGHI,I); IF (NREGHI<=NREGLO) [ MED(NREGLO)=MEDNUM; - OUTPUT NREGLO,MEDNUM;(' REGION(',I3,') = MATERIAL(',I2,')'); + OUTPUT NREGLO,MEDNUM;(' REGION(',I4,') = MATERIAL(',I3,')'); ] ELSE [ DO K=NREGLO,NREGHI [MED(K)=MEDNUM;] OUTPUT NREGLO,NREGHI,MEDNUM; - (' REGION(',I3,') TO REGION(',I3,') = MATERIAL(',I2,')'); + (' REGION(',I4,') TO REGION(',I4,') = MATERIAL(',I3,')'); ] ] "end do I" ] "end IF ~=0" From 20c9dfb640bf1f12bf60673e0b17c299160504b4 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Thu, 24 Sep 2020 11:22:29 -0400 Subject: [PATCH 09/23] Update egsnrc.macros for uniform run control - Add macro $URC-DEFAULT for initialization; defaults to .false. - Add variable is_uniform_run to exit egs_pjob_* routines. - Add macros $egs_track* to debug Mortran apps after units are closed. --- HEN_HOUSE/src/egsnrc.macros | 39 +++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/HEN_HOUSE/src/egsnrc.macros b/HEN_HOUSE/src/egsnrc.macros index c7a9a3051..c1ae852d3 100644 --- a/HEN_HOUSE/src/egsnrc.macros +++ b/HEN_HOUSE/src/egsnrc.macros @@ -2312,6 +2312,9 @@ REPLACE {$COMP-XDATA-DEFAULT} WITH {'default'} "EADL relaxation is now the default" REPLACE {$EADL-RELAX-DEFAULT} WITH {.true.} ; +"Uniform Run Control (URC) is false by default" +REPLACE {$URC-DEFAULT} WITH {.false.} +; "Sabbatucci and Salvat PE xsections not the default yet" REPLACE {$MCDF-PE-DEFAULT} WITH {.false.} ; @@ -3572,7 +3575,30 @@ REPLACE {$egs_warning(#,#);} WITH { REPLACE {$warning(#,#);} WITH { write(i_log,{P1}) {P2}; }; REPLACE {$egs_info(#,#);} WITH { write(i_log,{P1}) {P2}; }; REPLACE {$declare_write_buffer;} WITH {;}; - +"****************************************************************" +" Useful output macros that do not rely on standard output unit. " +" Useful for debugging code after egs_finish call which closes " +" all open units (for instance after a parallel job finishes). " +"****************************************************************" +REPLACE {$egs_track2(#,#,#,#);} WITH { + open({P1},file={P2},action='write',position='append'); + write({P1},{P3}) {P4}; + $FLUSH_UNIT({P1}); + close({P1}); +}; +REPLACE {$egs_track1(#,#,#);} WITH { + open(666,file={P1},action='write',position='append'); + write(666,{P2}) {P3}; + $FLUSH_UNIT(666); + close(666); +}; +REPLACE {$egs_track0(#,#);} WITH { + open(666,file='track.log',action='write',position='append'); + write(666,{P1}) {P2}; + $FLUSH_UNIT(666); + close(666); +}; +"************************************************************" " Common block containing various directories, file names, etc. " REPLACE {$mx_units} WITH {20}; @@ -3600,10 +3626,11 @@ REPLACE {;COMIN/EGS-IO/;} WITH {; i_nist_data, "unit no. for NIST data" i_mscat, "unit no. for multiple scattering data" i_photo_cs, "unit no. for photon cross-section data" - i_photo_relax, "unit no. for photon relaxation data" - xsec_out, "switches on/off xsection file output" - is_batch, "True for batch mode" - is_pegsless; "true if you are running without pegs file" + i_photo_relax, "unit no. for photon relaxation data" + xsec_out, "switches on/off xsection file output" + is_batch, "True for batch mode" + is_uniform_run, "True for uniform parallel run control" + is_pegsless; "true if you are running without pegs file" character input_file*256, output_file*256, pegs_file*256, file_extensions*$max_extension_length, hen_house*128, egs_home*128, work_dir*128, user_code*64, @@ -3611,7 +3638,7 @@ REPLACE {;COMIN/EGS-IO/;} WITH {; $INTEGER n_parallel, i_parallel, first_parallel,n_max_parallel, n_chunk, file_units, n_files,i_input,i_log,i_incoh, i_nist_data,i_mscat,i_photo_cs,i_photo_relax, xsec_out; - $LOGICAL is_batch,is_pegsless; + $LOGICAL is_batch, is_pegsless, is_uniform_run; }; "The following macro sets the EGS_HOME directory " From c59d015d06f8818eee0ef4a524f1f47a7588f61f Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Thu, 24 Sep 2020 11:27:33 -0400 Subject: [PATCH 10/23] Tune egs_parallel.mortran for uniform run control Update the EGSnrc Mortran code for parallel jobs to handle the uniform run control object: - Add logical blocks to routines egs_pjob_control and egs_pjob_finish to skip all lock file related operations. - The logical block at end of egs_pjob_finish is required to avoid calling egs_time_diff, which hangs if t_start or t_left are not defined (they are normally read from the lock file, but there is no lock file in uniform run control). --- HEN_HOUSE/src/egs_parallel.mortran | 57 +++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/HEN_HOUSE/src/egs_parallel.mortran b/HEN_HOUSE/src/egs_parallel.mortran index 322016e6e..e5c92aa7a 100644 --- a/HEN_HOUSE/src/egs_parallel.mortran +++ b/HEN_HOUSE/src/egs_parallel.mortran @@ -116,7 +116,7 @@ subroutine egs_pjob_control(ncase,n_run,n_left,n_tot,sum,sum2,res,dres); " n_left no. of histories left to run after current job has been submitted " n_tot: should be set to number of histories from previous runs (if any) " for first call from job number 1. For all other jobs and for all -" other calls from job 1, it is set by egs_pjob_control to the the +" other calls from job 1, it is set by egs_pjob_control to the " number of histories run so far (including previous runs and other " parallel jobs). " Note: n_case, n_run and n_tot are $LONG_INT @@ -176,6 +176,21 @@ character control_string*256; data first_time/.true./; save first_time,n_last; +/*********************************/ +/* Option to not use a lock file */ +/*********************************/ +IF( is_uniform_run ) [ + IF (first_time)[ + n_run = ncase/n_parallel; + first_time = .false.; + ] + ELSE[ + n_run = 0; + ] + n_left = 0; + return; +] + IF( n_parallel <= 0 ) [ n_run = ncase; return; ] @@ -319,6 +334,20 @@ $REAL tmp,tmp2,res,dres,t_run; real egs_time_diff; character control_string*256; +/****************************/ +/* Option without lock file */ +/****************************/ +IF( is_uniform_run )[ + IF( i_parallel = n_parallel )[ + n_job = 0;"I am the last job!!!" + goto :complete_calculation:; + ] + ELSE ["nothing to do here" + n_job = 1;"Only last job allowed to finish!" + return; + ] +] + " Lock and rewind the job control file " call egs_rewind_control_file(istat); IF( istat ~= 0 ) [ @@ -359,15 +388,25 @@ IF( n_job > 0 ) [ " Other jobs still running " /* call egs_unlock_control_file(istat); */ call egs_close_control_file(istat); call egs_remove_lockfile(istat); + +:complete_calculation:; i_parallel = 0; call egs_open_units(.false.); -call egs_date_and_time(t_end); -t_run = egs_time_diff(t_start,t_end); -$egs_info('(/a/,a,t55,i3/,a,t55,i3/,a,f9.1,a,f9.4,a/,a//)', -'**************** finished parallel execution ******************', -' number of parallel jobs requested: ',n_parallel, -' max. number of jobs executing simultaneously: ',n_max_parallel, -' elapsed time since first job started: ',t_run,' s (',t_run/3600,' h)', -'***************************************************************'); +IF ( is_uniform_run )[ + $egs_info('(/a/,a,t55,i3/,a//)', + '**************** finished parallel execution ******************', + ' number of parallel jobs requested: ',n_parallel, + '***************************************************************'); +] +ELSE[ + call egs_date_and_time(t_end); + t_run = egs_time_diff(t_start,t_end); + $egs_info('(/a/,a,t55,i3/,a,t55,i3/,a,f9.1,a,f9.4,a/,a//)', + '**************** finished parallel execution ******************', + ' number of parallel jobs requested: ',n_parallel, + ' max. number of jobs executing simultaneously: ',n_max_parallel, + ' elapsed time since first job started: ',t_run,' s (',t_run/3600,' h)', + '***************************************************************'); +] return; :error2_lockfile_read:; From f4f6e3566dd9d6cbd6a5a7a29f2aca55c78dd4f2 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Thu, 24 Sep 2020 11:38:58 -0400 Subject: [PATCH 11/23] Tune egs_utilities.mortran for uniform run control - Set variable is_uniform_run to its default $URC-DEFAULT. - Parse arguments for -u or --urc and set is_uniform_run accordingly. - Inform user that the uniform run control is activated. --- HEN_HOUSE/src/egs_utilities.mortran | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/HEN_HOUSE/src/egs_utilities.mortran b/HEN_HOUSE/src/egs_utilities.mortran index 8c1f670e0..49b68812d 100644 --- a/HEN_HOUSE/src/egs_utilities.mortran +++ b/HEN_HOUSE/src/egs_utilities.mortran @@ -91,7 +91,7 @@ REPLACE {;COMIN/my_times/;} WITH {; " -b or --batch Specify a 'batch' run. The difference " between a 'batch' run and an interactive " run is that in 'batch' mode unit 6 is -" connected to a file, whereas in intarctive +" connected to a file, whereas in interactive " mode unit 6 goes to the standard output. " The file name in batch run is determined " as follows: @@ -449,6 +449,12 @@ IF( n_parallel > 0 ) [ ] $egs_info('(a)',line); +IF (is_uniform_run)[ + $egs_info('(//a,i0,a,i0,a//)', + '-> User requests uniform run control. I am job # ', + i_parallel,' of ',n_parallel,' jobs'); +] + return; " Errors " @@ -578,6 +584,10 @@ IF( n_parallel > 0 | i_parallel > 0 ) [ ] ] +" Check for run control option " +$check_argument('-u','--urc',arg); +IF( have_arg ) is_uniform_run = .true.; + $egs_debug('(a,a)','HEN_HOUSE is ',$cstring(hen_house)); " Check for egs_home " @@ -984,6 +994,7 @@ $set_string(pegs_file,' '); $set_string(host_name,' '); n_parallel = 0; i_parallel = 0; n_chunk = 0; is_batch = .false.; first_parallel = 1; +is_uniform_run = $URC-DEFAULT; return; end; "***************************************************************************** From a868291c34bf643f783cb143377047af468a0559 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Thu, 24 Sep 2020 11:45:00 -0400 Subject: [PATCH 12/23] Improve SPRRZnrc for parallel jobs Improve the information provided in the final output file after a parallel run. Frequently, intermediate output files for individual jobs are deleted, hence it is desirable to preserve calculation information in the final *.egslog and *.egslst files. - Add call to ISUMRY before calling egs_combine_runs (a call to subroutine INPUTS might be more appropriate!). - After combining parallel jobs, jump to new label :TIMING-INFO: where total timing info is generated, followed by the final statistical analysis of the combined results. - Tweak the format of version message to allow for integers of any size. - Removed duplicate version message from subroutine INPUTS. --- HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran b/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran index 847c73045..016949e0f 100644 --- a/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran +++ b/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran @@ -1856,7 +1856,7 @@ $SET_ELAPSED_CPUTIME(CPUT0); "obtain the initial starting time" OUTPUT $MAXZPLANE,$MAXRADII; (//' SPRRZnrc(EGSnrc) '$VERSION / - ' on '$MACHINE' with',I3, ' depth regions &', I3, + ' on '$MACHINE' with ',I0, ' depth regions & ', I0, ' radial regions'//); "****************************************************************************** @@ -2356,6 +2356,7 @@ $SHOW-RNG-STATE(6); write(6,'(a)') ' *********'; write(iout,'(/a,$)') '********* FINAL RANDOM NUMBER STATE:'; $SHOW-RNG-STATE(iout); write(iout,'(a)') ' *********'; +:TIMING-INFO: $SET_ELAPSED_TOTAL_TIME(TIMEB); ETIMETOT=ETIMETOT+TIMEB; $SET_ELAPSED_CPUTIME(CPUT2); @@ -2605,10 +2606,11 @@ IF( n_parallel > 0 & ~is_finished ) [ call egs_pjob_finish(n_job); IF( n_job = 0 ) [ is_finished = .true.; + CALL ISUMRY; call egs_combine_runs(combine_results,'.egsdat'); NCASET=NCASEO; IHSTRY=NCASET; CALL SRCOTO(WEIGHT); - goto :STATS-ANAL:; + goto :TIMING-INFO:; ] ] #endif; @@ -3596,10 +3598,6 @@ ERROR_FLAG=0; "Initialization of bad input flag" DO J=1, $NMAX [NVALUE(J)=0;] "Initialization of Nvalue Array" -OUTPUT $MAXRADII,$MAXZPLANE; -(//' SPRRZnrc(EGSnrc) '$VERSION /' On '$MACHINE' .', -4x,'Max no of radial/ depth zones=',2I4); - "*********************************************************" "* set up the values_sought(i) and echo inputs to screen *" "*********************************************************" From 3acb191c7755d070406af33b9d8a7dce3e979c8e Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 28 Sep 2020 10:43:58 -0400 Subject: [PATCH 13/23] Update Mortran apps for uniform run control Update Mortran application for running parallel jobs without a lock file via the command line arguments -u or --urc. The number of histories is split equally among the parallel jobs. No changes are required for BEAMnrc apps and DOSXYZnrc! Improve RZ application output after parallel runs to ensure that enough information is preserved in the final *.egslog and *.egslst output files: - Add call to ISUMRY before calling egs_combine_runs. - After combining parallel jobs, jump to new label :output-results: where the last random numbers and total timing info are generated, followed by the final statistical analysis of the combined results. DOSRZnrc does not require this label and the jump is to :END-SIM:. Fix the following small bugs: - Except for CAVRZnrc, linking unit 15 to the *.errors file has been re-enabled in the *.io file since there is a final date and time output statement to the *.errors file. If this is not explicitly requested, a fort.15 file is left after parallel runs. - Template input files for CAVRZnrc and FLURZnrc were still using SI as default photon cross section library. Changed to XCOM. - Several comment lines in DOSRZnrc extended beyond the 80 character limit after brems was expanded to bremsstrahlung. Trimmed those. Test the applications under the new uniform run control object. Tests were performed for BEAM_EX10MeVe, BEAM_EX16MVp, and DOSXYZnrc examples, yielding a perfect match between parallel runs using lock files and the new uniform run control. The RZ codes were also tested and revealed no issue either. --- HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran | 6 +++++- .../cavrznrc/cavrznrc_template.egsinp | 2 +- HEN_HOUSE/user_codes/dosrznrc/dosrznrc.io | 2 +- HEN_HOUSE/user_codes/dosrznrc/dosrznrc.mortran | 17 ++++++++++++++++- HEN_HOUSE/user_codes/flurznrc/flurznrc.io | 13 +++++++------ HEN_HOUSE/user_codes/flurznrc/flurznrc.mortran | 5 ++++- .../flurznrc/flurznrc_template.egsinp | 2 +- HEN_HOUSE/user_codes/sprrznrc/sprrznrc.io | 2 +- HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran | 13 +++++++------ 9 files changed, 43 insertions(+), 19 deletions(-) diff --git a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran index 635093ed9..0521a8220 100644 --- a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran +++ b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran @@ -2961,6 +2961,8 @@ IF(NSUMCV>1)["store data for individual cavity regions" ] ] ] +:output-results:; + write(6,'(/a,$)') '****** FINAL RANDOM NUMBER STATE:'; $SHOW-RNG-STATE(6); write(6,'(a)') ' ******'; write(iout,'(/a,$)') '********* FINAL RANDOM NUMBER STATE:'; @@ -2994,6 +2996,7 @@ ELSE[ ] IF(IHSTRY.NE.0 .AND. TIMCPU.NE.0.0) ["this should always happen" WRITE(IOUT,280) TIMCPU/dble(IHSTRY),3600.*dble(IHSTRY)/TIMCPU; + WRITE(6,280) TIMCPU/dble(IHSTRY),3600.*dble(IHSTRY)/TIMCPU; ] ] @@ -3252,10 +3255,11 @@ IF( n_parallel > 0 & ~is_finished ) [ call egs_pjob_finish(n_job); IF( n_job = 0 ) [ is_finished = .true.; + call isumry; call egs_combine_runs(combine_results,'.egsdat'); NCASET=NCASEO; IHSTRY=NCASET; CALL SRCOTO(WEIGHT); - goto :STATS-ANAL:; + goto :output-results:; ] ] #endif; diff --git a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp index 179dc9ce1..40068ae35 100644 --- a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp +++ b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp @@ -341,7 +341,7 @@ Pair cross sections= BH # BH (default) or NRC. # NRC: use NRC pair production cross-sections # (in file $HEN_HOUSE/data/pair_nrc1.data). -Photon cross sections= si # si (Storm-Israel--the default), +Photon cross sections= xcom # si (Storm-Israel), # epdl (Evaluated Photon Data Library), # xcom (NIST Photon Cross Sections Database) # pegs4 (PEGS4 file photon data) diff --git a/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.io b/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.io index 6292a6650..fe85d97b4 100644 --- a/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.io +++ b/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.io @@ -38,4 +38,4 @@ # 1 .egslst # Unit 15 is explicitly opened in get_inputs -#15 .errors +15 .errors diff --git a/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.mortran b/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.mortran index a4d4856fc..7b0e3b005 100644 --- a/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.mortran +++ b/HEN_HOUSE/user_codes/dosrznrc/dosrznrc.mortran @@ -1358,8 +1358,16 @@ REPLACE {$VERSION} WITH { " ESAVEIN (R) If ELECTRON RANGE REJECTION is on, discard an " electron when E< ESAVEIN and RANGE < CDIST " where CDIST is closest distance to region of +<<<<<<< HEAD " interest specified below (ignores bremsstrahlung " losses below ESAVEIN). +||||||| merged common ancestors +" interest specified below. This ignores bremsstrahlung +" losses below ESAVEIN. +======= +" interest specified. This ignores bremsstrahlung +" losses below ESAVEIN. +>>>>>>> Changes to Mortran apps for URC implementation " This parameter must be input even if not used. " ESAVEIN is a total energy. " @@ -3481,10 +3489,11 @@ IF( n_parallel > 0 & ~is_finished ) [ call egs_pjob_finish(n_job); IF( n_job = 0 ) [ is_finished = .true.; + call isumry; call egs_combine_runs(combine_results,'.egsdat'); NCASET=NCASEO; IHSTRY=NCASET; CALL SRCOTO(WEIGHT); - goto :STATS-ANAL:; + goto :END-SIM:; ] ] #endif; @@ -5071,7 +5080,13 @@ nbr_split =VALUE(NUM_BREMPEVEN,1); IF(IBRSPL = 1)["brems splitting requested" "default bremsstrahlung splitting to maximum" "IF((nbr_split <= 0) | (nbr_split > $MAXBRSPLIT)) nbr_split =$MAXBRSPLIT;" +<<<<<<< HEAD "Now allows bremsstrahlung and annihilation radiation to be turned off" +||||||| merged common ancestors + "Changed to allow bremsstrahlung and annihilation radiation to be turned off" +======= + "To allow bremsstrahlung and annihilation radiation to be turned off" +>>>>>>> Changes to Mortran apps for URC implementation IF((nbr_split > $MAXBRSPLIT)) nbr_split =$MAXBRSPLIT;" ] ELSE ["no bremsstrahlung splitting" nbr_split = 1;] diff --git a/HEN_HOUSE/user_codes/flurznrc/flurznrc.io b/HEN_HOUSE/user_codes/flurznrc/flurznrc.io index c95bae886..7f061c0c7 100644 --- a/HEN_HOUSE/user_codes/flurznrc/flurznrc.io +++ b/HEN_HOUSE/user_codes/flurznrc/flurznrc.io @@ -40,15 +40,16 @@ # This should be cleaned up along the lines of cavrznrc and dosrznrc ! # 1 .egslst -2 .egsrns +#2 .egsrns #4 .egsdat -9 .egseff -10 .egsdose +#9 .egseff +#10 .egsdose #13 .egsgph # Unit 15 is explicitly opened in get_inputs #15 .errors 17 .spectra -22 .ploterr +#22 .ploterr 23 .plotdat -43 .egssrc -44 .egssrctmp +# It appears that the following 2 are unused +#43 .egssrc +#44 .egssrctmp diff --git a/HEN_HOUSE/user_codes/flurznrc/flurznrc.mortran b/HEN_HOUSE/user_codes/flurznrc/flurznrc.mortran index 11ae1c899..55c91287f 100644 --- a/HEN_HOUSE/user_codes/flurznrc/flurznrc.mortran +++ b/HEN_HOUSE/user_codes/flurznrc/flurznrc.mortran @@ -2668,6 +2668,8 @@ DO IX=1,NR[ ] ] +:output-results:; + write(6,'(/a,$)') '********* FINAL RANDOM NUMBER STATE:'; $SHOW-RNG-STATE(6); write(6,'(a)') ' *********'; write(iout,'(/a,$)') '********* FINAL RANDOM NUMBER STATE:'; @@ -2920,10 +2922,11 @@ IF( n_parallel > 0 & ~is_finished ) [ call egs_pjob_finish(n_job); IF( n_job = 0 ) [ is_finished = .true.; + call isumry; call egs_combine_runs(combine_results,'.egsdat'); NCASET=NCASEO; IHSTRY=NCASET; CALL SRCOTO(WEIGHT); - goto :STATS-ANAL:; + goto :output-results:; ] ] #endif; diff --git a/HEN_HOUSE/user_codes/flurznrc/flurznrc_template.egsinp b/HEN_HOUSE/user_codes/flurznrc/flurznrc_template.egsinp index 0b59a46cf..4186f8a59 100644 --- a/HEN_HOUSE/user_codes/flurznrc/flurznrc_template.egsinp +++ b/HEN_HOUSE/user_codes/flurznrc/flurznrc_template.egsinp @@ -327,7 +327,7 @@ Pair cross sections= BH # BH (default) or NRC. # NRC: use NRC pair production cross-sections # (in file $HEN_HOUSE/data/pair_nrc1.data). -Photon cross sections= si # si (Storm-Israel--the default), +Photon cross sections= xcom # si (Storm-Israel), # epdl (Evaluated Photon Data Library), # xcom (NIST Photon Cross Sections Database) # pegs4 (PEGS4 file photon data) diff --git a/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.io b/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.io index 524a97485..b0ad961a3 100644 --- a/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.io +++ b/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.io @@ -39,4 +39,4 @@ # 1 .egslst # Unit 15 is explicitly opened in get_inputs -#15 .errors +15 .errors diff --git a/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran b/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran index 016949e0f..f411f6a06 100644 --- a/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran +++ b/HEN_HOUSE/user_codes/sprrznrc/sprrznrc.mortran @@ -2351,12 +2351,12 @@ DO IX=1,NR[ ] ] +:output-results:; + write(6,'(/a)') '********* final random number state:'; $SHOW-RNG-STATE(6); write(6,'(a)') ' *********'; write(iout,'(/a,$)') '********* FINAL RANDOM NUMBER STATE:'; $SHOW-RNG-STATE(iout); write(iout,'(a)') ' *********'; - -:TIMING-INFO: $SET_ELAPSED_TOTAL_TIME(TIMEB); ETIMETOT=ETIMETOT+TIMEB; $SET_ELAPSED_CPUTIME(CPUT2); @@ -2367,7 +2367,6 @@ WRITE(IOUT,261)ETIMETOT,TIMCPU,TIMCPU/3600., RATIO,FHPH; WRITE(6,261)ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO,FHPH; - "****************************************************************************** " ;" *** SECTION 3 *** @@ -2610,7 +2609,7 @@ IF( n_parallel > 0 & ~is_finished ) [ call egs_combine_runs(combine_results,'.egsdat'); NCASET=NCASEO; IHSTRY=NCASET; CALL SRCOTO(WEIGHT); - goto :TIMING-INFO:; + goto :output-results:; ] ] #endif; @@ -4284,6 +4283,7 @@ write(iout,'(" *"/1X,79("*"))'); ' * It uses the EGSnrc Code System developed at NRC ',T80,'*'/ ' * (based on the EGS4 Code System Carlo developed by SLAC and NRC)', T80,'*'/ + ' * This is version '$VERSION' ',T80,'*'/ ' * running on '$MACHINE' ',T80,'*'/ ' *',T80, '*'/ ' *', T54,' ',$); @@ -4307,6 +4307,7 @@ write(iout,'(" *"/1X,79("*"))'); " ' * It uses the EGSnrc Code System developed at NRC ',T80,'*'/ " ' * (based on the EGS4 Code System Carlo developed by SLAC and NRC)', " T80,'*'/ +" ' * This is version '$VERSION' ',T80,'*'/ " ' * running on '$MACHINE' ',T80,'*'/ " ' *',T80, '*'/ " ' *', T60,$TIMEN_FORMAT,1X,$DATEN_FORMAT,T80,'*'/1X,79('*')); @@ -4666,8 +4667,8 @@ IF(ISPRREG=1)["create the .plotdat file" AXISTYPE=0; "0 for no logs" DO IX=1, NR [ XCOORD(IX) = RCYL(IX); - YCOORD(IX) = SCDOSE(IZ,IX,3)/Dmax; - UNCERT(IX) = SCDOSE(IZ,IX,3)*SCDOSE2(IZ,IX,3)/(Dmax*100.); + YCOORD(IZ) = SCDOSE(IZ,IX,3)/Dmax; + UNCERT(IZ) = SCDOSE(IZ,IX,3)*SCDOSE2(IZ,IX,3)/(Dmax*100.); ] "Convert slab number(IZ) to a character string" $CONVERT_INT(IZ)_TO_CHAR(CH_IZ); From a8ed1cd58a86de5bbc8417675faa5b4f1ba842f5 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 5 Oct 2020 10:58:04 -0400 Subject: [PATCH 14/23] Add variable is_uniform_run to EGS_IO structure Add variable is_uniform_run to the C structure EGS_IO in the C interfaces, to match the EGS-IO Mortran common block. This is not strictly required in the C++ apps, but it avoids a memory mismatch between the COMMON block and the C structure. --- HEN_HOUSE/interface/egs_interface1.h | 2 +- HEN_HOUSE/interface/egs_interface2.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HEN_HOUSE/interface/egs_interface1.h b/HEN_HOUSE/interface/egs_interface1.h index 884d91d65..9b7a3b549 100644 --- a/HEN_HOUSE/interface/egs_interface1.h +++ b/HEN_HOUSE/interface/egs_interface1.h @@ -609,7 +609,7 @@ struct EGS_IO { EGS_I32 n_parallel, i_parallel, first_parallel, n_max_parallel, n_chunk, n_files, i_input, i_log, i_incoh, i_nist_data, i_mscat, i_photo_cs, i_photo_relax, xsec_out, is_batch, - is_pegsless; + is_uniform_run, is_pegsless; }; diff --git a/HEN_HOUSE/interface/egs_interface2.h b/HEN_HOUSE/interface/egs_interface2.h index 0149d1e84..38b109a5d 100644 --- a/HEN_HOUSE/interface/egs_interface2.h +++ b/HEN_HOUSE/interface/egs_interface2.h @@ -625,7 +625,7 @@ struct EGS_IO { EGS_I32 n_parallel, i_parallel, first_parallel, n_max_parallel, n_chunk, n_files, i_input, i_log, i_incoh, i_nist_data, i_mscat, i_photo_cs, i_photo_relax, xsec_out, is_batch, - is_pegsless; + is_uniform_run, is_pegsless; }; /*! \brief A structure corresponding to the \c EMF_INPUTS common block. From 3910e1251661a5bac427d8ae8e0e576c991ff673 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 5 Oct 2020 11:02:56 -0400 Subject: [PATCH 15/23] Add -u (--urc) run control option to egs++ apps Implement a command line option -u (or --urc) to select the uniform run control object (RCO), for consistency across all EGSnrc applications. This is also consistent with existing option -s (--simple-run) to select the simple run control mechanism. When flag -u (or --urc) is passed to the egs++ app on the command line, the simulation runs under the uniform RCO: the last job combines all completed jobs and monitors running jobs for 5 seconds at 1 second intervals. Via the input file the user can change these intervals, and also select another job to act as the manager. Differences between the simple RCO (-s) and the uniform RCO (-u): - The simple RCO is like a uniform RCO where the last job combines available *egsdat files without the option to wait until a user-defined time interval for other jobs to finish. - The uniform RCO allows adjusting, via the input file, the checking interval and to define which job manages the combination of results. --- HEN_HOUSE/egs++/egs_application.cpp | 18 +++++++++++++++++- HEN_HOUSE/egs++/egs_application.h | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_application.cpp b/HEN_HOUSE/egs++/egs_application.cpp index 3268758aa..92e3a9a3a 100644 --- a/HEN_HOUSE/egs++/egs_application.cpp +++ b/HEN_HOUSE/egs++/egs_application.cpp @@ -223,7 +223,7 @@ void EGS_Application::storeGeometryStep(int ireg, int inew, } EGS_Application::EGS_Application(int argc, char **argv) : input(0), geometry(0), - source(0), rndm(0), run(0), simple_run(false), current_case(0), + source(0), rndm(0), run(0), simple_run(false), uniform_run(false), current_case(0), last_case(0), data_out(0), data_in(0), a_objects(0), ghistory(new EGS_GeometryHistory) { @@ -419,6 +419,19 @@ EGS_Application::EGS_Application(int argc, char **argv) : input(0), geometry(0), } } + // + // *** See if user wants uniform job control. + // (Takes precedence over simple job control) + // + for (int j=1; j 0 && n_parallel > 0) { batch_run = true; @@ -700,6 +713,9 @@ int EGS_Application::initRunControl() { if (simple_run) { run = new EGS_RunControl(this); } + else if (uniform_run) { + run = new EGS_UniformRunControl(this); + } else { run = EGS_RunControl::getRunControlObject(this); } diff --git a/HEN_HOUSE/egs++/egs_application.h b/HEN_HOUSE/egs++/egs_application.h index 52ec3ad6a..9081927c8 100644 --- a/HEN_HOUSE/egs++/egs_application.h +++ b/HEN_HOUSE/egs++/egs_application.h @@ -1071,7 +1071,8 @@ class EGS_EXPORT EGS_Application { i_parallel, //!< Job index in parallel runs first_parallel; //!< first parallel job number bool batch_run; //!< Interactive or batch run. - bool simple_run; //!< Use a simple run control even for parallel runs + bool simple_run; //!< Use a simple run control object for parallel runs + bool uniform_run; //!< Use a uniform run control object for parallel runs bool is_pegsless; //!< set to true if a pegsless run EGS_Particle p; /*!< Parameters of the particle that just From cc4777464961b39daf346693c32be30897dc5e81 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 5 Oct 2020 12:00:07 -0400 Subject: [PATCH 16/23] Fix total ncase for the job control file object Reset ncase to the user requested value by multiplying it by the number of parallel jobs. Previously, the job control file object (JCFO) effectively reduced the total number of histories by the number of parallel jobs. This bug was introduced in the initial implementation of the uniform RCO where ncase was reduced in the base class of the RCO. This works for the simple and uniform types, but not for the JCFO, where the number of histories are managed by the job control file object (via a lock file). Also, ensure that by default the last job is a watcher job in the uniform RCO even if there is no input block defining an RCO in the input file. --- HEN_HOUSE/egs++/egs_run_control.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_run_control.cpp b/HEN_HOUSE/egs++/egs_run_control.cpp index ce27cd792..426204469 100644 --- a/HEN_HOUSE/egs++/egs_run_control.cpp +++ b/HEN_HOUSE/egs++/egs_run_control.cpp @@ -79,6 +79,12 @@ EGS_RunControl::EGS_RunControl(EGS_Application *a) : geomErrorCount(0), "'number of histories' input\n"); } ncase = EGS_I64(ncase_double); + /***************************************************** + * Split histories into different parallel jobs. + * For the JCFO ncase reset to total as it is handled + * via the lock file mechanism dispatching smaller + * of histories chunks. + *****************************************************/ if (app->getNparallel()) { ncase /= app->getNparallel(); } @@ -266,11 +272,6 @@ EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : rco_type = uniform; -// Not needed: Done now in the base EGS_RunControl constructor -// if (npar){ -// ncase /= npar; -// } - if (input) { /*Change waiting time to check for parallel run completion*/ @@ -307,7 +308,6 @@ EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : } } - /* Request checking parallel run completion */ vector check_options; check_options.push_back("yes"); @@ -318,6 +318,12 @@ EGS_UniformRunControl::EGS_UniformRunControl(EGS_Application *a) : } } + else { // use defaults if no RCO input found + /* last job is watcher job */ + if (ipar == ifirst + npar - 1) { + watcher_job = true; + } + } } int EGS_UniformRunControl::startSimulation() { @@ -549,6 +555,11 @@ EGS_JCFControl::EGS_JCFControl(EGS_Application *a, int Nbuf) : rco_type = balanced; + /* Recover initial number of histories */ + if (npar) { + ncase *= npar; + } + if (input) { int err = input->getInput("nchunk",nchunk); if (err) { From 67d4608d775457b5086e8d9fcf45ebab1588b158 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 5 Oct 2020 15:25:45 -0400 Subject: [PATCH 17/23] Check uniform RCO job termination in Motran apps For the uniform run control object (RCO), check if the number of output files (i.e., *egsdat, *pardose, etc.) is equal to the number of combined parallel jobs. If not, then the last job sleeps for a set time ($URC-SLEEP, by default 1 second) and repeats the combining process a set number of times ($URC-INTERVALS, by default only once). Previously, the uniform RCO for Mortran apps was in fact identical to the egs++ simple URC. In that case, the last job combines only those jobs that finished before it. Note that the Mortran uniform RCO implementation does not delete existing output files such as *egsdat and *pardose at the beginning of a parallel run (as opposed to the egs++ implementation). This could produce ill effects when re-running a previous calculation with a modified input file: if not all output files are overwritten (e.g., fewer parallel jobs are requested for the new run), then a mix of old and new output files will be combined. --- HEN_HOUSE/src/egs_utilities.mortran | 17 ++++++++++++++++- HEN_HOUSE/src/egsnrc.macros | 6 ++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/HEN_HOUSE/src/egs_utilities.mortran b/HEN_HOUSE/src/egs_utilities.mortran index 49b68812d..a52f8010f 100644 --- a/HEN_HOUSE/src/egs_utilities.mortran +++ b/HEN_HOUSE/src/egs_utilities.mortran @@ -1012,10 +1012,14 @@ character*1024 tmp_string,base,command,outfile,parfile_name,base1, text_string; integer lnblnk1,istat,ipar,egs_system,egs_open_file; $INTEGER i,k,j,numparfiles,textindex; +integer urcSleep, urcCheckIntervals; $LOGICAL ex,iwin; iwin=.false.; "start off assuming a non-Windows system" +urcSleep = $URC-SLEEP; "Set to 1 s in egsnrc.macros" +urcCheckIntervals = $URC-INTERVALS;"Set to 1 in egsnrc.macros" + $set_string(base,' '); base = $cstring(egs_home) // $cstring(user_code) // $file_sep // $cstring(output_file) // '_w'; @@ -1028,6 +1032,8 @@ base1 = $cstring(egs_home) // $cstring(user_code) // $file_sep // $set_string(outfile,' '); outfile = $cstring(egs_home) // $cstring(user_code) // $file_sep // 'parfiles_tmp'; + +:check-output-files:; "try Unix/Linux first" $set_string(command,' '); command = 'ls ' // $cstring(base1) // ' | wc -l > ' // $cstring(outfile); @@ -1037,7 +1043,8 @@ IF(istat~=0)["now assume a Windows system" $cstring(outfile); istat = egs_system($cstring(command)); IF(istat~=0)[ -$egs_fatal(*,' Failed to write number of output files from parallel runs.'); + $egs_fatal(*, + ' Failed to write number of output files from parallel runs.'); ] ELSE [ iwin=.true.; @@ -1060,6 +1067,14 @@ ELSE[ ] close(ipar); +#ifdef HAVE_C_COMPILER; +IF ( is_uniform_run & numparfiles < n_parallel & urcCheckIntervals > 0 ) [ + call egs_sleep(urcSleep);"sleep for urcSleep seconds" + urcCheckIntervals -= 1; + goto :check-output-files:; +] +#endif; + "now remove parfiles_tmp" $set_string(command,' '); diff --git a/HEN_HOUSE/src/egsnrc.macros b/HEN_HOUSE/src/egsnrc.macros index c1ae852d3..37867180b 100644 --- a/HEN_HOUSE/src/egsnrc.macros +++ b/HEN_HOUSE/src/egsnrc.macros @@ -2315,6 +2315,12 @@ REPLACE {$EADL-RELAX-DEFAULT} WITH {.true.} "Uniform Run Control (URC) is false by default" REPLACE {$URC-DEFAULT} WITH {.false.} ; +"SLEEP INTERVAL for URC (1 s by default)" +"Time to wait for jobs to complete after last job finished" +REPLACE {$URC-SLEEP} WITH {1} +"Times to check for jobs to complete after last job finished" +REPLACE {$URC-INTERVALS} WITH {1} +; "Sabbatucci and Salvat PE xsections not the default yet" REPLACE {$MCDF-PE-DEFAULT} WITH {.false.} ; From 7ab5e62089ca2ccb792d9a4d819e8b904cc3a23f Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 5 Oct 2020 15:55:58 -0400 Subject: [PATCH 18/23] Fix lock file errors when using a BEAMnrc library Call beam_finish from beamlib_finish with a different ientry value of 3. This signals that the BEAM source library is done and that there is no need to call egs_pjo_finish, which tries to access the lock file for regular BEAMnrc parallel runs. Previously, while testing the uniform run control object in Mortran apps, fort.* files were left behind in the DOSXYZnrc app folder when using BEAMnrc libraries as a source of particles. This occurred whether or not the simulation relied on a lock file. Interestingly, the problem was noticed using gcc 7.5 on Ubuntu 18.04, but it does not occur with gcc 4.8 on CentOS 7. These spurious fort.* files contained error messages from the BEAM library source, reporting a failure to rewind the lock file. The error message was traced to the egs_pjob_finish routine attempting to rewind and read the lock file. The bug occurs because a BEAM library source does not create a lock file, yet at the end of a parallel run beamlib_finish calls beam_finish(ientry), with the same ientry=1 argument used in regular BEAMnrc simulations (which create a lock file). This bug does not affect the simulation results since it since it aborts the parallel job when it was already done, with a call to $egs_fatal which attempts to ouput an error message to an output file that is already closed. --- HEN_HOUSE/omega/beamnrc/beam_lib.mortran | 4 ++-- HEN_HOUSE/omega/beamnrc/beamnrc.mortran | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/HEN_HOUSE/omega/beamnrc/beam_lib.mortran b/HEN_HOUSE/omega/beamnrc/beam_lib.mortran index 25474c628..5cbe99dd4 100644 --- a/HEN_HOUSE/omega/beamnrc/beam_lib.mortran +++ b/HEN_HOUSE/omega/beamnrc/beam_lib.mortran @@ -267,8 +267,8 @@ subroutine beamlib_finish; implicit none; "write .egsdat file if requested" call beam_write_dat; -call beam_finish(1); -"call with 1 to signify end of beam lib run" +call beam_finish(3); +"call with 3 to signify end of beam lib run" return; end; "below is a dummy xvgrplot subroutine for use with BEAM shared diff --git a/HEN_HOUSE/omega/beamnrc/beamnrc.mortran b/HEN_HOUSE/omega/beamnrc/beamnrc.mortran index 79c07e64e..e7ae51000 100644 --- a/HEN_HOUSE/omega/beamnrc/beamnrc.mortran +++ b/HEN_HOUSE/omega/beamnrc/beamnrc.mortran @@ -5016,10 +5016,10 @@ ITMAX=3+LNEXC+LNINC; return; end; -subroutine beam_finish(ientry); +subroutine beam_finish(i_entry); implicit none; -$INTEGER ientry,egs_open_file; +$INTEGER i_entry, ientry, egs_open_file; ;COMIN/SCORE,SOURCE,STACK,IO_INFO,RANDOM,USER,CMs,EGS-IO,UPHIOT,TIMING-INFO, RWPHSP/; @@ -5038,6 +5038,8 @@ $INTEGER I,I1,I2,I3,I4,IT,ITMAX,IERR,NUMIND,COVIND; $DECLARE_TIMING_VARIABLES; +$LOGICAL is_beam_src; + external combine_results; #ifdef HAVE_C_COMPILER; @@ -5046,6 +5048,16 @@ integer n_job; #endif; is_finished = .false.; +is_beam_src = .false.; + +"Calling from a BEAM source library" +IF (i_entry = 3)[ + is_beam_src = .true.; + ientry = 1; +] +ELSE["Proceed as usual" + ientry = i_entry; +] "******************************************************************************* " @@ -5485,7 +5497,7 @@ call egs_finish; " Finish the simulation " #ifdef HAVE_C_COMPILER; ; -IF( n_parallel > 0 & ~is_finished ) [ +IF( n_parallel > 0 & ~is_finished & ~is_beam_src ) [ call egs_pjob_finish(n_job); IOUTLIST=egs_open_file(IOUTLIST,0,1,'.egslst'); IF( n_job = 0 ) [ From da12bc0c7bb0d834c8bba14c481a75f55a75d8ba Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Mon, 5 Oct 2020 16:16:44 -0400 Subject: [PATCH 19/23] Preserve information after DOSXYZnrc parallel run Add calls to summarizing subroutines to preserve information in the combined egslog and egslst files. This way, if the intermediate output files from individual jobs are removed, this information remains available. No changes were strictly required in DOSXYZnrc for the uniform run control object implementation. Previously, the only information in the final output file pertained to the total dose in each requested region of the phantom, because in DOSXYZnrc combining results is implemented for *.pardose files only. Note that the timing information is missing from the final output file. --- HEN_HOUSE/user_codes/dosxyznrc/dosxyznrc.mortran | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HEN_HOUSE/user_codes/dosxyznrc/dosxyznrc.mortran b/HEN_HOUSE/user_codes/dosxyznrc/dosxyznrc.mortran index dd54c994b..eaf9a835b 100644 --- a/HEN_HOUSE/user_codes/dosxyznrc/dosxyznrc.mortran +++ b/HEN_HOUSE/user_codes/dosxyznrc/dosxyznrc.mortran @@ -3388,6 +3388,8 @@ IF( n_parallel > 0 & ~is_finished ) [ ainflu=temp2; "need to define this because it is used to normalize dose" IF((isource = 0 | isource = 1 | isource = 3 | isource = 7) & beamarea>0.) ainflu=ainflu/beamarea; + call show_transport_parameter(iout); + call srcout; goto :ANALYZE-PARALLEL:; ] ] From 66fea4bc052f42a61104ce9cdd653af7c31bc228 Mon Sep 17 00:00:00 2001 From: Ernesto Mainegra-Hing Date: Thu, 8 Oct 2020 21:38:10 -0400 Subject: [PATCH 20/23] Fix timing and atomic relaxations in CAVRZnrc Account for the overall cpu time to properly compute the number of histories per hour. Previously, the total time was not included, but the total number of histories was. Turn off atomic relaxations in the cavrznrc_template.egsinpsample input file, otherwise if fails because bound Compton scattering is turned off. Funnily, it doesn't fail if the photon xsection library SI is used. Why? --- .../user_codes/cavrznrc/cavrznrc.mortran | 26 +++++++++++-------- .../cavrznrc/cavrznrc_template.egsinp | 2 ++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran index 0521a8220..903f6b366 100644 --- a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran +++ b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.mortran @@ -2970,7 +2970,7 @@ $SHOW-RNG-STATE(iout); write(iout,'(a)') ' *********'; $SET_ELAPSED_TOTAL_TIME(TIMEB); ETIMETOT=ETIMETOT+TIMEB; $SET_ELAPSED_CPUTIME(CPUT2); -TIMCPU=CPUT2-CPUT1; +TIMCPU=CPUT2-CPUT1+TMCPUO; IF(IRESTART=3)["just analyzing data--no elapsed time" WRITE(IOUT,250)TMCPUO,TMCPUO/3600; WRITE(6,250)TMCPUO,TMCPUO/3600; @@ -2985,18 +2985,22 @@ ELSEIF(IRESTART=5)["output time results for parallel runs" ] ELSE[ IF(TMCPUO.EQ.0)["This is first run" - RATIO=ETIMETOT/TIMCPU; - WRITE(IOUT,260)ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; - WRITE(6,260)ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; + RATIO=ETIMETOT/TIMCPU; + WRITE(IOUT,260)ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; + WRITE(6,260)ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; + IF(IHSTRY.NE.0 .AND. TIMCPU.NE.0.0) ["this should always happen" + WRITE(IOUT,280) TIMCPU/dble(IHSTRY),3600.*dble(IHSTRY)/TIMCPU; + WRITE(6,280) TIMCPU/dble(IHSTRY),3600.*dble(IHSTRY)/TIMCPU; + ] ] ELSE[ "There was previous run, but don't have elapsed time for it" - RATIO = ETIMETOT/(TIMCPU-TMCPUO); "ratio this run only" - WRITE(IOUT,261) ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; - WRITE(6,261) ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; - ] - IF(IHSTRY.NE.0 .AND. TIMCPU.NE.0.0) ["this should always happen" - WRITE(IOUT,280) TIMCPU/dble(IHSTRY),3600.*dble(IHSTRY)/TIMCPU; - WRITE(6,280) TIMCPU/dble(IHSTRY),3600.*dble(IHSTRY)/TIMCPU; + RATIO = ETIMETOT/(TIMCPU-TMCPUO); "ratio this run only" + WRITE(IOUT,261) ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; + WRITE(6,261) ETIMETOT,TIMCPU,TIMCPU/3600.,RATIO; + IF(IHSTRY.NE.0 .AND. TIMCPU.NE.0.0) ["this should always happen" + WRITE(IOUT,280) TMCPUO/dble(IHSTRY),3600.*dble(IHSTRY)/TMCPUO; + WRITE(6,280) TMCPUO/dble(IHSTRY),3600.*dble(IHSTRY)/TMCPUO; + ] ] ] diff --git a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp index 40068ae35..cd3002ab8 100644 --- a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp +++ b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc_template.egsinp @@ -299,6 +299,8 @@ Brems cross sections= BH # BH (default),NIST; # BH: Bethe-Heitler cross-sections used # NIST: NIST cross-sections used +Atomic relaxations= Off # On (EADL) or Off + Bound Compton scattering= Off # Off, On, simple or norej (default); # Off: Klein-Nishina used for Compton # scattering From c71955dfc842b7927e2b257a74ee7dad583ea6ca Mon Sep 17 00:00:00 2001 From: Frederic Tessier Date: Mon, 12 Apr 2021 11:23:53 -0400 Subject: [PATCH 21/23] Adjust eol spaces, remove leftover merge markers --- HEN_HOUSE/egs++/egs_run_control.h | 1 + HEN_HOUSE/scripts/egs-jobsub-xargs | 16 +++++++++------- HEN_HOUSE/src/egs_parallel.mortran | 7 ++++--- .../cavrznrc/cavrznrc_template.egsinp | 2 +- HEN_HOUSE/user_codes/dosrznrc/dosrznrc.mortran | 18 ++---------------- 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/HEN_HOUSE/egs++/egs_run_control.h b/HEN_HOUSE/egs++/egs_run_control.h index 3067d649a..b27d66c49 100644 --- a/HEN_HOUSE/egs++/egs_run_control.h +++ b/HEN_HOUSE/egs++/egs_run_control.h @@ -24,6 +24,7 @@ # Author: Iwan Kawrakow, 2005 # # Contributors: Frederic Tessier +# Ernesto Mainegra-Hing # ############################################################################### */ diff --git a/HEN_HOUSE/scripts/egs-jobsub-xargs b/HEN_HOUSE/scripts/egs-jobsub-xargs index b5d3f7d40..faa9e4c14 100755 --- a/HEN_HOUSE/scripts/egs-jobsub-xargs +++ b/HEN_HOUSE/scripts/egs-jobsub-xargs @@ -29,6 +29,8 @@ # Blake Walters # ############################################################################### + + # # Check enough arguments provided # @@ -38,9 +40,9 @@ if test $# -lt 3; then Usage: $my_name app inp pegs|pegsless [p=N] [config=xxx] [eh=xxx] [hh=xxx] [cpu_t=t] [email=xxx] -$my_name runs app using inp as input file (use two double quotes if +$my_name runs app using inp as input file (use two double quotes if there is no input file needed) and pegs as pegs4 data file. -Using config=xxx you can use a configuration different from the one +Using config=xxx you can use a configuration different from the one specified by the environment variable EGS_CONFIG. For parallel job submission one needs to pass the number of jobs @@ -51,7 +53,7 @@ EOF fi -# +# # Initialization # app=$1 @@ -139,7 +141,7 @@ if test "x$egs_home" = x; then cat >&2 <>>>>>> Changes to Mortran apps for URC implementation " This parameter must be input even if not used. " ESAVEIN is a total energy. " @@ -3821,7 +3813,7 @@ ELSEIF(IFULL = 3)["score scattered dose separately" ] ] ELSE IF( iarg = 24 ) [ "Raylegh scattering has occured" - latch(NP) = IBSET(latch(NP),7); + latch(NP) = IBSET(latch(NP),7); ] "end iarg = 24 block" ] " end IFULL=3 " @@ -3855,7 +3847,7 @@ IF (IKERMA = 1) ["want to score KERMA" "depositing sub-threshold energy for PE & Compton" "Auger only" - IF( iarg = 33 | iarg = 34 ) [ + IF( iarg = 33 | iarg = 34 ) [ "33=>sub-threshold fluorescence being deposited" "34=>sub-threshold Auger being deposited" "if as a result of pe or compt this is k.e. transferred to e-" @@ -5080,13 +5072,7 @@ nbr_split =VALUE(NUM_BREMPEVEN,1); IF(IBRSPL = 1)["brems splitting requested" "default bremsstrahlung splitting to maximum" "IF((nbr_split <= 0) | (nbr_split > $MAXBRSPLIT)) nbr_split =$MAXBRSPLIT;" -<<<<<<< HEAD - "Now allows bremsstrahlung and annihilation radiation to be turned off" -||||||| merged common ancestors - "Changed to allow bremsstrahlung and annihilation radiation to be turned off" -======= "To allow bremsstrahlung and annihilation radiation to be turned off" ->>>>>>> Changes to Mortran apps for URC implementation IF((nbr_split > $MAXBRSPLIT)) nbr_split =$MAXBRSPLIT;" ] ELSE ["no bremsstrahlung splitting" nbr_split = 1;] From b08afa94d5ca11a421ea9873f17a1359f4025412 Mon Sep 17 00:00:00 2001 From: Frederic Tessier Date: Mon, 12 Apr 2021 12:55:54 -0400 Subject: [PATCH 22/23] Remove -j12 option for testing in CI pipeline --- HEN_HOUSE/scripts/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HEN_HOUSE/scripts/configure b/HEN_HOUSE/scripts/configure index 9079ca252..34f7e58dd 100755 --- a/HEN_HOUSE/scripts/configure +++ b/HEN_HOUSE/scripts/configure @@ -142,7 +142,7 @@ for prog in gmake make make.exe gmake.exe mingw32-make.exe; do IFS=$save_IFS $as_executable "$make_dir/$prog" || continue have_make=yes - make_prog="$prog -j12" + make_prog="$prog" abs_make="$make_dir/$prog" break done From 02258354210b5af0e5f01868fb721f15dccaeea7 Mon Sep 17 00:00:00 2001 From: Frederic Tessier Date: Mon, 12 Apr 2021 13:58:58 -0400 Subject: [PATCH 23/23] Run C preprocessor on Mortran apps to parse #ifdef --- HEN_HOUSE/user_codes/cavrznrc/cavrznrc.make | 2 +- HEN_HOUSE/user_codes/cavsphnrc/cavsphnrc.make | 2 +- HEN_HOUSE/user_codes/edknrc/edknrc.make | 2 +- HEN_HOUSE/user_codes/examin/examin.make | 6 ++++++ HEN_HOUSE/user_codes/g/g.make | 6 ++++++ HEN_HOUSE/user_codes/ranlux_test/ranlux_test.make | 6 ++++++ HEN_HOUSE/user_codes/ranmar_test/ranmar_test.make | 6 ++++++ HEN_HOUSE/user_codes/tutor1/tutor1.make | 6 ++++++ HEN_HOUSE/user_codes/tutor2/tutor2.make | 6 ++++++ HEN_HOUSE/user_codes/tutor3/tutor3.make | 6 ++++++ HEN_HOUSE/user_codes/tutor4/tutor4.make | 6 ++++++ HEN_HOUSE/user_codes/tutor5/tutor5.make | 6 ++++++ HEN_HOUSE/user_codes/tutor6/tutor6.make | 6 ++++++ HEN_HOUSE/user_codes/tutor7/tutor7.make | 6 ++++++ 14 files changed, 69 insertions(+), 3 deletions(-) diff --git a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.make b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.make index 6bd3562cc..3cddf907b 100644 --- a/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.make +++ b/HEN_HOUSE/user_codes/cavrznrc/cavrznrc.make @@ -42,7 +42,7 @@ EGS_EXTRA_LIBS = $(FLIBS) $(BEAMLIB_EXTRA_LIBS) RANDOM = $(EGS_SOURCEDIR)ranmar # We use #ifdef ... #endif for parallel processing implementation -# => Fortran file extension must me .F so that the C-preprocessor is +# => Fortran file extension must be .F so that the C-preprocessor is # automatically invoked. # FEXT = F diff --git a/HEN_HOUSE/user_codes/cavsphnrc/cavsphnrc.make b/HEN_HOUSE/user_codes/cavsphnrc/cavsphnrc.make index 995329d41..5d1996882 100644 --- a/HEN_HOUSE/user_codes/cavsphnrc/cavsphnrc.make +++ b/HEN_HOUSE/user_codes/cavsphnrc/cavsphnrc.make @@ -40,7 +40,7 @@ EGS_EXTRA_LIBS = $(FLIBS) RANDOM = $(EGS_SOURCEDIR)ranmar # We use #ifdef ... #endif for parallel processing implementation -# => Fortran file extension must me .F so that the C-preprocessor is +# => Fortran file extension must be .F so that the C-preprocessor is # automatically invoked. # FEXT = F diff --git a/HEN_HOUSE/user_codes/edknrc/edknrc.make b/HEN_HOUSE/user_codes/edknrc/edknrc.make index d6cc19746..fa12f4549 100644 --- a/HEN_HOUSE/user_codes/edknrc/edknrc.make +++ b/HEN_HOUSE/user_codes/edknrc/edknrc.make @@ -40,7 +40,7 @@ EGS_EXTRA_LIBS = $(FLIBS) RANDOM = $(EGS_SOURCEDIR)ranmar # We use #ifdef ... #endif for parallel processing implementation -# => Fortran file extension must me .F so that the C-preprocessor is +# => Fortran file extension must be .F so that the C-preprocessor is # automatically invoked. # FEXT = F diff --git a/HEN_HOUSE/user_codes/examin/examin.make b/HEN_HOUSE/user_codes/examin/examin.make index d2b9ed1e5..48678b976 100644 --- a/HEN_HOUSE/user_codes/examin/examin.make +++ b/HEN_HOUSE/user_codes/examin/examin.make @@ -32,6 +32,12 @@ # EGS_EXTRA_OBJECTS = +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F + # The mortran sources # Note: order is important! # Note: don't forget the leading tabs on continuation lines! diff --git a/HEN_HOUSE/user_codes/g/g.make b/HEN_HOUSE/user_codes/g/g.make index 4d8c36ada..f00ed14c7 100644 --- a/HEN_HOUSE/user_codes/g/g.make +++ b/HEN_HOUSE/user_codes/g/g.make @@ -50,6 +50,12 @@ # EGS_EXTRA_OBJECTS = +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F + # We prefer using the ranmar generator => we overwrite the # default from $HEN_HOUSE/specs/all_common.spec # diff --git a/HEN_HOUSE/user_codes/ranlux_test/ranlux_test.make b/HEN_HOUSE/user_codes/ranlux_test/ranlux_test.make index 5e36d8f33..b39216607 100644 --- a/HEN_HOUSE/user_codes/ranlux_test/ranlux_test.make +++ b/HEN_HOUSE/user_codes/ranlux_test/ranlux_test.make @@ -36,6 +36,12 @@ RANDOM = $(EGS_SOURCEDIR)ranlux # EGS_EXTRA_OBJECTS = +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F + # The mortran sources # Note: order is important! # Note: don't forget the leading tabs on continuation lines! diff --git a/HEN_HOUSE/user_codes/ranmar_test/ranmar_test.make b/HEN_HOUSE/user_codes/ranmar_test/ranmar_test.make index 1c1b28021..e73e7c80f 100644 --- a/HEN_HOUSE/user_codes/ranmar_test/ranmar_test.make +++ b/HEN_HOUSE/user_codes/ranmar_test/ranmar_test.make @@ -36,6 +36,12 @@ RANDOM = $(EGS_SOURCEDIR)ranmar # EGS_EXTRA_OBJECTS = +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F + # The mortran sources # Note: order is important! # Note: don't forget the leading tabs on continuation lines! diff --git a/HEN_HOUSE/user_codes/tutor1/tutor1.make b/HEN_HOUSE/user_codes/tutor1/tutor1.make index 7517fbfed..da68b19f6 100644 --- a/HEN_HOUSE/user_codes/tutor1/tutor1.make +++ b/HEN_HOUSE/user_codes/tutor1/tutor1.make @@ -50,3 +50,9 @@ # being used). # EGS_EXTRA_OBJECTS = + +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F diff --git a/HEN_HOUSE/user_codes/tutor2/tutor2.make b/HEN_HOUSE/user_codes/tutor2/tutor2.make index 965c81aee..188872d6f 100644 --- a/HEN_HOUSE/user_codes/tutor2/tutor2.make +++ b/HEN_HOUSE/user_codes/tutor2/tutor2.make @@ -50,3 +50,9 @@ # being used). # EGS_EXTRA_OBJECTS = + +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F diff --git a/HEN_HOUSE/user_codes/tutor3/tutor3.make b/HEN_HOUSE/user_codes/tutor3/tutor3.make index acc7de742..acf29afdb 100644 --- a/HEN_HOUSE/user_codes/tutor3/tutor3.make +++ b/HEN_HOUSE/user_codes/tutor3/tutor3.make @@ -50,3 +50,9 @@ # being used). # EGS_EXTRA_OBJECTS = + +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F diff --git a/HEN_HOUSE/user_codes/tutor4/tutor4.make b/HEN_HOUSE/user_codes/tutor4/tutor4.make index 592333491..614bf668f 100644 --- a/HEN_HOUSE/user_codes/tutor4/tutor4.make +++ b/HEN_HOUSE/user_codes/tutor4/tutor4.make @@ -50,3 +50,9 @@ # being used). # EGS_EXTRA_OBJECTS = + +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F diff --git a/HEN_HOUSE/user_codes/tutor5/tutor5.make b/HEN_HOUSE/user_codes/tutor5/tutor5.make index fcb60bc40..68bc9f711 100644 --- a/HEN_HOUSE/user_codes/tutor5/tutor5.make +++ b/HEN_HOUSE/user_codes/tutor5/tutor5.make @@ -50,3 +50,9 @@ # being used). # EGS_EXTRA_OBJECTS = + +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F diff --git a/HEN_HOUSE/user_codes/tutor6/tutor6.make b/HEN_HOUSE/user_codes/tutor6/tutor6.make index 40cbf1a08..d1cea794e 100644 --- a/HEN_HOUSE/user_codes/tutor6/tutor6.make +++ b/HEN_HOUSE/user_codes/tutor6/tutor6.make @@ -50,3 +50,9 @@ # being used). # EGS_EXTRA_OBJECTS = + +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F diff --git a/HEN_HOUSE/user_codes/tutor7/tutor7.make b/HEN_HOUSE/user_codes/tutor7/tutor7.make index 04fce8bf2..af31d5f3d 100644 --- a/HEN_HOUSE/user_codes/tutor7/tutor7.make +++ b/HEN_HOUSE/user_codes/tutor7/tutor7.make @@ -51,6 +51,12 @@ # EGS_EXTRA_OBJECTS = +# We use #ifdef ... #endif for parallel processing implementation +# => Fortran file extension must be .F so that the C-preprocessor is +# automatically invoked. +# +FEXT = F + # We prefer using the ranmar generator => we overwrite the # default from $HEN_HOUSE/specs/all_common.spec #