diff --git a/CMakeLists.txt b/CMakeLists.txt index d71c7092..eabf3820 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,8 @@ unset(CMAKE_CXX_FLAGS_RELEASE CACHE) # read debug and optimized flags from command line option(BUILD_CROWNLIB_ONLY "Build only the CROWNLIB library" OFF) +set(REBUILD_CROWN_LIB "false") # used for non-production mode + if (NOT DEFINED DEBUG) message(STATUS "No Debug mode set, activate with -DDEBUG=true --> compile with debug symbols and run code generation with debug output") set(DEBUG "false") @@ -35,9 +37,17 @@ else() endif() # Only parse additional args if not building only the CROWNLIB library if(NOT BUILD_CROWNLIB_ONLY) + if (NOT DEFINED ANALYSIS) message(FATAL_ERROR "Please specify the Analysis to be used with -DANALYSIS=my_analysis_name") endif() + # if analysis is set, check the folder to find any potential payload files to be used + file(GLOB PAYLOADS ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads/*) + if (NOT PAYLOADS) + message(STATUS "No payload files found in ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads/ for analysis ${ANALYSIS}") + else() + message(STATUS "Found payload files in ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads/ for analysis ${ANALYSIS}") + endif() if (NOT DEFINED CONFIG) message(FATAL_ERROR "Please specify the config to be used with -DCONFIG=my_config_name") @@ -68,6 +78,10 @@ if(NOT BUILD_CROWNLIB_ONLY) message(FATAL_ERROR "Please specify the eras to be used with -DERAS=eras") endif() + if (NOT DEFINED PRODUCTION) + message(STATUS "No production mode set --> will rebuild the CROWNLIB library if necessary") + set(REBUILD_CROWN_LIB "true") + endif() if (NOT DEFINED THREADS) message(STATUS "No threads set, using single threaded mode with -DTHREADS=1") set(THREADS "1") @@ -279,7 +293,7 @@ if(BUILD_CROWNLIB_ONLY) endif() # check if CROWNLIB is already installed find_library(CROWNLIB_FOUND CROWNLIB HINTS ${INSTALLDIR}/lib ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/lib) -if(NOT CROWNLIB_FOUND) +if(NOT CROWNLIB_FOUND OR REBUILD_CROWN_LIB) message(STATUS "CROWNLIB not found, building it") # CROWNLIB not found, build it add_library(CROWNLIB SHARED ${SOURCES}) @@ -339,6 +353,9 @@ STRING(REGEX REPLACE "\n" ";" FILELIST ${FILELIST}) set(TARGET_NAMES "") # copy all correction files into the install location install(DIRECTORY data/ DESTINATION ${INSTALLDIR}/data) +if (PAYLOADS) + install(DIRECTORY ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads DESTINATION ${INSTALLDIR}) +endif() # also copy inish script needed for job tarball install(FILES init.sh DESTINATION ${INSTALLDIR}) diff --git a/code_generation/analysis_template.cxx b/code_generation/analysis_template.cxx index fb57920e..51953df3 100644 --- a/code_generation/analysis_template.cxx +++ b/code_generation/analysis_template.cxx @@ -107,6 +107,7 @@ int main(int argc, char *argv[]) { // initialize df ROOT::RDataFrame df0(basetree, input_files); + ROOT::RDF::Experimental::AddProgressBar(df0); Logger::get("main")->info("Starting Setup of Dataframe with {} events", nevents); std::vector> cutReports; diff --git a/code_generation/analysis_template_friends.cxx b/code_generation/analysis_template_friends.cxx index 049475da..03ae7ff5 100644 --- a/code_generation/analysis_template_friends.cxx +++ b/code_generation/analysis_template_friends.cxx @@ -149,6 +149,7 @@ int main(int argc, char *argv[]) { } // initialize df ROOT::RDataFrame df0(dataset); + ROOT::RDF::Experimental::AddProgressBar(df0); // print all available branches to the log Logger::get("main")->debug("Available branches:"); for (auto const &branch : df0.GetColumnNames()) { diff --git a/code_generation/code_generation.py b/code_generation/code_generation.py index 3672265d..50ae5f6c 100644 --- a/code_generation/code_generation.py +++ b/code_generation/code_generation.py @@ -347,7 +347,6 @@ def write_code(self, calls: str, includes: str, run_commands: str) -> None: ) .replace("{ANALYSISTAG}", '"Analysis={}"'.format(self.analysis_name)) .replace("{CONFIGTAG}", '"Config={}"'.format(self.config_name)) - .replace("{PROGRESS_CALLBACK}", self.set_process_tracking()) .replace("{OUTPUT_QUANTITIES}", self.set_output_quantities()) .replace("{SHIFT_QUANTITIES_MAP}", self.set_shift_quantities_map()) .replace("{QUANTITIES_SHIFT_MAP}", self.set_quantities_shift_map()) @@ -548,8 +547,6 @@ def generate_run_commands(self) -> str: outputname=self._outputfiles_generated[scope], outputstring=outputstring, ) - # add code for tracking the progress - runcommands += self.set_process_tracking() # add code for the time taken for the dataframe setup runcommands += self.set_setup_printout() # add trigger of dataframe execution, for nonempty scopes @@ -684,37 +681,6 @@ def zero_events_fallback(self) -> str: return printout - def set_process_tracking(self) -> str: - """This function replaces the template placeholder for the process tracking with the correct process tracking. - - Returns: - The code to be added to the template - """ - tracking = "" - scope = self.scopes[-1] - tracking += " ULong64_t {scope}_processed = 0;\n".format(scope=scope) - tracking += " std::mutex {scope}_bar_mutex;\n".format(scope=scope) - tracking += " auto c_{scope} = df{counter}_{scope}.Count();\n".format( - counter=self.main_counter[scope], scope=scope - ) - tracking += " c_{scope}.OnPartialResultSlot(quantile, [&{scope}_bar_mutex, &{scope}_processed, &quantile, &nevents](unsigned int /*slot*/, ULong64_t /*_c*/) {{".format( - scope=scope - ) - tracking += ( - "\n std::lock_guard lg({scope}_bar_mutex);\n".format( - scope=scope - ) - ) - tracking += " {scope}_processed += quantile;\n".format(scope=scope) - tracking += " float percentage = 100 * (float){scope}_processed / (float)nevents;\n".format( - scope=scope - ) - tracking += ' Logger::get("main")->info("{{0:d}} / {{1:d}} ({{2:.2f}} %) Events processed ...", {scope}_processed, nevents, percentage);\n'.format( - scope=scope - ) - tracking += " });\n" - return tracking - def set_shift_quantities_map(self) -> str: """ This function is used to generate a mapping of all quantities and the shifts, diff --git a/init.sh b/init.sh index 30606bed..c79de9a8 100644 --- a/init.sh +++ b/init.sh @@ -7,8 +7,15 @@ pathadd() { } # get the directory of the script SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)" -distro=$(lsb_release -i | cut -f2) -os_version=$(lsb_release -r | cut -f2) +if ! command -v lsb_release &> /dev/null +then + source /etc/os-release + distro=$NAME + os_version=$VERSION_ID +else + distro=$(lsb_release -i | cut -f2) + os_version=$(lsb_release -r | cut -f2) +fi echo "Setting up CROWN for $distro Version $os_version" # check if the distro is centos if [[ "$distro" == "CentOS" ]]; then @@ -17,15 +24,18 @@ if [[ "$distro" == "CentOS" ]]; then # source /cvmfs/sft-nightlies.cern.ch/lcg/views/dev3/latest/x86_64-centos7-gcc11-opt/setup.sh # source /cvmfs/sft-nightlies.cern.ch/lcg/views/dev3/latest/x86_64-centos7-clang12-opt/setup.sh # source /cvmfs/sft-nightlies.cern.ch/lcg/views/dev3/latest/x86_64-centos7-gcc11-dbg/setup.sh - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-centos7-gcc11-opt/setup.sh + source /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-centos7-gcc11-opt/setup.sh else echo "Unsupported CentOS version, exiting..." return 0 fi -elif [[ "$distro" == "RedHatEnterprise" ]]; then +elif [[ "$distro" == "RedHatEnterprise" || "$distro" == "AlmaLinux" || "$distro" == "RockyLinux" ]]; then if [[ ${os_version:0:1} == "8" ]]; then # elif uname -a | grep -E 'el8' -q # no lcg 103 available for centOS 8 source /cvmfs/sft.cern.ch/lcg/views/LCG_102/x86_64-centos8-gcc11-opt/setup.sh + elif [[ ${os_version:0:1} == "9" ]]; then # elif uname -a | grep -E 'el8' -q + # no lcg 103 available for centOS 8 + source /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-el9-gcc11-opt/setup.sh else echo "Unsupported CentOS version, exiting..." return 0 @@ -34,7 +44,7 @@ elif [[ "$distro" == "Ubuntu" ]]; then if [[ ${os_version:0:2} == "20" ]]; then source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-ubuntu2004-gcc9-opt/setup.sh elif [[ ${os_version:0:2} == "22" ]]; then - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-ubuntu2204-gcc11-opt/setup.sh + source /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh else echo "Unsupported Ubuntu version, exiting..." return 0