Payload deployment, ROOT 6.30 (#241)

CMakeLists.txt

@@ -5,6 +5,8 @@ unset(CMAKE_CXX_FLAGS_RELEASE CACHE)
 
 # read debug and optimized flags from command line
 option(BUILD_CROWNLIB_ONLY "Build only the CROWNLIB library" OFF)
+set(REBUILD_CROWN_LIB "false") # used for non-production mode
+
 if (NOT DEFINED DEBUG)
     message(STATUS "No Debug mode set, activate with -DDEBUG=true --> compile with debug symbols and run code generation with debug output")
     set(DEBUG "false")
@@ -35,9 +37,17 @@ else()
 endif()
 # Only parse additional args if not building only the CROWNLIB library
 if(NOT BUILD_CROWNLIB_ONLY)
+
     if (NOT DEFINED ANALYSIS)
         message(FATAL_ERROR "Please specify the Analysis to be used with -DANALYSIS=my_analysis_name")
     endif()
+    # if analysis is set, check the folder to find any potential payload files to be used
+    file(GLOB PAYLOADS ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads/*)
+    if (NOT PAYLOADS)
+        message(STATUS "No payload files found in ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads/ for analysis ${ANALYSIS}")
+    else()
+        message(STATUS "Found payload files in ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads/ for analysis ${ANALYSIS}")
+    endif()
 
     if (NOT DEFINED CONFIG)
         message(FATAL_ERROR "Please specify the config to be used with -DCONFIG=my_config_name")
@@ -68,6 +78,10 @@ if(NOT BUILD_CROWNLIB_ONLY)
         message(FATAL_ERROR "Please specify the eras to be used with -DERAS=eras")
     endif()
 
+    if (NOT DEFINED PRODUCTION)
+        message(STATUS "No production mode set --> will rebuild the CROWNLIB library if necessary")
+        set(REBUILD_CROWN_LIB "true")
+    endif()
     if (NOT DEFINED THREADS)
         message(STATUS "No threads set, using single threaded mode with -DTHREADS=1")
         set(THREADS "1")
@@ -279,7 +293,7 @@ if(BUILD_CROWNLIB_ONLY)
 endif()
 # check if CROWNLIB is already installed
 find_library(CROWNLIB_FOUND CROWNLIB HINTS ${INSTALLDIR}/lib ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/lib)
-if(NOT CROWNLIB_FOUND)
+if(NOT CROWNLIB_FOUND OR REBUILD_CROWN_LIB)
     message(STATUS "CROWNLIB not found, building it")
     # CROWNLIB not found, build it
     add_library(CROWNLIB SHARED ${SOURCES})
@@ -339,6 +353,9 @@ STRING(REGEX REPLACE "\n" ";" FILELIST ${FILELIST})
 set(TARGET_NAMES "")
 # copy all correction files into the install location
 install(DIRECTORY data/ DESTINATION ${INSTALLDIR}/data)
+if (PAYLOADS)
+    install(DIRECTORY ${CMAKE_SOURCE_DIR}/analysis_configurations/${ANALYSIS}/payloads DESTINATION ${INSTALLDIR})
+endif()
 
 # also copy inish script needed for job tarball
 install(FILES init.sh DESTINATION ${INSTALLDIR})

code_generation/analysis_template.cxx

@@ -107,6 +107,7 @@ int main(int argc, char *argv[]) {
 
     // initialize df
     ROOT::RDataFrame df0(basetree, input_files);
+    ROOT::RDF::Experimental::AddProgressBar(df0);
     Logger::get("main")->info("Starting Setup of Dataframe with {} events",
                               nevents);
     std::vector<ROOT::RDF::RResultPtr<ROOT::RDF::RCutFlowReport>> cutReports;

code_generation/analysis_template_friends.cxx

@@ -149,6 +149,7 @@ int main(int argc, char *argv[]) {
     }
     // initialize df
     ROOT::RDataFrame df0(dataset);
+    ROOT::RDF::Experimental::AddProgressBar(df0);
     // print all available branches to the log
     Logger::get("main")->debug("Available branches:");
     for (auto const &branch : df0.GetColumnNames()) {

code_generation/code_generation.py

@@ -347,7 +347,6 @@ def write_code(self, calls: str, includes: str, run_commands: str) -> None:
                 )
                 .replace("{ANALYSISTAG}", '"Analysis={}"'.format(self.analysis_name))
                 .replace("{CONFIGTAG}", '"Config={}"'.format(self.config_name))
-                .replace("{PROGRESS_CALLBACK}", self.set_process_tracking())
                 .replace("{OUTPUT_QUANTITIES}", self.set_output_quantities())
                 .replace("{SHIFT_QUANTITIES_MAP}", self.set_shift_quantities_map())
                 .replace("{QUANTITIES_SHIFT_MAP}", self.set_quantities_shift_map())
@@ -548,8 +547,6 @@ def generate_run_commands(self) -> str:
                     outputname=self._outputfiles_generated[scope],
                     outputstring=outputstring,
                 )
-        # add code for tracking the progress
-        runcommands += self.set_process_tracking()
         # add code for the time taken for the dataframe setup
         runcommands += self.set_setup_printout()
         # add trigger of dataframe execution, for nonempty scopes
@@ -684,37 +681,6 @@ def zero_events_fallback(self) -> str:
 
         return printout
 
-    def set_process_tracking(self) -> str:
-        """This function replaces the template placeholder for the process tracking with the correct process tracking.
-
-        Returns:
-            The code to be added to the template
-        """
-        tracking = ""
-        scope = self.scopes[-1]
-        tracking += "        ULong64_t {scope}_processed = 0;\n".format(scope=scope)
-        tracking += "        std::mutex {scope}_bar_mutex;\n".format(scope=scope)
-        tracking += "        auto c_{scope} = df{counter}_{scope}.Count();\n".format(
-            counter=self.main_counter[scope], scope=scope
-        )
-        tracking += "        c_{scope}.OnPartialResultSlot(quantile, [&{scope}_bar_mutex, &{scope}_processed, &quantile, &nevents](unsigned int /*slot*/, ULong64_t /*_c*/) {{".format(
-            scope=scope
-        )
-        tracking += (
-            "\n            std::lock_guard<std::mutex> lg({scope}_bar_mutex);\n".format(
-                scope=scope
-            )
-        )
-        tracking += "            {scope}_processed += quantile;\n".format(scope=scope)
-        tracking += "            float percentage = 100 * (float){scope}_processed / (float)nevents;\n".format(
-            scope=scope
-        )
-        tracking += '            Logger::get("main")->info("{{0:d}} / {{1:d}} ({{2:.2f}} %) Events processed ...", {scope}_processed, nevents, percentage);\n'.format(
-            scope=scope
-        )
-        tracking += "        });\n"
-        return tracking
-
     def set_shift_quantities_map(self) -> str:
         """
         This function is used to generate a mapping of all quantities and the shifts,

init.sh

@@ -7,8 +7,15 @@ pathadd() {
 }
 # get the directory of the script
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)"
-distro=$(lsb_release -i | cut -f2)
-os_version=$(lsb_release -r | cut -f2)
+if ! command -v lsb_release &> /dev/null
+then
+    source /etc/os-release
+    distro=$NAME
+    os_version=$VERSION_ID
+else
+    distro=$(lsb_release -i | cut -f2)
+    os_version=$(lsb_release -r | cut -f2)
+fi
 echo "Setting up CROWN for $distro Version $os_version"
 # check if the distro is centos
 if [[ "$distro" == "CentOS" ]]; then
@@ -17,15 +24,18 @@ if [[ "$distro" == "CentOS" ]]; then
         # source /cvmfs/sft-nightlies.cern.ch/lcg/views/dev3/latest/x86_64-centos7-gcc11-opt/setup.sh
         # source /cvmfs/sft-nightlies.cern.ch/lcg/views/dev3/latest/x86_64-centos7-clang12-opt/setup.sh
         # source /cvmfs/sft-nightlies.cern.ch/lcg/views/dev3/latest/x86_64-centos7-gcc11-dbg/setup.sh
-        source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-centos7-gcc11-opt/setup.sh
+        source /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-centos7-gcc11-opt/setup.sh
     else
         echo "Unsupported CentOS version, exiting..."
         return 0
     fi
-elif [[ "$distro" == "RedHatEnterprise" ]]; then
+elif [[ "$distro" == "RedHatEnterprise" || "$distro" == "AlmaLinux" || "$distro" == "RockyLinux" ]]; then
     if [[ ${os_version:0:1} == "8" ]]; then # elif uname -a | grep -E 'el8' -q
         # no lcg 103 available for centOS 8
         source /cvmfs/sft.cern.ch/lcg/views/LCG_102/x86_64-centos8-gcc11-opt/setup.sh
+    elif [[ ${os_version:0:1} == "9" ]]; then # elif uname -a | grep -E 'el8' -q
+        # no lcg 103 available for centOS 8
+        source /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-el9-gcc11-opt/setup.sh
     else
         echo "Unsupported CentOS version, exiting..."
         return 0
@@ -34,7 +44,7 @@ elif [[ "$distro" == "Ubuntu" ]]; then
     if [[ ${os_version:0:2} == "20" ]]; then
         source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-ubuntu2004-gcc9-opt/setup.sh
     elif [[ ${os_version:0:2} == "22" ]]; then
-        source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-ubuntu2204-gcc11-opt/setup.sh
+        source /cvmfs/sft.cern.ch/lcg/views/LCG_105/x86_64-ubuntu2204-gcc11-opt/setup.sh
     else
         echo "Unsupported Ubuntu version, exiting..."
         return 0