Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ML inference #235

Draft
wants to merge 41 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
c06eb4f
moving to newer ROOT version for EL8
nfaltermann Nov 10, 2023
75d1b28
adding libPyMVA for ML inference
nfaltermann Nov 10, 2023
906443c
first working version
nfaltermann Nov 10, 2023
9e774be
refactoring for other ML model parser
nfaltermann Nov 11, 2023
909e604
PyTorch model parser
nfaltermann Nov 11, 2023
b65437c
cosmetics
nfaltermann Nov 11, 2023
0fdf436
build requirements
nfaltermann Nov 11, 2023
a804bb7
ONNX parser ... but corresponding libs conflict with other parsers (?)
nfaltermann Nov 11, 2023
524565d
fixing syst shifts
nfaltermann Nov 15, 2023
3fb2ca9
condense multiple outputs
nfaltermann Nov 16, 2023
f984e12
supporting multiple different instances at once
nfaltermann Nov 16, 2023
06c0423
add HLTMu20 PU profile histos
demuller Nov 27, 2023
51bdcf4
switch to builtin progress tracking
harrypuuter Jan 16, 2024
77f31e8
update lcg stack for ROOT 6.30
harrypuuter Jan 16, 2024
b452b7e
add automatic installation of payload folder, if existent, rebuild cr…
harrypuuter Jan 16, 2024
dab70ea
fix payloads path
harrypuuter Jan 16, 2024
b626d76
merge lcg_105
nfaltermann Jan 16, 2024
f39695a
Merge remote-tracking branch 'origin/lcg_105' into sofie_dev
nfaltermann Jan 16, 2024
e9e65fc
addition of QCD validation region (topreco)
demuller Feb 2, 2024
0e9a3c7
Merge pull request #247 from demuller/sofie_dev
nfaltermann Feb 2, 2024
e8ae518
Merge branch 'sofie_dev' of github.com:KIT-CMS/CROWN into sofie_dev
nfaltermann Feb 2, 2024
e3c2a31
clang
nfaltermann Feb 2, 2024
db39040
sync with main
nfaltermann Feb 3, 2024
5ce8813
fix of btag SF application for QCD validation region
demuller Feb 9, 2024
7fd2040
Merge pull request #249 from demuller/sofie_dev
nfaltermann Feb 9, 2024
943c585
function to select quantities on exact match
nfaltermann Feb 13, 2024
3d5c6c9
Merge branch 'intfilter' into sofie_dev
nfaltermann Feb 13, 2024
b9b4bc1
electron selection based on bitmaps; add function for selecting elect…
demuller Feb 29, 2024
f1327ad
remove duplicated SelectInt RNode definition
demuller Mar 4, 2024
8eeb919
fix prescale + clang
nfaltermann Mar 14, 2024
4a86853
Merge pull request #253 from demuller/bitmap
nfaltermann Mar 14, 2024
787112e
Merge branch 'prescale_fix' into sofie_dev
nfaltermann Mar 14, 2024
68a25b7
updated top pT reweighting
nfaltermann Jun 7, 2024
a2c9968
fixing some weird merging stuff (?)
nfaltermann Jun 7, 2024
d282ddd
Revert "updated top pT reweighting"
nfaltermann Jun 7, 2024
0a220b4
update top pT reweighting
nfaltermann Jun 7, 2024
f3f642f
delete PU HLTMu20 info (now stored in payloads of CROWNs)
demuller Jul 18, 2024
75ab541
Merge pull request #274 from demuller/sofie_dev
nfaltermann Jul 19, 2024
49264a9
added 2018 btag eff map for tight WP
demuller Oct 5, 2024
1fab01c
Merge branch 'KIT-CMS:sofie_dev' into sofie_dev
demuller Oct 5, 2024
4443053
Merge pull request #283 from demuller/sofie_dev
nfaltermann Oct 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ message(STATUS "Creating Project")
project(CROWN CXX)
message(STATUS "Finding Packages")
# Find ROOT and print details
find_package(ROOT 6.26 REQUIRED COMPONENTS ROOTVecOps ROOTDataFrame RooFit GenVector)
find_package(ROOT 6.28 REQUIRED COMPONENTS ROOTVecOps ROOTDataFrame RooFit GenVector PyMVA)
# add OpenMP and MPI
find_package(OpenMP)
find_package(MPI)
Expand Down Expand Up @@ -287,7 +287,7 @@ if(BUILD_CROWNLIB_ONLY)
message(STATUS "Building only the CROWNLIB library")
add_library(CROWNLIB SHARED ${SOURCES})
target_include_directories(CROWNLIB PRIVATE ${CMAKE_SOURCE_DIR} ${ROOT_INCLUDE_DIRS})
target_link_libraries(CROWNLIB ROOT::ROOTVecOps ROOT::ROOTDataFrame ROOT::RooFit ROOT::GenVector logging correctionlib nlohmann_json::nlohmann_json)
target_link_libraries(CROWNLIB ROOT::ROOTVecOps ROOT::ROOTDataFrame ROOT::RooFit ROOT::GenVector ROOT::PyMVA logging correctionlib nlohmann_json::nlohmann_json)
install(TARGETS CROWNLIB DESTINATION ${INSTALLDIR}/lib )
return()
endif()
Expand All @@ -298,7 +298,7 @@ if(NOT CROWNLIB_FOUND OR REBUILD_CROWN_LIB)
# CROWNLIB not found, build it
add_library(CROWNLIB SHARED ${SOURCES})
target_include_directories(CROWNLIB PRIVATE ${CMAKE_SOURCE_DIR} ${ROOT_INCLUDE_DIRS})
target_link_libraries(CROWNLIB ROOT::ROOTVecOps ROOT::ROOTDataFrame ROOT::RooFit ROOT::GenVector logging correctionlib nlohmann_json::nlohmann_json)
target_link_libraries(CROWNLIB ROOT::ROOTVecOps ROOT::ROOTDataFrame ROOT::RooFit ROOT::GenVector ROOT::PyMVA logging correctionlib nlohmann_json::nlohmann_json)
install(TARGETS CROWNLIB DESTINATION ${INSTALLDIR}/lib)
else()
message(STATUS "Found CROWNLIB in ${CROWNLIB_FOUND}")
Expand Down Expand Up @@ -380,7 +380,7 @@ foreach(FILENAME ${FILELIST})
add_executable(${TARGET_NAME} ${FULL_PATH} ${GENERATED_CXX_FILES})
# Adds a pre-build event to the Target copying the correctionlib.so file into the /lib folder in the install directory
target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_SOURCE_DIR} ${ROOT_INCLUDE_DIRS} $ORIGIN/lib/ lib/)
target_link_libraries(${TARGET_NAME} ROOT::ROOTVecOps ROOT::ROOTDataFrame ROOT::RooFit ROOT::GenVector logging correctionlib nlohmann_json::nlohmann_json CROWNLIB)
target_link_libraries(${TARGET_NAME} ROOT::ROOTVecOps ROOT::ROOTDataFrame ROOT::RooFit ROOT::GenVector ROOT::PyMVA logging correctionlib nlohmann_json::nlohmann_json CROWNLIB)
add_custom_command(TARGET ${TARGET_NAME} PRE_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${CORRECTION_LIB_PATH}"
Expand Down
1 change: 1 addition & 0 deletions code_generation/analysis_template.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "include/lorentzvectors.hxx"
#include "include/met.hxx"
#include "include/metfilter.hxx"
#include "include/ml.hxx"
#include "include/pairselection.hxx"
#include "include/tripleselection.hxx"
#include "include/physicsobjects.hxx"
Expand Down
1 change: 1 addition & 0 deletions code_generation/analysis_template_friends.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "include/lorentzvectors.hxx"
#include "include/met.hxx"
#include "include/metfilter.hxx"
#include "include/ml.hxx"
#include "include/pairselection.hxx"
#include "include/physicsobjects.hxx"
#include "include/quantities.hxx"
Expand Down
34 changes: 0 additions & 34 deletions code_generation/code_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,6 @@ def write_code(self, calls: str, includes: str, run_commands: str) -> None:
)
.replace("{ANALYSISTAG}", '"Analysis={}"'.format(self.analysis_name))
.replace("{CONFIGTAG}", '"Config={}"'.format(self.config_name))
.replace("{PROGRESS_CALLBACK}", self.set_process_tracking())
.replace("{OUTPUT_QUANTITIES}", self.set_output_quantities())
.replace("{SHIFT_QUANTITIES_MAP}", self.set_shift_quantities_map())
.replace("{QUANTITIES_SHIFT_MAP}", self.set_quantities_shift_map())
Expand Down Expand Up @@ -548,8 +547,6 @@ def generate_run_commands(self) -> str:
outputname=self._outputfiles_generated[scope],
outputstring=outputstring,
)
# add code for tracking the progress
runcommands += self.set_process_tracking()
# add code for the time taken for the dataframe setup
runcommands += self.set_setup_printout()
# add trigger of dataframe execution, for nonempty scopes
Expand Down Expand Up @@ -684,37 +681,6 @@ def zero_events_fallback(self) -> str:

return printout

def set_process_tracking(self) -> str:
"""This function replaces the template placeholder for the process tracking with the correct process tracking.

Returns:
The code to be added to the template
"""
tracking = ""
scope = self.scopes[-1]
tracking += " ULong64_t {scope}_processed = 0;\n".format(scope=scope)
tracking += " std::mutex {scope}_bar_mutex;\n".format(scope=scope)
tracking += " auto c_{scope} = df{counter}_{scope}.Count();\n".format(
counter=self.main_counter[scope], scope=scope
)
tracking += " c_{scope}.OnPartialResultSlot(quantile, [&{scope}_bar_mutex, &{scope}_processed, &quantile, &nevents](unsigned int /*slot*/, ULong64_t /*_c*/) {{".format(
scope=scope
)
tracking += (
"\n std::lock_guard<std::mutex> lg({scope}_bar_mutex);\n".format(
scope=scope
)
)
tracking += " {scope}_processed += quantile;\n".format(scope=scope)
tracking += " float percentage = 100 * (float){scope}_processed / (float)nevents;\n".format(
scope=scope
)
tracking += ' Logger::get("main")->info("{{0:d}} / {{1:d}} ({{2:.2f}} %) Events processed ...", {scope}_processed, nevents, percentage);\n'.format(
scope=scope
)
tracking += " });\n"
return tracking

def set_shift_quantities_map(self) -> str:
"""
This function is used to generate a mapping of all quantities and the shifts,
Expand Down
1 change: 1 addition & 0 deletions code_generation/subset_template.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "include/lorentzvectors.hxx"
#include "include/met.hxx"
#include "include/metfilter.hxx"
#include "include/ml.hxx"
#include "include/pairselection.hxx"
#include "include/tripleselection.hxx"
#include "include/physicsobjects.hxx"
Expand Down
Binary file not shown.
Loading