From 971ae2da7efec5a167b4ab6ecf365c0e27ca41ca Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:27:49 +0200 Subject: [PATCH] Add a filter algorithm (#210) --- k4FWCore/include/k4FWCore/FilterPredicate.h | 27 ++++++++++ k4FWCore/include/k4FWCore/FunctionalUtils.h | 47 ++++++++++++++++- test/k4FWCoreTest/CMakeLists.txt | 6 ++- test/k4FWCoreTest/options/CheckOutputFiles.py | 14 +++++ .../options/ExampleFunctionalFilterFile.py | 52 +++++++++++++++++++ .../components/ExampleFunctionalFilter.cpp | 52 +++++++++++++++++++ 6 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 k4FWCore/include/k4FWCore/FilterPredicate.h create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalFilterFile.py create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalFilter.cpp diff --git a/k4FWCore/include/k4FWCore/FilterPredicate.h b/k4FWCore/include/k4FWCore/FilterPredicate.h new file mode 100644 index 00000000..dd469f87 --- /dev/null +++ b/k4FWCore/include/k4FWCore/FilterPredicate.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Functional/FilterPredicate.h" +#include "k4FWCore/FunctionalUtils.h" + +namespace k4FWCore { + + template + using FilterPredicate = Gaudi::Functional::FilterPredicate; +} diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h index 36e81af5..003aad28 100644 --- a/k4FWCore/include/k4FWCore/FunctionalUtils.h +++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h @@ -26,6 +26,7 @@ #include "k4FWCore/DataWrapper.h" #include "podio/CollectionBase.h" +#include "GaudiKernel/DataObjectHandle.h" #include "GaudiKernel/EventContext.h" #include "GaudiKernel/ThreadLocalContext.h" @@ -42,14 +43,18 @@ namespace k4FWCore { namespace details { // This function will be used to modify std::shared_ptr to the actual collection type - template auto maybeTransformToEDM4hep(P& arg) { return arg; } - template requires std::same_as>> auto maybeTransformToEDM4hep(P& arg) { return arg; } + template + requires(!std::is_same_v>) + const auto& maybeTransformToEDM4hep(const P& arg) { + return arg; + } + template requires std::is_base_of_v const auto& maybeTransformToEDM4hep(P* arg) { @@ -216,6 +221,44 @@ namespace k4FWCore { return out; } + // Functional handles + template class FunctionalDataObjectReadHandle : public ::details::ReadHandle { + template + FunctionalDataObjectReadHandle(std::tuple&& args, std::index_sequence) + : FunctionalDataObjectReadHandle(std::get(std::move(args))...) {} + + public: + /// Autodeclaring constructor with property name, mode, key and documentation. + /// @note the use std::enable_if is required to avoid ambiguities + template >> + FunctionalDataObjectReadHandle(OWNER* owner, std::string propertyName, K key = {}, std::string doc = "") + : ::details::ReadHandle(owner, Gaudi::DataHandle::Reader, std::move(propertyName), std::move(key), + std::move(doc)) {} + + template + FunctionalDataObjectReadHandle(std::tuple&& args) + : FunctionalDataObjectReadHandle(std::move(args), std::index_sequence_for{}) {} + + const T& get() const; + }; + + template const T& FunctionalDataObjectReadHandle::get() const { + auto dataObj = this->fetch(); + if (!dataObj) { + throw GaudiException("Cannot retrieve \'" + this->objKey() + "\' from transient store.", + this->m_owner ? this->owner()->name() : "no owner", StatusCode::FAILURE); + } + auto ptr = dynamic_cast>*>(dataObj); + return maybeTransformToEDM4hep(ptr->getData()); + } + + struct BaseClass_t { + template using InputHandle = FunctionalDataObjectReadHandle; + // template using OutputHandle = DataObjectWriteHandle; + + using BaseClass = Gaudi::Algorithm; + }; + } // namespace details } // namespace k4FWCore diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index 33c6581a..f31e5ede 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -108,6 +108,9 @@ add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIE add_test_with_env(Testk4runHelpOnly options/TestArgs.py --help PROPERTIES PASS_REGULAR_EXPRESSION "show this help message and exit") +# WARNING: If test names are changed, they also need to be changed below for the dependencies of +# the FunctionalCheckFiles test + add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py) add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py) add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py) @@ -131,9 +134,10 @@ add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTra add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py) add_test_with_env(FunctionalTransformerHist options/ExampleFunctionalTransformerHist.py) add_test_with_env(FunctionalCollectionMerger options/ExampleFunctionalCollectionMerger.py) +add_test_with_env(FunctionalFilterFile options/ExampleFunctionalFilterFile.py) add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py) -set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc;FunctionalTransformerHist;FunctionalCollectionMerger") +set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc;FunctionalTransformerHist;FunctionalCollectionMerger;FunctionalFilterFile") # Do this after checking the files not to overwrite them add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py index 6968a4f6..2c6937db 100644 --- a/test/k4FWCoreTest/options/CheckOutputFiles.py +++ b/test/k4FWCoreTest/options/CheckOutputFiles.py @@ -45,6 +45,15 @@ def check_collections(filename, names): raise RuntimeError("Collections in frame do not match expected collections") +def check_events(filename, number): + print(f'Checking file "{filename}" for {number} events') + podio_reader = podio.root_io.Reader(filename) + frames = podio_reader.get("events") + if len(frames) != number: + print(f"File {filename} has {len(frames)} events but {number} are expected") + raise RuntimeError("Number of events does not match expected number") + + check_collections("functional_transformer.root", ["MCParticles", "NewMCParticles"]) check_collections( "functional_transformer_multiple.root", @@ -140,3 +149,8 @@ def check_collections(filename, names): ev = frames[0] if len(ev.get("NewMCParticles")) != 4: raise RuntimeError(f"Expected 4 NewMCParticles but got {len(ev.get('NewMCParticles'))}") + +check_events( + "functional_filter.root", + 5, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFilterFile.py b/test/k4FWCoreTest/options/ExampleFunctionalFilterFile.py new file mode 100644 index 00000000..e723cfb7 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalFilterFile.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an creating some collections and filtering after +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalTransformer, + ExampleFunctionalProducer, + ExampleFunctionalFilter, +) +from k4FWCore import ApplicationMgr, IOSvc +from Configurables import EventDataSvc + +iosvc = IOSvc("IOSvc") +iosvc.output = "functional_filter.root" + +transformer = ExampleFunctionalTransformer( + "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"] +) + +producer = ExampleFunctionalProducer("Producer", OutputCollection=["MCParticles"]) + +filt = ExampleFunctionalFilter( + "Filter", + InputCollection="NewMCParticles", +) + +ApplicationMgr( + TopAlg=[producer, transformer, filt], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalFilter.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalFilter.cpp new file mode 100644 index 00000000..eb16e6a5 --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalFilter.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Accumulators.h" +#include "edm4hep/MCParticleCollection.h" + +#include "k4FWCore/FilterPredicate.h" + +#include + +struct ExampleFunctionalFilter final : k4FWCore::FilterPredicate { + // The pair in KeyValue can be changed from python and it corresponds + // to the name of the input collection + ExampleFunctionalFilter(const std::string& name, ISvcLocator* svcLoc) + : FilterPredicate(name, svcLoc, KeyValue("InputCollection", {"MCParticles"})) {} + + // This is the function that will be called to check if the event passes + // Note that the function has to be const, as well as the collections + // we get from the input + bool operator()(const edm4hep::MCParticleCollection& input) const override { + ++m_counter; + // Only pass for half of the events + bool condition = m_counter.sum() % 2; + if (condition) { + info() << "Event passed" << endmsg; + } else { + info() << "Event did not pass" << endmsg; + } + return condition; + } + + // Thread-safe counter + mutable Gaudi::Accumulators::Counter<> m_counter{this, "Counter"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalFilter)