From 5dc2a6370304fb96f3731bffdf7c29f4c4734ae9 Mon Sep 17 00:00:00 2001 From: Juan Miguel Carceller <22276694+jmcarcell@users.noreply.github.com> Date: Tue, 21 May 2024 21:53:54 +0200 Subject: [PATCH] Support running multithreaded: Add an IOSvc, an algorithm for reading and writing for functionals in k4FWCore (#173) --- .github/workflows/test.yml | 2 +- CMakeLists.txt | 2 + k4FWCore/CMakeLists.txt | 6 + k4FWCore/components/IIOSvc.h | 57 ++++ k4FWCore/components/IOSvc.cpp | 159 +++++++++++ k4FWCore/components/IOSvc.h | 96 +++++++ k4FWCore/components/Reader.cpp | 151 +++++++++++ k4FWCore/components/Writer.cpp | 256 ++++++++++++++++++ k4FWCore/include/k4FWCore/BaseClass.h | 4 +- k4FWCore/include/k4FWCore/Consumer.h | 102 +++++++ k4FWCore/include/k4FWCore/DataHandle.h | 6 + k4FWCore/include/k4FWCore/FunctionalUtils.h | 213 +++++++++++++++ k4FWCore/include/k4FWCore/Producer.h | 48 ++++ k4FWCore/include/k4FWCore/Transformer.h | 227 ++++++++++++++++ k4FWCore/scripts/k4run | 3 +- python/CMakeLists.txt | 19 ++ python/k4FWCore/ApplicationMgr.py | 99 +++++++ python/k4FWCore/IOSvc.py | 42 +++ python/k4FWCore/__init__.py | 20 ++ test/k4FWCoreTest/CMakeLists.txt | 61 ++++- test/k4FWCoreTest/options/CheckOutputFiles.py | 120 ++++++++ ...pleFunctionalConsumerRuntimeCollections.py | 57 ++++ ...ionalConsumerRuntimeCollectionsMultiple.py | 83 ++++++ ...ransformer.py => ExampleFunctionalFile.py} | 33 +-- .../options/ExampleFunctionalFileMultiple.py | 44 +++ .../options/ExampleFunctionalMTFile.py | 65 +++++ .../options/ExampleFunctionalMTMemory.py | 68 +++++ ...Multiple.py => ExampleFunctionalMemory.py} | 36 ++- .../ExampleFunctionalMultipleMemory.py | 63 +++++ .../ExampleFunctionalOutputCommands.py | 49 ++++ .../options/ExampleFunctionalProducer.py | 53 ++++ ... ExampleFunctionalProducerAbsolutePath.py} | 18 +- ...y => ExampleFunctionalProducerMultiple.py} | 26 +- ...pleFunctionalProducerRuntimeCollections.py | 59 ++++ ... => ExampleFunctionalSeveralInputFiles.py} | 30 +- ...FunctionalTransformerRuntimeCollections.py | 82 ++++++ ...alTransformerRuntimeCollectionsMultiple.py | 139 ++++++++++ ...xampleFunctionalTransformerRuntimeEmpty.py | 63 +++++ .../options/runEventHeaderCheck.py | 17 +- ...runExampleFunctionalTransformerMultiple.py | 67 ----- .../options/runFunctionalChain.py | 60 ---- test/k4FWCoreTest/options/runFunctionalMix.py | 139 +++++++--- .../scripts/check_KeepDropSwitch.py | 5 +- .../components/ExampleEventHeaderConsumer.cpp | 7 +- .../components/ExampleFunctionalConsumer.cpp | 34 +-- .../ExampleFunctionalConsumerMultiple.cpp | 52 ++-- ...leFunctionalConsumerRuntimeCollections.cpp | 65 +++++ ...onalConsumerRuntimeCollectionsMultiple.cpp | 91 +++++++ .../components/ExampleFunctionalProducer.cpp | 8 +- .../ExampleFunctionalProducerMultiple.cpp | 46 ++-- ...leFunctionalProducerRuntimeCollections.cpp | 55 ++++ .../ExampleFunctionalTransformer.cpp | 33 ++- .../ExampleFunctionalTransformerMultiple.cpp | 38 ++- ...unctionalTransformerRuntimeCollections.cpp | 66 +++++ ...lTransformerRuntimeCollectionsMultiple.cpp | 195 +++++++++++++ ...ampleFunctionalTransformerRuntimeEmpty.cpp | 59 ++++ .../k4FWCoreTest_CheckExampleEventData.cpp | 13 +- .../k4FWCoreTest_CreateExampleEventData.cpp | 11 +- 58 files changed, 3313 insertions(+), 409 deletions(-) create mode 100644 k4FWCore/components/IIOSvc.h create mode 100644 k4FWCore/components/IOSvc.cpp create mode 100644 k4FWCore/components/IOSvc.h create mode 100644 k4FWCore/components/Reader.cpp create mode 100644 k4FWCore/components/Writer.cpp create mode 100644 k4FWCore/include/k4FWCore/Consumer.h create mode 100644 k4FWCore/include/k4FWCore/FunctionalUtils.h create mode 100644 k4FWCore/include/k4FWCore/Producer.h create mode 100644 k4FWCore/include/k4FWCore/Transformer.h create mode 100644 python/CMakeLists.txt create mode 100644 python/k4FWCore/ApplicationMgr.py create mode 100644 python/k4FWCore/IOSvc.py create mode 100644 python/k4FWCore/__init__.py create mode 100644 test/k4FWCoreTest/options/CheckOutputFiles.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py rename test/k4FWCoreTest/options/{runExampleFunctionalTransformer.py => ExampleFunctionalFile.py} (58%) create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalMTFile.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py rename test/k4FWCoreTest/options/{runExampleFunctionalConsumerMultiple.py => ExampleFunctionalMemory.py} (60%) create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalProducer.py rename test/k4FWCoreTest/options/{runExampleFunctionalProducer.py => ExampleFunctionalProducerAbsolutePath.py} (73%) rename test/k4FWCoreTest/options/{runExampleFunctionalProducerMultiple.py => ExampleFunctionalProducerMultiple.py} (65%) create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py rename test/k4FWCoreTest/options/{runExampleFunctionalConsumer.py => ExampleFunctionalSeveralInputFiles.py} (66%) create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py create mode 100644 test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py delete mode 100644 test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py delete mode 100644 test/k4FWCoreTest/options/runFunctionalChain.py create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp create mode 100644 test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a9c319c..d96db79d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: mkdir -p build install;\ source /cvmfs/${{ matrix.cvmfs_base }}/${{ matrix.ENVIRONMENT }}/setup.sh;\ cd build;\ - cmake -DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_STANDARD=17 -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always " -G Ninja ..;' + cmake -DCMAKE_INSTALL_PREFIX=../install -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_FLAGS=" -fdiagnostics-color=always " -G Ninja ..;' - name: Compile run: | docker exec CI_container /bin/bash -c 'cd ./Package;\ diff --git a/CMakeLists.txt b/CMakeLists.txt index c52ccdef..a0b68b67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,10 +73,12 @@ endif() add_subdirectory(k4FWCore) add_subdirectory(k4Interface) +add_subdirectory(python) if(BUILD_TESTING) add_subdirectory(test/k4FWCoreTest) endif() + option(ENABLE_CPACK "Whether or not to use cpack config" OFF) if(ENABLE_CPACK) include(cmake/${PROJECT_NAME}CPack.cmake) diff --git a/k4FWCore/CMakeLists.txt b/k4FWCore/CMakeLists.txt index d28602bd..6a33b8f6 100644 --- a/k4FWCore/CMakeLists.txt +++ b/k4FWCore/CMakeLists.txt @@ -57,3 +57,9 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/k4FWCore/python/k4FWCore ${CMAKE_CURRENT_BINARY_DIR}/genConfDir/k4FWCore) + +# This is needed to overwrite the __init__.py, see a long comment in the +# CMakeLists.txt in the test folder +add_custom_command(TARGET k4FWCorePlugins POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${PROJECT_SOURCE_DIR}/python/k4FWCore/__init__.py ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore/__init__.py) diff --git a/k4FWCore/components/IIOSvc.h b/k4FWCore/components/IIOSvc.h new file mode 100644 index 00000000..d5721e0a --- /dev/null +++ b/k4FWCore/components/IIOSvc.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_IIOSVC_H +#define FWCORE_IIOSVC_H + +#include "GaudiKernel/IInterface.h" + +#include "podio/CollectionBase.h" +#include "podio/ROOTWriter.h" + +#include +#include + +/** + * The interface implemented by any class making IO and reading RawEvent Data + */ +class IIOSvc : virtual public IInterface { +public: + struct EndOfInput : std::logic_error { + EndOfInput() : logic_error("Reached end of input while more data were expected"){}; + }; + +public: + /// InterfaceID + DeclareInterfaceID(IIOSvc, 1, 0); + + /** + * @brief Read the next event from the input file + * @return A tuple containing the collections read, the collection names and the frame that owns the collections + */ + virtual std::tuple>, std::vector, podio::Frame> + next() = 0; + virtual std::shared_ptr> getCollectionNames() const = 0; + + virtual std::shared_ptr getWriter() = 0; + virtual void deleteWriter() = 0; + virtual void deleteReader() = 0; + virtual bool checkIfWriteCollection(const std::string& collName) = 0; +}; + +#endif diff --git a/k4FWCore/components/IOSvc.cpp b/k4FWCore/components/IOSvc.cpp new file mode 100644 index 00000000..b88858bd --- /dev/null +++ b/k4FWCore/components/IOSvc.cpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "IOSvc.h" + +#include "podio/Frame.h" +#include "podio/FrameCategories.h" + +#include "k4FWCore/FunctionalUtils.h" +#include "k4FWCore/KeepDropSwitch.h" + +#include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/IEventProcessor.h" + +#include +#include + +StatusCode IOSvc::initialize() { + StatusCode sc = Service::initialize(); + if (sc.isFailure()) { + error() << "Unable to initialize base class Service." << endmsg; + return sc; + } + if (!m_readingFileNames.empty()) { + m_reader = std::make_unique(); + try { + m_reader->openFiles(m_readingFileNames); + } catch (std::runtime_error& e) { + error() << "Error when opening files: " << e.what() << endmsg; + return StatusCode::FAILURE; + } + m_entries = m_reader->getEntries(podio::Category::Event); + } + + m_switch = KeepDropSwitch(m_outputCommands); + + m_incidentSvc = service("IncidentSvc"); + if (!m_incidentSvc) { + error() << "Unable to locate IIncidentSvc interface" << endmsg; + return StatusCode::FAILURE; + } + m_incidentSvc->addListener(this, IncidentType::EndEvent); + + m_dataSvc = service("EventDataSvc"); + if (!m_dataSvc) { + error() << "Unable to locate the EventDataSvc" << endmsg; + return StatusCode::FAILURE; + } + + m_hiveWhiteBoard = service("EventDataSvc"); + + return StatusCode::SUCCESS; +} + +StatusCode IOSvc::finalize() { return Service::finalize(); } + +std::tuple>, std::vector, podio::Frame> IOSvc::next() { + podio::Frame frame; + { + std::scoped_lock lock(m_changeBufferLock); + info() << "m_nextEntry = " << m_nextEntry << " m_entries = " << m_entries << endmsg; + if (m_nextEntry < m_entries) { + frame = podio::Frame(std::move(m_reader->readEntry(podio::Category::Event, m_nextEntry))); + } else { + return std::make_tuple(std::vector>(), std::vector(), + std::move(frame)); + } + m_nextEntry++; + if (m_collectionNames.empty()) { + m_collectionNames = frame.getAvailableCollections(); + } + } + + if (m_nextEntry >= m_entries) { + // if (true) { + auto ep = serviceLocator()->as(); + StatusCode sc = ep->stopRun(); + if (sc.isFailure()) { + error() << "Error when stopping run" << endmsg; + throw GaudiException("Error when stopping run", name(), StatusCode::FAILURE); + } + info() << "m_nextEntry = " << m_nextEntry << " m_entries = " << m_entries << endmsg; + } + + std::vector> collections; + + for (const auto& name : m_collectionNames) { + auto ptr = const_cast(frame.get(name)); + collections.push_back(std::shared_ptr(ptr)); + } + + return std::make_tuple(collections, m_collectionNames, std::move(frame)); +} + +// After every event if there is still a frame in the TES +// that means it hasn't been written so the collections inside the Frame +// should be removed so that they are deleted when the Frame is deleted +// and not deleted when clearing the store +void IOSvc::handle(const Incident& incident) { + StatusCode code; + if (m_hiveWhiteBoard) { + if (!incident.context().valid()) { + info() << "No context found in IOSvc" << endmsg; + return; + } + debug() << "Setting store to " << incident.context().slot() << endmsg; + code = m_hiveWhiteBoard->selectStore(incident.context().slot()); + if (code.isFailure()) { + error() << "Error when setting store" << endmsg; + throw GaudiException("Error when setting store", name(), StatusCode::FAILURE); + } + } + DataObject* p; + code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p); + if (code.isFailure()) { + return; + } + + auto frame = dynamic_cast*>(p); + if (!frame) { + error() << "Expected Frame in " << k4FWCore::frameLocation << " but there was something else" << endmsg; + return; + } + for (const auto& coll : frame->getData().getAvailableCollections()) { + DataObject* collPtr; + code = m_dataSvc->retrieveObject("/Event/" + coll, collPtr); + if (code.isSuccess()) { + debug() << "Removing the collection: " << coll << " from the store" << endmsg; + code = m_dataSvc->unregisterObject(collPtr); + } + // else { + // info() << "Collection not found: " << coll << endmsg; + // } + } +} + +void IOSvc::setReadingCollectionNames(const std::vector& names) { m_collectionNames = names; } + +void IOSvc::setReadingFileNames(const std::vector& names) { m_readingFileNames = names; } + +bool IOSvc::checkIfWriteCollection(const std::string& collName) { return m_switch.isOn(collName); } + +DECLARE_COMPONENT(IOSvc) diff --git a/k4FWCore/components/IOSvc.h b/k4FWCore/components/IOSvc.h new file mode 100644 index 00000000..b77025d1 --- /dev/null +++ b/k4FWCore/components/IOSvc.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_IOSVC_H +#define FWCORE_IOSVC_H + +#include "Gaudi/Property.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include "GaudiKernel/IHiveWhiteBoard.h" +#include "GaudiKernel/IIncidentListener.h" +#include "GaudiKernel/IIncidentSvc.h" +#include "GaudiKernel/Service.h" + +#include "podio/ROOTReader.h" +#include "podio/ROOTWriter.h" + +#include "IIOSvc.h" +#include "k4FWCore/KeepDropSwitch.h" + +#include +#include + +class IOSvc : public extends { + using extends::extends; + +public: + ~IOSvc() override = default; + + StatusCode initialize() override; + StatusCode finalize() override; + + std::tuple>, std::vector, podio::Frame> next() + override; + + std::shared_ptr> getCollectionNames() const override { + return std::make_shared>(m_collectionNames); + } + + void setReadingCollectionNames(const std::vector& names); + void setReadingFileNames(const std::vector& names); + +protected: + Gaudi::Property> m_collectionNames{ + this, "CollectionNames", {}, "List of collections to read"}; + Gaudi::Property> m_readingFileNames{this, "input", {}, "List of files to read"}; + Gaudi::Property m_writingFileName{this, "output", {}, "List of files to write output to"}; + Gaudi::Property> m_outputCommands{ + this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."}; + Gaudi::Property m_inputType{this, "ioType", "ROOT", "Type of input file (ROOT, RNTuple)"}; + + std::mutex m_changeBufferLock; + + KeepDropSwitch m_switch; + + std::unique_ptr m_reader{nullptr}; + std::shared_ptr m_writer{nullptr}; + + std::shared_ptr getWriter() override { + if (!m_writer) { + m_writer = std::make_shared(m_writingFileName.value()); + } + return m_writer; + } + + // Gaudi doesn't always run the destructor of the Services so we have to + // manually ask for the writer to be deleted so it will call finish() + void deleteWriter() override { m_writer.reset(); } + void deleteReader() override { m_reader.reset(); } + + SmartIF m_dataSvc; + SmartIF m_incidentSvc; + SmartIF m_hiveWhiteBoard; + void handle(const Incident& incident) override; + + int m_entries{0}; + int m_nextEntry{0}; + + bool checkIfWriteCollection(const std::string& collName) override; +}; + +#endif diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp new file mode 100644 index 00000000..c05c5ce2 --- /dev/null +++ b/k4FWCore/components/Reader.cpp @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "Gaudi/Functional/details.h" +#include "Gaudi/Functional/utilities.h" +#include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/FunctionalFilterDecision.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include "GaudiKernel/StatusCode.h" + +#include "podio/CollectionBase.h" +#include "podio/Frame.h" +#include "podio/ROOTReader.h" + +#include "IIOSvc.h" +#include "k4FWCore/FunctionalUtils.h" + +#include + +template using vector_of_ = std::vector; + +class CollectionPusher : public Gaudi::Functional::details::BaseClass_t { + using Traits_ = Gaudi::Functional::Traits::useDefaults; + using Out = std::shared_ptr; + using base_class = Gaudi::Functional::details::BaseClass_t; + static_assert(std::is_base_of_v, "BaseClass must inherit from Algorithm"); + + template + using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t>; + std::vector>> m_outputs; + Gaudi::Property> m_inputCollections{ + this, "InputCollections", {"First collection"}, "List of input collections"}; + // Gaudi::Property m_input{this, "Input", "Event", "Input file"}; + +public: + CollectionPusher(std::string name, ISvcLocator* locator) + : base_class(std::move(name), locator), + m_inputCollections{this, + "InputCollections", + {"Event"}, + [this](Gaudi::Details::PropertyBase& b) { + const std::string cmd = System::cmdLineArgs()[0]; + if (cmd.find("genconf") != std::string::npos) { + return; + } + if (m_inputCollections.value().size() == 1 && m_inputCollections.value()[0] == "Event") { + return; + } + + for (auto& c : m_inputCollections.value()) { + m_outputs.emplace_back(c, this); + } + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}} {} + + // derived classes can NOT implement execute + StatusCode execute(const EventContext&) const override final { + try { + auto out = (*this)(); + + auto outColls = std::get>>(out); + auto outputLocations = std::get>(out); + + // if (out.size() != m_outputs.size()) { + // throw GaudiException("Error during transform: expected " + std::to_string(m_outputs.size()) + + // " containers, got " + std::to_string(out.size()) + " instead", + // this->name(), StatusCode::FAILURE); + // } + for (size_t i = 0; i != outColls.size(); ++i) { + m_outputs[i].put(std::move(outColls[i])); + } + return Gaudi::Functional::FilterDecision::PASSED; + } catch (GaudiException& e) { + (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg; + return e.code(); + } + } + + virtual std::tuple, std::vector> operator()() const = 0; + +private: + ServiceHandle m_dataSvc{this, "EventDataSvc", "EventDataSvc"}; +}; + +class Reader final : public CollectionPusher { +public: + Reader(const std::string& name, ISvcLocator* svcLoc) : CollectionPusher(name, svcLoc) { + setProperty("Cardinality", 1).ignore(); + } + + bool isReEntrant() const override { return false; } + + // Gaudi doesn't run the destructor of the Services so we have to + // manually ask for the reader to be deleted so it will call finish() + // See https://gitlab.cern.ch/gaudi/Gaudi/-/issues/169 + ~Reader() { iosvc->deleteReader(); } + + ServiceHandle iosvc{this, "IOSvc", "IOSvc"}; + + StatusCode initialize() override { + if (!iosvc.isValid()) { + error() << "Unable to locate IIOSvc interface" << endmsg; + return StatusCode::FAILURE; + } + + return StatusCode::SUCCESS; + } + + StatusCode finalize() override { + if (iosvc) { + iosvc->deleteReader(); + } + return StatusCode::SUCCESS; + } + + // The IOSvc takes care of reading and passing the data + // By convention the Frame is pushed to the store + // so that it's deleted at the right time + std::tuple>, std::vector> operator()() + const override { + auto val = iosvc->next(); + + auto eds = eventSvc().as(); + auto frame = std::move(std::get(val)); + + auto tmp = new AnyDataWrapper(std::move(frame)); + if (eds->registerObject("/Event" + k4FWCore::frameLocation, tmp).isFailure()) { + error() << "Failed to register Frame object" << endmsg; + } + + return std::make_tuple(std::get<0>(val), std::get<1>(val)); + } +}; + +DECLARE_COMPONENT(Reader) diff --git a/k4FWCore/components/Writer.cpp b/k4FWCore/components/Writer.cpp new file mode 100644 index 00000000..38ae00e5 --- /dev/null +++ b/k4FWCore/components/Writer.cpp @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Functional/Consumer.h" +#include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/IDataManagerSvc.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include "GaudiKernel/SmartDataPtr.h" +#include "GaudiKernel/StatusCode.h" + +#include "podio/Frame.h" + +#include "IIOSvc.h" + +#include "k4FWCore/DataWrapper.h" +#include "k4FWCore/FunctionalUtils.h" + +#include +#include + +class Writer final : public Gaudi::Functional::Consumer { +public: + Writer(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc) { + // Non-reentrant algorithms have a cardinality of 1 + setProperty("Cardinality", 1).ignore(); + } + + bool isReEntrant() const override { return false; } + + // Many members are mutable because it's assumed that the Writer is called only once + mutable std::set m_availableCollections; + // These are the collections that are not in the frame and we want to add to the frame + mutable std::vector m_collectionsToAdd; + // These are the collections we want to save to the frame + mutable std::vector m_collectionsToSave; + + ServiceHandle iosvc{this, "IOSvc", "IOSvc"}; + SmartIF m_hiveWhiteBoard; + SmartIF m_dataSvc; + mutable bool m_first{true}; + + StatusCode initialize() override { + if (!iosvc.isValid()) { + error() << "Unable to locate IIOSvc interface" << endmsg; + return StatusCode::FAILURE; + } + + m_dataSvc = service("EventDataSvc", true); + if (!m_dataSvc) { + error() << "Unable to locate EventDataSvc service" << endmsg; + return StatusCode::FAILURE; + } + + m_hiveWhiteBoard = service("EventDataSvc", true); + if (!m_hiveWhiteBoard) { + debug() << "Unable to locate IHiveWhiteBoard interface. This isn't a problem if we are not running in a " + "multi-threaded environment" + << endmsg; + } + + return StatusCode::SUCCESS; + } + + StatusCode finalize() override { + podio::Frame config_metadata_frame; + + //// prepare job options metadata /////////////////////// + // retrieve the configuration of the job + // and write it to file as vector of strings + std::vector config_data; + for (const auto& per_property : Gaudi::svcLocator()->getOptsSvc().items()) { + std::stringstream config_stream; + // sample output: + // HepMCToEDMConverter.genparticles = "GenParticles"; + // Note that quotes are added to all property values, + // which leads to problems with ints, lists, dicts and bools. + // For theses types, the quotes must be removed in postprocessing. + config_stream << std::get<0>(per_property) << " = \"" << std::get<1>(per_property) << "\";" << std::endl; + config_data.push_back(config_stream.str()); + } + // Some default components are not captured by the job option service + // and have to be traversed like this. Note that Gaudi!577 will improve this. + for (const auto* name : {"NTupleSvc"}) { + std::stringstream config_stream; + auto svc = service(name); + if (!svc.isValid()) + continue; + for (const auto* property : svc->getProperties()) { + config_stream << name << "." << property->name() << " = \"" << property->toString() << "\";" << std::endl; + } + config_data.push_back(config_stream.str()); + } + + config_metadata_frame.putParameter("gaudiConfigOptions", config_data); + if (auto env_key4hep_stack = std::getenv("KEY4HEP_STACK")) { + config_metadata_frame.putParameter("key4hepstack", env_key4hep_stack); + } + iosvc->getWriter()->writeFrame(config_metadata_frame, "configuration_metadata"); + + iosvc->deleteWriter(); + return StatusCode::SUCCESS; + } + + void getOutputCollections() const { + SmartIF m_mgr; + m_mgr = eventSvc(); + + SmartDataPtr root(eventSvc(), "/Event"); + if (!root) { + error() << "Failed to retrieve root object /Event" << endmsg; + return; + } + + auto pObj = root->registry(); + if (!pObj) { + error() << "Failed to retrieve the root registry object" << endmsg; + return; + } + auto mgr = eventSvc().as(); + if (!mgr) { + error() << "Failed to retrieve IDataManagerSvc" << endmsg; + return; + } + std::vector leaves; + StatusCode sc = m_mgr->objectLeaves(pObj, leaves); + if (!sc.isSuccess()) { + error() << "Failed to retrieve object leaves" << endmsg; + return; + } + for (auto& pReg : leaves) { + if (pReg->name() == k4FWCore::frameLocation) { + continue; + } + // info() << "Found leaf: " << pReg->name() << endmsg; + /// We are only interested in leaves with an object + // if (!pReg->address() || !pReg->object()) { + // info() << "Leaf " << pReg->name() << " has no object" << endmsg; + // continue; + // } + m_availableCollections.insert(pReg->name().substr(1, pReg->name().size() - 1)); + const std::string& id = pReg->identifier(); + } + } + + void operator()(const EventContext& ctx) const override { + if (m_hiveWhiteBoard) { + // It's never set to valid but it has the slot information + // if (ctx.valid()) { + // info() << "No context found in Writer" << endmsg; + // return; + // } + debug() << "Setting store to " << ctx.slot() << endmsg; + if (m_hiveWhiteBoard->selectStore(ctx.slot()).isFailure()) { + error() << "Error when setting store" << endmsg; + throw GaudiException("Error when setting store", name(), StatusCode::FAILURE); + } + } + + DataObject* p; + StatusCode code = m_dataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p); + AnyDataWrapper* ptr; + // This is the case when we are reading from a file + if (code.isSuccess()) { + m_dataSvc->unregisterObject(p).ignore(); + ptr = dynamic_cast*>(p); + } + // This is the case when no reading is being done + // Will be deleted by the store + else { + ptr = new AnyDataWrapper(podio::Frame()); + } + + const auto& frameCollections = ptr->getData().getAvailableCollections(); + if (m_first) { + // Assume all the output collections are the same for all events + // and cache them + getOutputCollections(); + for (const auto& coll : m_availableCollections) { + if (iosvc->checkIfWriteCollection(coll)) { + m_collectionsToSave.push_back(coll); + if (std::find(frameCollections.begin(), frameCollections.end(), coll) == frameCollections.end()) { + m_collectionsToAdd.push_back(coll); + } + } + } + m_first = false; + } + + // Remove the collections owned by a Frame (if any) so that they are not + // deleted by the store (and later deleted by the Frame, triggering a double + // delete) + for (auto& coll : frameCollections) { + DataObject* storeCollection; + if (m_dataSvc->retrieveObject("/Event/" + coll, storeCollection).isFailure()) { + error() << "Failed to retrieve collection " << coll << endmsg; + return; + } + // We take ownership back from the store + if (m_dataSvc->unregisterObject(storeCollection).isFailure()) { + error() << "Failed to unregister collection " << coll << endmsg; + return; + } + } + + for (auto& coll : m_collectionsToAdd) { + DataObject* storeCollection; + if (m_dataSvc->retrieveObject("/Event/" + coll, storeCollection).isFailure()) { + error() << "Failed to retrieve collection " << coll << endmsg; + return; + } + // We take ownership back from the store + if (m_dataSvc->unregisterObject(storeCollection).isFailure()) { + error() << "Failed to unregister collection " << coll << endmsg; + return; + } + const auto collection = dynamic_cast>*>(storeCollection); + if (!collection) { + // Check the case when the data has been produced using the old DataHandle + const auto old_collection = dynamic_cast(storeCollection); + if (!old_collection) { + error() << "Failed to cast collection " << coll << endmsg; + return; + } else { + std::unique_ptr uptr( + const_cast(old_collection->collectionBase())); + ptr->getData().put(std::move(uptr), coll); + } + + } else { + std::unique_ptr uptr(collection->getData().get()); + ptr->getData().put(std::move(uptr), coll); + } + } + + debug() << "Writing frame" << endmsg; + iosvc->getWriter()->writeFrame(ptr->getData(), podio::Category::Event, m_collectionsToSave); + } +}; + +DECLARE_COMPONENT(Writer) diff --git a/k4FWCore/include/k4FWCore/BaseClass.h b/k4FWCore/include/k4FWCore/BaseClass.h index 4d58f8d3..3d679251 100644 --- a/k4FWCore/include/k4FWCore/BaseClass.h +++ b/k4FWCore/include/k4FWCore/BaseClass.h @@ -26,7 +26,9 @@ // Base class used for the Traits template argument of the // Gaudi::Functional algorithms -struct BaseClass_t { +struct [[deprecated( + "Functional algorithms using the BaseClass.h header are deprecated and will be removed in the " + "future")]] BaseClass_t { template using InputHandle = DataObjectReadHandle>; template using OutputHandle = DataObjectWriteHandle>; diff --git a/k4FWCore/include/k4FWCore/Consumer.h b/k4FWCore/include/k4FWCore/Consumer.h new file mode 100644 index 00000000..5a9911bf --- /dev/null +++ b/k4FWCore/include/k4FWCore/Consumer.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_CONSUMER_H +#define FWCORE_CONSUMER_H + +#include +#include "Gaudi/Functional/details.h" +#include "Gaudi/Functional/utilities.h" + +// #include "GaudiKernel/CommonMessaging.h" + +#include "k4FWCore/FunctionalUtils.h" + +#include +#include + +namespace k4FWCore { + + namespace details { + + template struct Consumer; + + template + struct Consumer + : Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_> { + using Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_>::DataHandleMixin; + + static_assert(((std::is_base_of_v || isMapToCollLike::value) && ...), + "Consumer input types must be EDM4hep collections or maps to collections"); + + template + using InputHandle_t = Gaudi::Functional::details::InputHandle_t>; + + std::tuple::type>>...> m_inputs; + std::array>, sizeof...(In)> m_inputLocations{}; + + using base_class = Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_>; + + using KeyValues = typename base_class::KeyValues; + + template + Consumer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence) + : base_class(std::move(name), locator), + // The input locations are filled by creating a property with a callback function + // that creates the handles because that is when the input locations become available + // (from a steering file, for example) and the handles have to be created for + // Gaudi to know the data flow + m_inputLocations{Gaudi::Property>{ + this, std::get(inputs).first, to_DataObjID(std::get(inputs).second), + [this](Gaudi::Details::PropertyBase&) { + std::vector::type>> handles; + for (auto& value : this->m_inputLocations[I].value()) { + auto handle = InputHandle_t::type>(value, this); + handles.push_back(std::move(handle)); + } + std::get(m_inputs) = std::move(handles); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {} + + Consumer(std::string name, ISvcLocator* locator, + Gaudi::Functional::details::RepeatValues_ const& inputs) + : Consumer(std::move(name), locator, inputs, std::index_sequence_for{}) {} + + // derived classes are NOT allowed to implement execute ... + StatusCode execute(const EventContext& ctx) const override final { + try { + filter_evtcontext_tt::apply(*this, ctx, m_inputs); + return Gaudi::Functional::FilterDecision::PASSED; + } catch (GaudiException& e) { + (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg; + return e.code(); + } + } + + // ... instead, they must implement the following operator + virtual void operator()(const In&...) const = 0; + }; + + } // namespace details + + template + using Consumer = details::Consumer; + +} // namespace k4FWCore + +#endif // FWCORE_CONSUMER_H diff --git a/k4FWCore/include/k4FWCore/DataHandle.h b/k4FWCore/include/k4FWCore/DataHandle.h index bea0ff5a..37ada6f1 100644 --- a/k4FWCore/include/k4FWCore/DataHandle.h +++ b/k4FWCore/include/k4FWCore/DataHandle.h @@ -29,6 +29,7 @@ #include "TTree.h" +#include #include /** @@ -131,6 +132,11 @@ template const T* DataHandle::get() { DataWrapper* tmp = static_cast*>(dataObjectp); return reinterpret_cast(tmp->collectionBase()); } else { + // When a functional has pushed a std::shared_ptr into the store + auto ptr = static_cast>*>(dataObjectp)->getData(); + if (ptr) { + return reinterpret_cast(ptr.get()); + } std::string errorMsg("The type provided for " + DataObjectHandle>::pythonRepr() + " is different from the one of the object in the store."); throw GaudiException(errorMsg, "wrong product type", StatusCode::FAILURE); diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h new file mode 100644 index 00000000..61aab1aa --- /dev/null +++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_FUNCTIONALUTILS_H +#define FWCORE_FUNCTIONALUTILS_H + +#include +#include +#include "Gaudi/Functional/details.h" +#include "GaudiKernel/DataObjID.h" +#include "k4FWCore/DataWrapper.h" +#include "podio/CollectionBase.h" + +#include "GaudiKernel/EventContext.h" +#include "GaudiKernel/ThreadLocalContext.h" + +// #include "GaudiKernel/CommonMessaging.h" + +#include +#include +#include +#include + +namespace k4FWCore { + + static const std::string frameLocation = "/_Frame"; + + namespace details { + + // This function will be used to modify std::shared_ptr to the actual collection type + template const auto& maybeTransformToEDM4hep(const P& arg) { return arg; } + template + requires std::is_base_of_v + const auto& maybeTransformToEDM4hep(P* arg) { + return *arg; + } + + template + requires std::same_as> + const auto& maybeTransformToEDM4hep(P arg) { + return static_cast(*arg); + } + + template > + using addPtrIfColl = std::conditional_t, T>; + + template const auto& transformtoEDM4hep(const std::shared_ptr& arg) { + return static_cast(*arg); + } + + // Check if the type is a map like type, where map type is the special map + // type to have an arbitrary number of collections as input or output: + // std::map where Coll is the collection type + template struct isMapToCollLike : std::false_type {}; + + template + requires std::is_base_of_v> + struct isMapToCollLike> : std::true_type {}; + + template inline constexpr bool isMapToCollLike_v = isMapToCollLike::value; + + // transformType function to transform the types from the ones that the user wants + // like edm4hep::MCParticleCollection, to the ones that are actually stored in the + // event store, like std::shared_ptr + // For std::map, th + template struct transformType { + using type = T; + }; + + template + requires std::is_base_of_v || isMapToCollLike_v + struct transformType { + using type = std::shared_ptr; + }; + + template auto convertToSharedPtr(T&& arg) { + return std::shared_ptr(std::make_shared(std::move(arg))); + } + + template struct filter_evtcontext_tt { + static_assert(!std::disjunction_v...>, + "EventContext can only appear as first argument"); + + template static auto apply(const Algorithm& algo, Handles& handles) { + return std::apply( + [&](const auto&... handle) { return algo(get(handle, algo, Gaudi::Hive::currentContext())...); }, handles); + } + + template + static auto apply(const Algorithm& algo, const EventContext&, Handles& handles) { + auto inputTuple = std::tuple...>(); + + // Build the input tuple by picking up either std::map with an arbitrary + // number of collections or single collections + readMapInputs<0, In...>(handles, &algo, inputTuple); + + return std::apply( + [&](const auto&... input) { return algo(maybeTransformToEDM4hep(input)...); }, inputTuple); + } + }; + + template + void readMapInputs(const std::tuple& handles, auto thisClass, InputTuple& inputTuple) { + if constexpr (Index < sizeof...(Handles)) { + if constexpr (isMapToCollLike_v>>) { + // In case of map types like std::map + // we have to remove the reference to get the actual type + using EDM4hepType = + std::remove_reference_t>::mapped_type>; + auto inputMap = std::map(); + for (auto& handle : std::get(handles)) { + auto in = get(handle, thisClass, Gaudi::Hive::currentContext()); + inputMap.emplace(handle.objKey(), *static_cast(in.get())); + } + std::get(inputTuple) = std::move(inputMap); + + } else { + try { + auto in = get(std::get(handles)[0], thisClass, Gaudi::Hive::currentContext()); + std::get(inputTuple) = static_cast>*>(in.get()); + } catch (GaudiException& e) { + // When the type of the collection is different from the one requested, this can happen because + // 1. a mistake was made in the input types of a functional algorithm + // 2. the data was produced using the old DataHandle, which is never going to be in the input type + if (e.message().find("different from") != std::string::npos) { + thisClass->error() << "Trying to cast the collection " << std::get(handles)[0].objKey() + << " to the requested type " << thisClass->name() << endmsg; + DataObject* p; + IDataProviderSvc* svc = thisClass->serviceLocator()->template service("EventDataSvc"); + svc->retrieveObject("/Event/" + std::get(handles)[0].objKey(), p).ignore(); + const auto wrp = dynamic_cast>>*>(p); + if (!wrp) { + throw GaudiException(thisClass->name(), + "Failed to cast collection " + std::get(handles)[0].objKey(), + StatusCode::FAILURE); + } + std::get(inputTuple) = const_cast>*>(wrp->getData()); + } else { + throw e; + } + } + } + + // Recursive call for the next index + readMapInputs(handles, thisClass, inputTuple); + } + } + + template + void putMapOutputs(std::tuple&& handles, const auto& m_outputs, auto thisClass) { + if constexpr (Index < sizeof...(Handles)) { + if constexpr (isMapToCollLike_v>>) { + int i = 0; + if (std::get(handles).size() != std::get(m_outputs).size()) { + std::string msg = "Size of the output map " + std::to_string(std::get(handles).size()) + + " does not match the expected size from the steering file " + + std::to_string(std::get(m_outputs).size()) + ". Expected the collections: "; + for (auto& out : std::get(m_outputs)) { + msg += out.objKey() + " "; + } + msg += " but got the collections: "; + for (auto& out : std::get(handles)) { + msg += out.first + " "; + } + throw GaudiException(thisClass->name(), msg, StatusCode::FAILURE); + } + for (auto& [key, val] : std::get(handles)) { + if (key != std::get(m_outputs)[i].objKey()) { + throw GaudiException(thisClass->name(), + "Output key in the map \"" + key + + "\" does not match the expected key from the steering file \"" + + std::get(m_outputs)[i].objKey() + "\"", + StatusCode::FAILURE); + } + Gaudi::Functional::details::put(std::get(m_outputs)[i], convertToSharedPtr(std::move(val))); + i++; + } + } else { + Gaudi::Functional::details::put(std::get(m_outputs)[0], + convertToSharedPtr(std::move(std::get(handles)))); + } + + // Recursive call for the next index + putMapOutputs(std::move(handles), m_outputs, thisClass); + } + } + + inline std::vector to_DataObjID(const std::vector& in) { + std::vector out; + out.reserve(in.size()); + std::transform(in.begin(), in.end(), std::back_inserter(out), [](const std::string& i) { return DataObjID{i}; }); + return out; + } + + } // namespace details +} // namespace k4FWCore + +#endif // CORE_FUNCTIONALUTILS_H diff --git a/k4FWCore/include/k4FWCore/Producer.h b/k4FWCore/include/k4FWCore/Producer.h new file mode 100644 index 00000000..a5e404ef --- /dev/null +++ b/k4FWCore/include/k4FWCore/Producer.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_PRODUCER_H +#define FWCORE_PRODUCER_H + +#include "k4FWCore/Transformer.h" + +#include + +namespace k4FWCore { + + namespace details { + + template struct Producer; + + template + struct Producer(), Traits_> : MultiTransformer(), Traits_> { + using MultiTransformer(), Traits_>::MultiTransformer; + }; + + template struct Producer : Transformer { + using Transformer::Transformer; + }; + + } // namespace details + + template + using Producer = details::Producer; + +} // namespace k4FWCore + +#endif // FWCORE_PRODUCER_H diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h new file mode 100644 index 00000000..f5030ddd --- /dev/null +++ b/k4FWCore/include/k4FWCore/Transformer.h @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_TRANSFORMER_H +#define FWCORE_TRANSFORMER_H + +#include +#include "Gaudi/Functional/details.h" +#include "Gaudi/Functional/utilities.h" + +#include "k4FWCore/FunctionalUtils.h" + +// #include "GaudiKernel/CommonMessaging.h" + +#include +#include + +namespace k4FWCore { + + namespace details { + + template struct Transformer; + + template + struct Transformer + : Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_> { + using Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_>::DataHandleMixin; + + static_assert(((std::is_base_of_v || isMapToCollLike::value) && ...), + "Transformer and Producer input types must be EDM4hep collections or maps to collections"); + static_assert((std::is_base_of_v || isMapToCollLike::value), + "Transformer and Producer output types must be EDM4hep collections or maps to collections"); + + template + using InputHandle_t = Gaudi::Functional::details::InputHandle_t>; + template + using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t>; + + std::tuple::type>>...> m_inputs; + std::tuple::type>>> m_outputs; + std::array>, sizeof...(In)> m_inputLocations{}; + std::array>, 1> m_outputLocations{}; + + using base_class = Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_>; + + using KeyValues = typename base_class::KeyValues; + + template + Transformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence, + const OArgs& outputs, std::index_sequence) + : base_class(std::move(name), locator), + // The input locations are filled by creating a property with a + // callback function that creates the handles because when the + // callback runs is when the input locations become available (from + // a steering file, for example) and the handles have to be created + // for Gaudi to know the data flow + m_inputLocations{Gaudi::Property>{ + this, std::get(inputs).first, to_DataObjID(std::get(inputs).second), + [this](Gaudi::Details::PropertyBase&) { + std::vector::type>> h; + for (auto& value : this->m_inputLocations[I].value()) { + auto handle = InputHandle_t::type>(value, this); + h.push_back(std::move(handle)); + } + std::get(m_inputs) = std::move(h); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...}, + // Same as above for the output locations + m_outputLocations{Gaudi::Property>{ + this, std::get(outputs).first, to_DataObjID(std::get(outputs).second), + [this](Gaudi::Details::PropertyBase&) { + std::vector::type>> h; + // Is this needed? + // std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(), + // [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); }); + for (auto& inpID : this->m_outputLocations[J].value()) { + if (inpID.key().empty()) { + continue; + } + auto handle = OutputHandle_t::type>(inpID, this); + h.push_back(std::move(handle)); + } + std::get<0>(m_outputs) = std::move(h); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {} + + constexpr static std::size_t N_in = sizeof...(In); + constexpr static std::size_t N_out = 1; + + Transformer(std::string name, ISvcLocator* locator, + Gaudi::Functional::details::RepeatValues_ const& inputs, + Gaudi::Functional::details::RepeatValues_ const& outputs) + : Transformer(std::move(name), locator, inputs, std::index_sequence_for{}, outputs, + std::index_sequence_for{}) {} + + // derived classes are NOT allowed to implement execute ... + StatusCode execute(const EventContext& ctx) const override final { + try { + if constexpr (isMapToCollLike::value) { + std::tuple tmp = filter_evtcontext_tt::apply(*this, ctx, this->m_inputs); + putMapOutputs<0, Out>(std::move(tmp), m_outputs, this); + } else { + Gaudi::Functional::details::put( + std::get<0>(this->m_outputs)[0], + convertToSharedPtr(std::move(filter_evtcontext_tt::apply(*this, ctx, this->m_inputs)))); + } + return Gaudi::Functional::FilterDecision::PASSED; + } catch (GaudiException& e) { + (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg; + return e.code(); + } + } + + // ... instead, they must implement the following operator + virtual Out operator()(const In&...) const = 0; + }; + + template struct MultiTransformer; + + template + struct MultiTransformer(const In&...), Traits_> + : Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_> { + using Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_>::DataHandleMixin; + + static_assert(((std::is_base_of_v || isMapToCollLike::value) && ...), + "Transformer and Producer input types must be EDM4hep collections or maps to collections"); + static_assert(((std::is_base_of_v || isMapToCollLike::value) && ...), + "Transformer and Producer output types must be EDM4hep collections or maps to collections"); + + template + using InputHandle_t = Gaudi::Functional::details::InputHandle_t>; + template + using OutputHandle_t = Gaudi::Functional::details::OutputHandle_t>; + + std::tuple::type>>...> m_inputs; + std::tuple::type>>...> m_outputs; + std::array>, sizeof...(In)> m_inputLocations{}; + std::array>, sizeof...(Out)> m_outputLocations{}; + + using base_class = Gaudi::Functional::details::DataHandleMixin, std::tuple<>, Traits_>; + + using KeyValues = typename base_class::KeyValues; + + template + MultiTransformer(std::string name, ISvcLocator* locator, const IArgs& inputs, std::index_sequence, + const OArgs& outputs, std::index_sequence) + : base_class(std::move(name), locator), + m_inputLocations{Gaudi::Property>{ + this, std::get(inputs).first, to_DataObjID(std::get(inputs).second), + [this](Gaudi::Details::PropertyBase&) { + std::vector::type>> h; + for (auto& value : this->m_inputLocations[I].value()) { + auto handle = InputHandle_t::type>(value, this); + h.push_back(std::move(handle)); + } + std::get(m_inputs) = std::move(h); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...}, + m_outputLocations{Gaudi::Property>{ + this, std::get(outputs).first, to_DataObjID(std::get(outputs).second), + [this](Gaudi::Details::PropertyBase&) { + std::vector::type>> h; + // Is this needed? + // std::sort(this->m_outputLocations[J].value().begin(), this->m_outputLocations[J].value().end(), + // [](const DataObjID& a, const DataObjID& b) { return a.key() < b.key(); }); + for (auto& inpID : this->m_outputLocations[J].value()) { + if (inpID.key().empty()) { + continue; + } + auto handle = OutputHandle_t::type>(inpID, this); + h.push_back(std::move(handle)); + } + std::get(m_outputs) = std::move(h); + }, + Gaudi::Details::Property::ImmediatelyInvokeHandler{true}}...} {} + + constexpr static std::size_t N_in = sizeof...(In); + constexpr static std::size_t N_out = sizeof...(Out); + + MultiTransformer(std::string name, ISvcLocator* locator, + Gaudi::Functional::details::RepeatValues_ const& inputs, + Gaudi::Functional::details::RepeatValues_ const& outputs) + : MultiTransformer(std::move(name), locator, inputs, std::index_sequence_for{}, outputs, + std::index_sequence_for{}) {} + + // derived classes are NOT allowed to implement execute ... + StatusCode execute(const EventContext& ctx) const override final { + try { + auto tmp = filter_evtcontext_tt::apply(*this, ctx, this->m_inputs); + putMapOutputs<0, Out...>(std::move(tmp), m_outputs, this); + return Gaudi::Functional::FilterDecision::PASSED; + } catch (GaudiException& e) { + (e.code() ? this->warning() : this->error()) << e.tag() << " : " << e.message() << endmsg; + return e.code(); + } + } + + // ... instead, they must implement the following operator + virtual std::tuple operator()(const In&...) const = 0; + }; + + } // namespace details + + template + using MultiTransformer = details::MultiTransformer; + + template + using Transformer = details::Transformer; + +} // namespace k4FWCore + +#endif // FWCORE_TRANSFORMER_H diff --git a/k4FWCore/scripts/k4run b/k4FWCore/scripts/k4run index 36ed60ee..3facf5d5 100755 --- a/k4FWCore/scripts/k4run +++ b/k4FWCore/scripts/k4run @@ -3,7 +3,6 @@ import os import sys import argparse -from multiprocessing import cpu_count import logging from k4FWCore.utils import load_file @@ -232,7 +231,7 @@ def main(): # Do the real processing retcode = c.run(opts.gdb) # User requested stop returns non-zero exit code see: https://github.com/key4hep/k4FWCore/issues/125 - if ApplicationMgr().EvtMax == -1 and retcode == 4: + if retcode == 4: retcode = 0 sys.exit(retcode) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 00000000..d370a894 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,19 @@ +#[[ +Copyright (c) 2014-2024 Key4hep-Project. + +This file is part of Key4hep. +See https://key4hep.github.io/key4hep-doc/ for further info. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +]] +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/k4FWCore DESTINATION python) diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py new file mode 100644 index 00000000..8c5dc2f3 --- /dev/null +++ b/python/k4FWCore/ApplicationMgr.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from Configurables import ApplicationMgr as AppMgr +from Configurables import Reader, Writer, IOSvc, Gaudi__Sequencer +import os +from podio.root_io import Reader as PodioReader + + +class ApplicationMgr: + """ApplicationMgr is a class that wraps the Gaudi ApplicationMgr class to + + - Give the reader the collections it's going to read so that + the scheduler can know which algorithms can't run until the + collections are available + - When running multithreaded and EvtMax is -1, set the number of events to + be run to the number of events in the file so that no more events than + necessary are scheduled + - Wrap inside a sequencer the set of algorithms and a Writer (if any) so that + when running multithreaded the writer runs after the algorithms + """ + + def __init__(self, **kwargs): + self._mgr = AppMgr(**kwargs) + + for conf in frozenset(self._mgr.allConfigurables.values()): + if not isinstance(conf, IOSvc): + continue + props = conf.getPropertiesWithDescription() + reader = writer = None + add_reader = add_writer = False + for alg in self._mgr.TopAlg: + if isinstance(alg, Reader): + reader = alg + elif isinstance(alg, Writer): + writer = alg + if reader is None and props["input"][0]: + reader = Reader("k4FWCore__Reader") + add_reader = True + # It seems for a single string the default without a value is '' + # while for a list it's an empty list + if writer is None and props["output"][0] and props["output"][0] != "": + writer = Writer("k4FWCore__Writer") + # Let's tell the Reader one of the input files so it can + # know which collections it's going to read + if reader is not None: + # Open the files and get the number of events. This is necessary to + # avoid errors when running multithreaded since if we have, for + # example, 10 events and we are running 9 at the same time, then + # (possibly) the first 9 complete and 9 more are scheduled, out of + # which only one will be finished without errors. If we know the + # number of events in advance then we can just schedule those. + if props["input"][0]: + if os.path.exists(props["input"][0][0]): + path = props["input"][0][0] + else: + path = os.getcwd() + "/" + props["input"][0][0] + podio_reader = PodioReader(path) + if self._mgr.EvtMax == -1: + self._mgr.EvtMax = podio_reader._reader.getEntries("events") + frame = podio_reader.get("events")[0] + collections = list(frame.getAvailableCollections()) + reader.InputCollections = collections + self._mgr.TopAlg = ([reader] if add_reader else []) + self._mgr.TopAlg + # Assume the writer is at the end + # Algorithms are wrapped with Sequential=False so that they can run in parallel + # The algorithms and Writer are wrapped with Sequential=True so that they can not + # run in parallel + if writer: + self._mgr.TopAlg = [ + Gaudi__Sequencer( + "k4FWCore__Sequencer", + Sequential=True, + Members=[ + Gaudi__Sequencer( + "k4FWCore__Algs", + Members=self._mgr.TopAlg, + Sequential=False, + ShortCircuit=False, + ), + writer, + ], + ) + ] diff --git a/python/k4FWCore/IOSvc.py b/python/k4FWCore/IOSvc.py new file mode 100644 index 00000000..edae89fb --- /dev/null +++ b/python/k4FWCore/IOSvc.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from Configurables import IOSvc as IO +import os + + +class IOSvc: + def __init__(self, *args, **kwargs): + self._svc = IO(**kwargs) + + def __getattr__(self, attr): + return getattr(self._svc, attr) + + def __setattr__(self, attr, value): + if attr == "_svc": + super().__setattr__(attr, value) + return + + # Allow to specify a single string for input when what we want is a list + if attr == "input": + if isinstance(value, str): + value = [value] + if attr == "output": + if os.path.dirname(value) and not os.path.exists(os.path.dirname(value)): + os.makedirs(os.path.dirname(value)) + setattr(self._svc, attr, value) diff --git a/python/k4FWCore/__init__.py b/python/k4FWCore/__init__.py new file mode 100644 index 00000000..a3db4cf6 --- /dev/null +++ b/python/k4FWCore/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .ApplicationMgr import ApplicationMgr +from .IOSvc import IOSvc diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index 8bb5209d..88962474 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -29,8 +29,8 @@ set(K4RUN ${PROJECT_SOURCE_DIR}/k4FWCore/scripts/k4run) function(set_test_env testname) set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "ROOT_INCLUDE_PATH=$<$:$/../include>:$<$:$/../include>:$ENV{ROOT_INCLUDE_PATH}") - set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${CMAKE_PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:$<$:$>:$<$:$>:$<$:$>:$ENV{LD_LIBRARY_PATH}") - set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${CMAKE_PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}") + set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir/${PROJECT_NAME}:${PROJECT_BINARY_DIR}/test/k4FWCoreTest:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir/k4FWCoreTest:$<$:$>:$<$:$>:$<$:$>:$ENV{LD_LIBRARY_PATH}") + set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/${PROJECT_NAME}/genConfDir:${PROJECT_BINARY_DIR}/test/k4FWCoreTest/genConfDir:$ENV{PYTHONPATH}") endfunction() function(add_test_with_env testname) @@ -49,8 +49,8 @@ function(add_test_with_env testname) endif() endforeach() add_test(NAME ${testname} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMAND ${K4RUN} ${TEST_ARGUMENTS}) + # WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND ${K4RUN} ${CMAKE_CURRENT_LIST_DIR}/${TEST_ARGUMENTS}) if(TEST_PROPERTIES_FOUND) set_tests_properties(${testname} PROPERTIES ${TEST_PROPERTIES}) endif() @@ -62,7 +62,7 @@ add_test_with_env(CreateExampleEventDataInDirectory options/createExampleEventDa add_test_with_env(CheckExampleEventData options/checkExampleEventData.py PROPERTIES DEPENDS CreateExampleEventData) add_test_with_env(CheckExampleEventData_noCollections options/checkExampleEventData.py --collections= PROPERTIES DEPENDS CreateExampleEventData) -add_test_with_env(CheckExampleEventData_toolong -n 999 options/checkExampleEventData.py PROPERTIES PASS_REGULAR_EXPRESSION +add_test_with_env(CheckExampleEventData_toolong options/checkExampleEventData.py -n 999 PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop" DEPENDS CreateExampleEventData) add_test_with_env(CheckExampleEventData_unbounded options/checkExampleEventData.py -n -1 PROPERTIES PASS_REGULAR_EXPRESSION "Application Manager Terminated successfully with a user requested ScheduledStop" DEPENDS CreateExampleEventData) @@ -71,6 +71,7 @@ set_property(TEST ReadExampleEventData APPEND PROPERTY DEPENDS CreateExampleEven add_test_with_env(ReadExampleDataFromNthEvent options/readExampleDataFromNthEvent.py PROPERTIES DEPENDS CreateExampleEventData) add_test_with_env(AlgorithmWithTFile options/TestAlgorithmWithTFile.py) +set_property(TEST AlgorithmWithTFile PROPERTY WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) add_test(NAME AlgorithmWithTFileCheckFrameworkOutput WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND python scripts/check_TestAlgorithmWithTFile_framework_nonempty.py) @@ -90,7 +91,7 @@ PROPERTIES PASS_REGULAR_EXPRESSION "TwasBrilligAndTheSlithyToves" ) add_test(NAME checkKeepDropSwitch WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMAND python scripts/check_KeepDropSwitch.py) + COMMAND python scripts/check_KeepDropSwitch.py ${PROJECT_BINARY_DIR}/test/k4FWCoreTest/output_k4test_exampledata_2.root) set_test_env(checkKeepDropSwitch) set_property(TEST checkKeepDropSwitch APPEND PROPERTY DEPENDS ReadExampleEventData) add_test_with_env(TestUniqueIDGenSvc options/TestUniqueIDGenSvc.py) @@ -99,17 +100,47 @@ add_test_with_env(EventHeaderCheck options/runEventHeaderCheck.py PROPERTIES DEP add_test(NAME TestExec WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND python options/TestExec.py) set_test_env(TestExec) -add_test_with_env(Testk4runNoArguments PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run , use --help to get a complete list of arguments") +add_test(NAME Testk4runNoArgumentsHelp COMMAND ${K4RUN}) +set_test_env(Testk4runNoArgumentsHelp) +set_tests_properties(Testk4runNoArgumentsHelp PROPERTIES PASS_REGULAR_EXPRESSION "Usage: k4run , use --help to get a complete list of arguments") add_test_with_env(Testk4runCustomArguments options/TestArgs.py --foo=42 PROPERTIES PASS_REGULAR_EXPRESSION "The answer is 42") add_test_with_env(Testk4runVerboseOutput options/TestArgs.py --verbose PROPERTIES PASS_REGULAR_EXPRESSION " VERBOSE ") add_test_with_env(Testk4runHelpOnly options/TestArgs.py --help PROPERTIES PASS_REGULAR_EXPRESSION "show this help message and exit") -add_test_with_env(ExampleFunctionalProducer options/runExampleFunctionalProducer.py) -add_test_with_env(ExampleFunctionalConsumer options/runExampleFunctionalConsumer.py PROPERTIES DEPENDS ExampleFunctionalProducer) -add_test_with_env(ExampleFunctionalTransformer options/runExampleFunctionalTransformer.py PROPERTIES DEPENDS ExampleFunctionalProducer) -add_test_with_env(ExampleFunctionalProducerMultiple options/runExampleFunctionalProducerMultiple.py) -add_test_with_env(ExampleFunctionalConsumerMultiple options/runExampleFunctionalConsumerMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) -add_test_with_env(ExampleFunctionalTransformerMultiple options/runExampleFunctionalTransformerMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) -add_test_with_env(FunctionalChain options/runFunctionalChain.py) -add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) +add_test_with_env(FunctionalMemory options/ExampleFunctionalMemory.py) +add_test_with_env(FunctionalMTMemory options/ExampleFunctionalMTMemory.py) +add_test_with_env(FunctionalMultipleMemory options/ExampleFunctionalMultipleMemory.py) +add_test_with_env(FunctionalProducer options/ExampleFunctionalProducer.py) +add_test_with_env(FunctionalProducerAnother options/ExampleFunctionalProducer.py --second) +add_test_with_env(FunctionalProducerMultiple options/ExampleFunctionalProducerMultiple.py) +add_test_with_env(FunctionalProducerAbsolutePath options/ExampleFunctionalProducerAbsolutePath.py) +add_test_with_env(FunctionalFile options/ExampleFunctionalFile.py PROPERTIES DEPENDS FunctionalProducer) +add_test_with_env(FunctionalSeveralInputFiles options/ExampleFunctionalSeveralInputFiles.py) +set_tests_properties(FunctionalSeveralInputFiles PROPERTIES DEPENDS "FunctionalProducer;FunctionalProducerAnother") +add_test_with_env(FunctionalMTFile options/ExampleFunctionalMTFile.py PROPERTIES DEPENDS FunctionalProducer) +add_test_with_env(FunctionalMultipleFile options/ExampleFunctionalFileMultiple.py PROPERTIES DEPENDS FunctionalProducerMultiple) +add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS FunctionalProducerMultiple) +add_test_with_env(FunctionalMixIOSvc options/runFunctionalMix.py --iosvc PROPERTIES DEPENDS FunctionalProducerMultiple) +add_test_with_env(FunctionalOutputCommands options/ExampleFunctionalOutputCommands.py PROPERTIES DEPENDS FunctionalProducerMultiple) +add_test_with_env(FunctionalConsumerRuntimeCollections options/ExampleFunctionalConsumerRuntimeCollections.py) +add_test_with_env(FunctionalConsumerRuntimeCollectionsMultiple options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py) +add_test_with_env(FunctionalProducerRuntimeCollections options/ExampleFunctionalProducerRuntimeCollections.py) +add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctionalTransformerRuntimeCollections.py) +add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTransformerRuntimeEmpty.py) +add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py) + +add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py) +set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc") +# Do this after checking the files not to overwrite them +add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION + "Application Manager Terminated successfully with a user requested ScheduledStop") + +# The following is done to make the tests work without installing the files in +# the installation directory. The k4FWCore in the build directory is populated by +# Gaudi so we can't just make a k4FWCore folder and put the python files there. +# We put everything in one of the folders generated by Gaudi so that importing from +# k4FWCore will give the right result +add_custom_command(TARGET k4FWCoreTestPlugins POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${PROJECT_SOURCE_DIR}/python/k4FWCore/ ${PROJECT_BINARY_DIR}/k4FWCore/genConfDir/k4FWCore) diff --git a/test/k4FWCoreTest/options/CheckOutputFiles.py b/test/k4FWCoreTest/options/CheckOutputFiles.py new file mode 100644 index 00000000..1cd73238 --- /dev/null +++ b/test/k4FWCoreTest/options/CheckOutputFiles.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +try: + import podio +except ImportError: + import os + + print(f'PYTHONPATH={os.environ["PYTHONPATH"]}') + raise + + +def check_collections(filename, names): + print(f'Checking file "{filename}" for collections {names}') + podio_reader = podio.root_io.Reader(filename) + frames = podio_reader.get("events") + if not len(frames) and len(names): + print(f"File {filename} is empty but {names} are expected") + raise RuntimeError("File is empty but should not be") + for frame in frames: + available = set(frame.getAvailableCollections()) + if available != set(names): + print( + f"These collections should be in the frame but are not: {set(names) - available}" + ) + print( + f"These collections are in the frame but should not be: {available - set(names)}" + ) + raise RuntimeError("Collections in frame do not match expected collections") + + +check_collections("functional_transformer.root", ["MCParticles", "NewMCParticles"]) +check_collections( + "functional_transformer_multiple.root", + [ + "VectorFloat", + "MCParticles1", + "MCParticles2", + "SimTrackerHits", + "TrackerHits", + "Tracks", + "Counter", + "NewMCParticles", + ], +) +check_collections( + "functional_transformer_multiple_output_commands.root", + ["VectorFloat", "MCParticles1", "MCParticles2", "SimTrackerHits", "TrackerHits"], +) +check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"]) +check_collections( + "functional_transformer_runtime_empty.root", + ["MCParticles0", "MCParticles1", "MCParticles2"], +) +check_collections( + "functional_transformerMT.root", + [ + "VectorFloat", + "MCParticles1", + "MCParticles2", + "SimTrackerHits", + "TrackerHits", + "Tracks", + "NewMCParticles", + ], +) + + +mix_collections = [ + # From file + "VectorFloat", + "MCParticles1", + "MCParticles2", + "SimTrackerHits", + "TrackerHits", + "Tracks", + # Produced by functional + "FunctionalVectorFloat", + "FunctionalMCParticles", + "FunctionalMCParticles2", + "FunctionalSimTrackerHits", + "FunctionalTrackerHits", + "FunctionalTracks", + # Produced by an old algorithm + "OldAlgorithmMCParticles", + "OldAlgorithmSimTrackerHits", + "OldAlgorithmTrackerHits", + "OldAlgorithmTracks", + "OldAlgorithmVectorFloat", + # Produced by the last transformer + "Counter", + "TransformedFunctionalMCParticles1", +] + + +# Not working, collections produced by functional algorithms are not being written to the file +# check_collections( +# "output_k4test_exampledata_functional_mix.root", +# mix_collections, +# ) + +check_collections( + "output_k4test_exampledata_functional_mix_iosvc.root", + mix_collections, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py new file mode 100644 index 00000000..68b38c28 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a producer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalProducer, + ExampleFunctionalConsumerRuntimeCollections, +) +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc + +producer0 = ExampleFunctionalProducer( + "Producer0", + OutputCollection=["MCParticles0"], +) +producer1 = ExampleFunctionalProducer( + "Producer1", + OutputCollection=["MCParticles1"], +) +producer2 = ExampleFunctionalProducer( + "Producer2", + OutputCollection=["MCParticles2"], +) +consumer = ExampleFunctionalConsumerRuntimeCollections( + "Consumer", + # InputCollection="MCParticles0 MCParticles1 MCParticles2", + InputCollection=["MCParticles0", "MCParticles1", "MCParticles2"], + Offset=0, +) + + +ApplicationMgr( + TopAlg=[producer0, producer1, producer2, consumer], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py new file mode 100644 index 00000000..dbeb62da --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a producer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalProducerMultiple, + ExampleFunctionalConsumerRuntimeCollectionsMultiple, +) +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc + +producer0 = ExampleFunctionalProducerMultiple( + "Producer0", + OutputCollectionFloat=["VectorFloat0"], + OutputCollectionParticles1=["MCParticles0"], + OutputCollectionParticles2=["MCParticles1"], + OutputCollectionSimTrackerHits=["SimTrackerHits0"], + OutputCollectionTrackerHits=["TrackerHits0"], + OutputCollectionTracks=["Tracks0"], + ExampleInt=5, +) +producer1 = ExampleFunctionalProducerMultiple( + "Producer1", + OutputCollectionFloat=["VectorFloat1"], + OutputCollectionParticles1=["MCParticles2"], + OutputCollectionParticles2=["MCParticles3"], + OutputCollectionSimTrackerHits=["SimTrackerHits1"], + OutputCollectionTrackerHits=["TrackerHits1"], + OutputCollectionTracks=["Tracks1"], + ExampleInt=5, +) +producer2 = ExampleFunctionalProducerMultiple( + "Producer2", + OutputCollectionFloat=["VectorFloat2"], + OutputCollectionParticles1=["MCParticles4"], + OutputCollectionParticles2=["MCParticles5"], + OutputCollectionSimTrackerHits=["SimTrackerHits2"], + OutputCollectionTrackerHits=["TrackerHits2"], + OutputCollectionTracks=["Tracks2"], + ExampleInt=5, +) + +consumer = ExampleFunctionalConsumerRuntimeCollectionsMultiple( + "Consumer", + Particles=[ + "MCParticles0", + "MCParticles1", + "MCParticles2", + "MCParticles3", + "MCParticles4", + ], + Tracks=["Tracks0", "Tracks1", "Tracks2"], + SimTrackerHits=["SimTrackerHits0"], + Offset=0, +) + + +ApplicationMgr( + TopAlg=[producer0, producer1, producer2, consumer], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/runExampleFunctionalTransformer.py b/test/k4FWCoreTest/options/ExampleFunctionalFile.py similarity index 58% rename from test/k4FWCoreTest/options/runExampleFunctionalTransformer.py rename to test/k4FWCoreTest/options/ExampleFunctionalFile.py index 2047c939..18f7475d 100644 --- a/test/k4FWCoreTest/options/runExampleFunctionalTransformer.py +++ b/test/k4FWCoreTest/options/ExampleFunctionalFile.py @@ -22,34 +22,21 @@ from Gaudi.Configuration import INFO from Configurables import ExampleFunctionalTransformer -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioOutput -from Configurables import PodioInput +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer.root" - -inp = PodioInput() -inp.collections = [ - "MCParticles", -] - -out = PodioOutput("out") -out.filename = "output_k4test_exampledata_transformer.root" -# The collections that we don't drop will also be present in the output file -out.outputCommands = ["drop MCParticles"] +svc = IOSvc("IOSvc") +svc.input = "output_k4test_exampledata_producer.root" +svc.output = "functional_transformer.root" transformer = ExampleFunctionalTransformer( - "ExampleFunctionalTransformer", - InputCollection="MCParticles", - OutputCollection="NewMCParticles", + "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"] ) -ApplicationMgr( - TopAlg=[inp, transformer, out], +mgr = ApplicationMgr( + TopAlg=[transformer], EvtSel="NONE", - EvtMax=10, - ExtSvc=[k4DataSvc("EventDataSvc")], + EvtMax=-1, + ExtSvc=[EventDataSvc("EventDataSvc")], OutputLevel=INFO, ) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py new file mode 100644 index 00000000..0fae21d4 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ExampleFunctionalTransformerMultiple +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc + +svc = IOSvc("IOSvc") +svc.input = "output_k4test_exampledata_producer_multiple.root" +svc.output = "functional_transformer_multiple.root" + +transformer = ExampleFunctionalTransformerMultiple( + "Transformer", + # InputCollection="MCParticles", + # OutputCollection="NewMCParticles") +) + +mgr = ApplicationMgr( + TopAlg=[transformer], + EvtSel="NONE", + EvtMax=-1, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py new file mode 100644 index 00000000..2af168f6 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalMTFile.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO, WARNING +from Configurables import ExampleFunctionalTransformer, ExampleFunctionalConsumer +from Configurables import HiveSlimEventLoopMgr, HiveWhiteBoard, AvalancheSchedulerSvc +from k4FWCore import ApplicationMgr, IOSvc + +evtslots = 6 +threads = 6 + +whiteboard = HiveWhiteBoard( + "EventDataSvc", + EventSlots=evtslots, + ForceLeaves=True, +) + +slimeventloopmgr = HiveSlimEventLoopMgr( + "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING +) + +scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True, OutputLevel=WARNING) + +svc = IOSvc("IOSvc") +svc.input = "output_k4test_exampledata_producer_multiple.root" +svc.output = "functional_transformerMT.root" + +consumer = ExampleFunctionalConsumer( + "Consumer1", + InputCollection=["MCParticles1"], + Offset=0, +) + +transformer = ExampleFunctionalTransformer( + "Transformer", InputCollection=["MCParticles1"], OutputCollection=["NewMCParticles"] +) + +mgr = ApplicationMgr( + TopAlg=[consumer, transformer], + EvtSel="NONE", + EvtMax=-1, + ExtSvc=[whiteboard], + EventLoop=slimeventloopmgr, + MessageSvcType="InertMessageSvc", + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py new file mode 100644 index 00000000..4d11bcb0 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO, WARNING +from Configurables import ( + ExampleFunctionalProducer, + ExampleFunctionalTransformer, + ExampleFunctionalConsumer, +) +from Configurables import HiveWhiteBoard, HiveSlimEventLoopMgr, AvalancheSchedulerSvc +from k4FWCore import ApplicationMgr + +evtslots = 5 +threads = 3 + +whiteboard = HiveWhiteBoard( + "EventDataSvc", + EventSlots=evtslots, + ForceLeaves=True, +) + +slimeventloopmgr = HiveSlimEventLoopMgr(SchedulerName="AvalancheSchedulerSvc", OutputLevel=WARNING) + +scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=WARNING) +scheduler.ShowDataDependencies = True +scheduler.ShowDataFlow = True +scheduler.ShowControlFlow = True + +transformer = ExampleFunctionalTransformer( + "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"] +) + +producer = ExampleFunctionalProducer("Producer", OutputCollection=["MCParticles"]) + +consumer = ExampleFunctionalConsumer( + "Consumer", + InputCollection=["NewMCParticles"], + Offset=10, +) + + +ApplicationMgr( + TopAlg=[producer, transformer, consumer], + EvtSel="NONE", + EvtMax=10, + EventLoop=slimeventloopmgr, + ExtSvc=[whiteboard], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py similarity index 60% rename from test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py rename to test/k4FWCoreTest/options/ExampleFunctionalMemory.py index d103921b..c79f1090 100644 --- a/test/k4FWCoreTest/options/runExampleFunctionalConsumerMultiple.py +++ b/test/k4FWCoreTest/options/ExampleFunctionalMemory.py @@ -21,32 +21,30 @@ # to check that the contents of the file are the expected ones from Gaudi.Configuration import INFO -from Configurables import ExampleFunctionalConsumerMultiple -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioInput +from Configurables import ( + ExampleFunctionalTransformer, + ExampleFunctionalProducer, + ExampleFunctionalConsumer, +) +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer_multiple.root" +transformer = ExampleFunctionalTransformer( + "Transformer", InputCollection=["MCParticles"], OutputCollection=["NewMCParticles"] +) -inp = PodioInput() -inp.collections = [ - "VectorFloat", - "MCParticles1", - "MCParticles2", - "SimTrackerHits", - "TrackerHits", - "Tracks", -] +producer = ExampleFunctionalProducer("Producer", OutputCollection=["MCParticles"]) -consumer = ExampleFunctionalConsumerMultiple( - "ExampleFunctionalConsumerMultiple", +consumer = ExampleFunctionalConsumer( + "Consumer", + InputCollection=["NewMCParticles"], + Offset=10, ) ApplicationMgr( - TopAlg=[inp, consumer], + TopAlg=[producer, transformer, consumer], EvtSel="NONE", EvtMax=10, - ExtSvc=[podioevent], + ExtSvc=[EventDataSvc("EventDataSvc")], OutputLevel=INFO, ) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py new file mode 100644 index 00000000..2808ecea --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalMultipleMemory.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalProducerMultiple, + ExampleFunctionalTransformerMultiple, + ExampleFunctionalConsumerMultiple, +) +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr + +transformer = ExampleFunctionalTransformerMultiple( + "Transformer", + InputCollectionFloat=["VectorFloat"], + InputCollectionParticles=["MCParticles1"], + InputCollectionSimTrackerHits=["SimTrackerHits"], + InputCollectionTrackerHits=["TrackerHits"], + InputCollectionTracks=["Tracks"], + OutputCollectionCounter=["Counter"], + OutputCollectionParticles=["NewMCParticles"], + Offset=10, +) + +producer = ExampleFunctionalProducerMultiple("Producer") + +consumer = ExampleFunctionalConsumerMultiple( + "Consumer", + InputCollectionFloat=["VectorFloat"], + InputCollectionParticles=["NewMCParticles"], + InputCollectionSimTrackerHits=["SimTrackerHits"], + InputCollectionTrackerHits=["TrackerHits"], + InputCollectionTracks=["Tracks"], + Offset=10, +) + + +ApplicationMgr( + TopAlg=[producer, transformer, consumer], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py new file mode 100644 index 00000000..d6db96aa --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ExampleFunctionalTransformerMultiple +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc + +svc = IOSvc("IOSvc") +svc.input = "output_k4test_exampledata_producer_multiple.root" +svc.output = "functional_transformer_multiple_output_commands.root" +svc.outputCommands = [ + "drop Tracks", + "drop Counter", + "drop NewMCParticles", +] + +transformer = ExampleFunctionalTransformerMultiple( + "Transformer", + # InputCollection="MCParticles", + # OutputCollection="NewMCParticles") +) + +mgr = ApplicationMgr( + TopAlg=[transformer], + EvtSel="NONE", + EvtMax=-1, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducer.py b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py new file mode 100644 index 00000000..12ff4dac --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalProducer.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example using a producer with a single output and saving that to a file + +from Gaudi.Configuration import INFO +from Configurables import ExampleFunctionalProducer +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc +from Configurables import Writer +from k4FWCore.parseArgs import parser + +parser.add_argument("--second", action="store_true") +args = parser.parse_known_args() + +iosvc = IOSvc("IOSvc") +name = ( + "output_k4test_exampledata_producer.root" + if not args[0].second + else "output_k4test_exampledata_producer2.root" +) +iosvc.output = name +# Collections can be dropped +# out.outputCommands = ["drop *"] + + +producer = ExampleFunctionalProducer("ExampleFunctionalProducer") + +writer = Writer("Writer") + +ApplicationMgr( + TopAlg=[producer, writer], + EvtSel="NONE", + EvtMax=10 if not args[0].second else 20, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/runExampleFunctionalProducer.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py similarity index 73% rename from test/k4FWCoreTest/options/runExampleFunctionalProducer.py rename to test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py index e66ff707..98cf71a3 100644 --- a/test/k4FWCoreTest/options/runExampleFunctionalProducer.py +++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py @@ -20,25 +20,19 @@ # This is an example using a producer with a single output and saving that to a file from Gaudi.Configuration import INFO +from Configurables import EventDataSvc from Configurables import ExampleFunctionalProducer -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioOutput - -podioevent = k4DataSvc("EventDataSvc") - -out = PodioOutput("out") -out.filename = "output_k4test_exampledata_producer.root" -# Collections can be dropped -# out.outputCommands = ["drop *"] +from k4FWCore import ApplicationMgr, IOSvc +io = IOSvc("IOSvc") +io.output = "/tmp/a/b/c/output_k4test_exampledata_producer.root" producer = ExampleFunctionalProducer("ExampleFunctionalProducer") ApplicationMgr( - TopAlg=[producer, out], + TopAlg=[producer], EvtSel="NONE", EvtMax=10, - ExtSvc=[k4DataSvc("EventDataSvc")], + ExtSvc=[EventDataSvc("EventDataSvc")], OutputLevel=INFO, ) diff --git a/test/k4FWCoreTest/options/runExampleFunctionalProducerMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py similarity index 65% rename from test/k4FWCoreTest/options/runExampleFunctionalProducerMultiple.py rename to test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py index b3920470..5ceae857 100644 --- a/test/k4FWCoreTest/options/runExampleFunctionalProducerMultiple.py +++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py @@ -21,32 +21,30 @@ from Gaudi.Configuration import INFO from Configurables import ExampleFunctionalProducerMultiple -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioOutput +from k4FWCore import ApplicationMgr, IOSvc +from Configurables import EventDataSvc -podioevent = k4DataSvc("EventDataSvc") -out = PodioOutput("out") -out.filename = "output_k4test_exampledata_producer_multiple.root" +iosvc = IOSvc("IOSvc") +iosvc.output = "output_k4test_exampledata_producer_multiple.root" # Collections can be dropped # out.outputCommands = ["drop *"] producer = ExampleFunctionalProducerMultiple( "ExampleFunctionalProducerMultiple", - OutputCollectionFloat="VectorFloat", - OutputCollectionParticles1="MCParticles1", - OutputCollectionParticles2="MCParticles2", - OutputCollectionSimTrackerHits="SimTrackerHits", - OutputCollectionTrackerHits="TrackerHits", - OutputCollectionTracks="Tracks", + OutputCollectionFloat=["VectorFloat"], + OutputCollectionParticles1=["MCParticles1"], + OutputCollectionParticles2=["MCParticles2"], + OutputCollectionSimTrackerHits=["SimTrackerHits"], + OutputCollectionTrackerHits=["TrackerHits"], + OutputCollectionTracks=["Tracks"], ExampleInt=5, ) ApplicationMgr( - TopAlg=[producer, out], + TopAlg=[producer], EvtSel="NONE", EvtMax=10, - ExtSvc=[k4DataSvc("EventDataSvc")], + ExtSvc=[EventDataSvc("EventDataSvc")], OutputLevel=INFO, ) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py new file mode 100644 index 00000000..f0ecc472 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalProducerRuntimeCollections, + ExampleFunctionalConsumer, +) +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc + +producer = ExampleFunctionalProducerRuntimeCollections( + "Producer", + OutputCollections=["MCParticles0", "MCParticles1", "MCParticles2"], +) + +consumer0 = ExampleFunctionalConsumer( + "Consumer0", + InputCollection=["MCParticles0"], + Offset=0, +) +consumer1 = ExampleFunctionalConsumer( + "Consumer1", + InputCollection=["MCParticles1"], + Offset=0, +) +consumer2 = ExampleFunctionalConsumer( + "Consumer2", + InputCollection=["MCParticles2"], + Offset=0, +) + + +ApplicationMgr( + TopAlg=[producer, consumer0, consumer1, consumer2], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/runExampleFunctionalConsumer.py b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py similarity index 66% rename from test/k4FWCoreTest/options/runExampleFunctionalConsumer.py rename to test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py index e8637492..fc35473c 100644 --- a/test/k4FWCoreTest/options/runExampleFunctionalConsumer.py +++ b/test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py @@ -17,32 +17,26 @@ # limitations under the License. # -# This is an example reading from a file and using a consumer with a single input +# This is an example reading from a file and using a consumer with several inputs # to check that the contents of the file are the expected ones from Gaudi.Configuration import INFO from Configurables import ExampleFunctionalConsumer -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioInput +from Configurables import EventDataSvc +from k4FWCore import ApplicationMgr, IOSvc -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer.root" - -inp = PodioInput() -inp.collections = [ - "MCParticles", +svc = IOSvc("IOSvc") +svc.input = [ + "output_k4test_exampledata_producer.root", + "output_k4test_exampledata_producer2.root", ] -consumer = ExampleFunctionalConsumer( - "ExampleFunctionalConsumer", - InputCollection="MCParticles", -) +consumer = ExampleFunctionalConsumer("Consumer", InputCollection=["MCParticles"], Offset=0) -ApplicationMgr( - TopAlg=[inp, consumer], +mgr = ApplicationMgr( + TopAlg=[consumer], EvtSel="NONE", - EvtMax=10, - ExtSvc=[podioevent], + EvtMax=30, + ExtSvc=[EventDataSvc("EventDataSvc")], OutputLevel=INFO, ) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py new file mode 100644 index 00000000..0f85a361 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollections.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalTransformerRuntimeCollections, + ExampleFunctionalConsumer, + ExampleFunctionalProducer, +) +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc + +producer0 = ExampleFunctionalProducer( + "Producer0", + OutputCollection=["MCParticles0"], +) +producer1 = ExampleFunctionalProducer( + "Producer1", + OutputCollection=["MCParticles1"], +) +producer2 = ExampleFunctionalProducer( + "Producer2", + OutputCollection=["MCParticles2"], +) + +transformer = ExampleFunctionalTransformerRuntimeCollections( + "Transformer", + InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"], + OutputCollections=["NewMCParticles0", "NewMCParticles1", "NewMCParticles2"], +) + +consumer0 = ExampleFunctionalConsumer( + "Consumer0", + InputCollection=["NewMCParticles0"], + Offset=0, +) +consumer1 = ExampleFunctionalConsumer( + "Consumer1", + InputCollection=["NewMCParticles1"], + Offset=0, +) +consumer2 = ExampleFunctionalConsumer( + "Consumer2", + InputCollection=["NewMCParticles2"], + Offset=0, +) + + +ApplicationMgr( + TopAlg=[ + producer0, + producer1, + producer2, + transformer, + consumer0, + consumer1, + consumer2, + ], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py new file mode 100644 index 00000000..c7868b44 --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalTransformerRuntimeCollectionsMultiple, + ExampleFunctionalConsumerMultiple, + ExampleFunctionalProducerMultiple, +) +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc + +producer0 = ExampleFunctionalProducerMultiple( + "Producer0", + OutputCollectionFloat=["VectorFloat0"], + OutputCollectionParticles1=["MCParticles0"], + OutputCollectionParticles2=["MCParticles1"], + OutputCollectionSimTrackerHits=["SimTrackerHits0"], + OutputCollectionTrackerHits=["TrackerHits0"], + OutputCollectionTracks=["Tracks0"], + ExampleInt=5, +) +producer1 = ExampleFunctionalProducerMultiple( + "Producer1", + OutputCollectionFloat=["VectorFloat1"], + OutputCollectionParticles1=["MCParticles2"], + OutputCollectionParticles2=["MCParticles3"], + OutputCollectionSimTrackerHits=["SimTrackerHits1"], + OutputCollectionTrackerHits=["TrackerHits1"], + OutputCollectionTracks=["Tracks1"], + ExampleInt=5, +) +producer2 = ExampleFunctionalProducerMultiple( + "Producer2", + OutputCollectionFloat=["VectorFloat2"], + OutputCollectionParticles1=["MCParticles4"], + OutputCollectionParticles2=["MCParticles5"], + OutputCollectionSimTrackerHits=["SimTrackerHits2"], + OutputCollectionTrackerHits=["TrackerHits2"], + OutputCollectionTracks=["Tracks2"], + ExampleInt=5, +) + +transformer = ExampleFunctionalTransformerRuntimeCollectionsMultiple( + "Transformer", + InputCollectionFloat=["VectorFloat0", "VectorFloat1", "VectorFloat2"], + InputCollectionParticles=["MCParticles0", "MCParticles2", "MCParticles4"], + InputCollectionSimTrackerHits=[ + "SimTrackerHits0", + "SimTrackerHits1", + "SimTrackerHits2", + ], + InputCollectionTrackerHits=["TrackerHits0", "TrackerHits1", "TrackerHits2"], + InputCollectionTracks=["Tracks0", "Tracks1", "Tracks2"], + OutputCollectionFloat=["NewVectorFloat0", "NewVectorFloat1", "NewVectorFloat2"], + OutputCollectionParticles1=[ + "NewMCParticles0", + "NewMCParticles2", + "NewMCParticles4", + ], + OutputCollectionParticles2=[ + "", + ], + OutputCollectionSimTrackerHits=[ + "NewSimTrackerHits0", + "NewSimTrackerHits1", + "NewSimTrackerHits2", + ], + OutputCollectionTrackerHits=[ + "NewTrackerHits0", + "NewTrackerHits1", + "NewTrackerHits2", + ], + OutputCollectionTracks=["NewTracks0", "NewTracks1", "NewTracks2"], + Offset=0, +) + +consumer0 = ExampleFunctionalConsumerMultiple( + "Consumer0", + InputCollectionFloat=["NewVectorFloat0"], + InputCollectionParticles=["NewMCParticles0"], + InputCollectionSimTrackerHits=["NewSimTrackerHits0"], + InputCollectionTrackerHits=["NewTrackerHits0"], + InputCollectionTracks=["NewTracks0"], +) + +consumer1 = ExampleFunctionalConsumerMultiple( + "Consumer1", + InputCollectionFloat=["NewVectorFloat1"], + InputCollectionParticles=["NewMCParticles2"], + InputCollectionSimTrackerHits=["NewSimTrackerHits1"], + InputCollectionTrackerHits=["NewTrackerHits1"], + InputCollectionTracks=["NewTracks1"], +) + +consumer2 = ExampleFunctionalConsumerMultiple( + "Consumer2", + InputCollectionFloat=["NewVectorFloat2"], + InputCollectionParticles=["NewMCParticles4"], + InputCollectionSimTrackerHits=["NewSimTrackerHits2"], + InputCollectionTrackerHits=["NewTrackerHits2"], + InputCollectionTracks=["NewTracks2"], +) + + +ApplicationMgr( + TopAlg=[ + producer0, + producer1, + producer2, + transformer, + consumer0, + consumer1, + consumer2, + ], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py new file mode 100644 index 00000000..129d79ce --- /dev/null +++ b/test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is an example reading from a file and using a consumer with several inputs +# to check that the contents of the file are the expected ones + +from Gaudi.Configuration import INFO +from Configurables import ( + ExampleFunctionalTransformerRuntimeEmpty, + ExampleFunctionalProducer, +) +from k4FWCore import ApplicationMgr, IOSvc +from Configurables import EventDataSvc + +iosvc = IOSvc("IOSvc") +iosvc.output = "functional_transformer_runtime_empty.root" + +producer0 = ExampleFunctionalProducer( + "Producer0", + OutputCollection=["MCParticles0"], +) +producer1 = ExampleFunctionalProducer( + "Producer1", + OutputCollection=["MCParticles1"], +) +producer2 = ExampleFunctionalProducer( + "Producer2", + OutputCollection=["MCParticles2"], +) + +transformer = ExampleFunctionalTransformerRuntimeEmpty( + InputCollections=["MCParticles0", "MCParticles1", "MCParticles2"], + OutputCollections=[""], +) + +ApplicationMgr( + TopAlg=[ + producer0, + producer1, + producer2, + transformer, + ], + EvtSel="NONE", + EvtMax=10, + ExtSvc=[EventDataSvc("EventDataSvc")], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py index b7d0af72..d15eefb3 100644 --- a/test/k4FWCoreTest/options/runEventHeaderCheck.py +++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py @@ -19,23 +19,22 @@ # from Gaudi.Configuration import INFO -from Configurables import k4DataSvc -from Configurables import PodioInput from Configurables import ExampleEventHeaderConsumer -from Configurables import ApplicationMgr +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc, IOSvc, Reader -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "eventHeader.root" +svc = IOSvc("IOSvc") +svc.input = ["eventHeader.root"] +# svc.CollectionNames = ['MCParticles'] -inp = PodioInput() -inp.collections = [] +reader = Reader("Reader") consumer = ExampleEventHeaderConsumer("EventHeaderCheck", runNumber=42, eventNumberOffset=42) ApplicationMgr( - TopAlg=[inp, consumer], + TopAlg=[reader, consumer], EvtSel="NONE", EvtMax=-1, - ExtSvc=[podioevent], + ExtSvc=[EventDataSvc("EventDataSvc")], OutputLevel=INFO, ) diff --git a/test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py b/test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py deleted file mode 100644 index 44356fe1..00000000 --- a/test/k4FWCoreTest/options/runExampleFunctionalTransformerMultiple.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2014-2024 Key4hep-Project. -# -# This file is part of Key4hep. -# See https://key4hep.github.io/key4hep-doc/ for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from Gaudi.Configuration import INFO -from Configurables import ExampleFunctionalTransformerMultiple -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioOutput -from Configurables import PodioInput - -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer_multiple.root" - -inp = PodioInput() -inp.collections = [ - "VectorFloat", - "MCParticles1", - "SimTrackerHits", - "TrackerHits", - "Tracks", -] - -out = PodioOutput("out") -out.filename = "output_k4test_exampledata_transformer_multiple.root" -# The collections that we don't drop will also be present in the output file -out.outputCommands = [ - "drop VectorFloat", - "drop MCParticles1", - "drop SimTrackerHits", - "drop TrackerHits", - "drop Tracks", -] - -transformer = ExampleFunctionalTransformerMultiple( - "ExampleFunctionalTransformerMultiple", - InputCollectionFloat="VectorFloat", - InputCollectionParticles="MCParticles1", - InputCollectionSimTrackerHits="SimTrackerHits", - InputCollectionTrackerHits="TrackerHits", - InputCollectionTracks="Tracks", - OutputCollectionCounter="Counter", - OutputCollectionParticles="NewMCParticles", -) - -ApplicationMgr( - TopAlg=[inp, transformer, out], - EvtSel="NONE", - EvtMax=10, - ExtSvc=[k4DataSvc("EventDataSvc")], - OutputLevel=INFO, -) diff --git a/test/k4FWCoreTest/options/runFunctionalChain.py b/test/k4FWCoreTest/options/runFunctionalChain.py deleted file mode 100644 index ec2affd0..00000000 --- a/test/k4FWCoreTest/options/runFunctionalChain.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2014-2024 Key4hep-Project. -# -# This file is part of Key4hep. -# See https://key4hep.github.io/key4hep-doc/ for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This is an example of how to run several functional algorithms in a chain -# The producer produces a collection, the first consumer checks that the collection -# is the expected one, the transformer transforms the collection and creates a new -# and the second consumer checks that the new collection is the expected one - -from Gaudi.Configuration import INFO -from Configurables import ExampleFunctionalProducer -from Configurables import ExampleFunctionalConsumer -from Configurables import ExampleFunctionalTransformer -from Configurables import ApplicationMgr -from Configurables import k4DataSvc -from Configurables import PodioOutput - -event_data_svc = k4DataSvc("EventDataSvc") - -out = PodioOutput("out") -out.filename = "functional_chain.root" - -producer = ExampleFunctionalProducer("ExampleFunctionalProducer") -consumer = ExampleFunctionalConsumer( - "ExampleFunctionalConsumer", - InputCollection="MCParticles", -) -transformer = ExampleFunctionalTransformer( - "ExampleFunctionalTransformer", - InputCollection="MCParticles", - OutputCollection="NewMCParticles", -) -new_consumer = ExampleFunctionalConsumer( - "ExampleFunctionalConsumer2", - InputCollection="NewMCParticles", -) -new_consumer.PossibleOffset = 10 - -ApplicationMgr( - TopAlg=[producer, consumer, transformer, new_consumer, out], - EvtSel="NONE", - EvtMax=10, - ExtSvc=[event_data_svc], - OutputLevel=INFO, -) diff --git a/test/k4FWCoreTest/options/runFunctionalMix.py b/test/k4FWCoreTest/options/runFunctionalMix.py index 2752e028..d80bac86 100644 --- a/test/k4FWCoreTest/options/runFunctionalMix.py +++ b/test/k4FWCoreTest/options/runFunctionalMix.py @@ -20,74 +20,129 @@ # This is an example mixing functional and non-functional algorithms # -from Gaudi.Configuration import INFO -from Configurables import ExampleFunctionalConsumerMultiple, ExampleFunctionalTransformerMultiple -from Configurables import ExampleFunctionalProducerMultiple, k4FWCoreTest_CreateExampleEventData +from Gaudi.Configuration import INFO, DEBUG +from Configurables import ( + ExampleFunctionalConsumerMultiple, + ExampleFunctionalTransformerMultiple, +) +from Configurables import ( + ExampleFunctionalProducerMultiple, + k4FWCoreTest_CreateExampleEventData, +) from Configurables import k4FWCoreTest_CheckExampleEventData from Configurables import ApplicationMgr from Configurables import k4DataSvc from Configurables import PodioInput, PodioOutput +from k4FWCore.parseArgs import parser + +parser.add_argument( + "--iosvc", + help="Use the IOSvc instead of PodioInput and PodioOutput", + action="store_true", + default=False, +) +args = parser.parse_known_args()[0] + +print(args.iosvc) + +if not args.iosvc: + podioevent = k4DataSvc("EventDataSvc") + podioevent.input = "output_k4test_exampledata_producer_multiple.root" + + inp = PodioInput() + inp.collections = [ + "VectorFloat", + "MCParticles1", + "MCParticles2", + "SimTrackerHits", + "TrackerHits", + "Tracks", + ] + + out = PodioOutput() + out.filename = "output_k4test_exampledata_functional_mix.root" + out.outputCommands = ["keep *"] + +else: + from k4FWCore import IOSvc, ApplicationMgr -podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer_multiple.root" + iosvc = IOSvc("IOSvc") + iosvc.input = "output_k4test_exampledata_producer_multiple.root" + iosvc.output = "output_k4test_exampledata_functional_mix_iosvc.root" -inp = PodioInput() -inp.collections = [ - "VectorFloat", - "MCParticles1", - "MCParticles2", - "SimTrackerHits", - "TrackerHits", - "Tracks", -] +# Check input with functional and old algorithms -consumer_input_functional = ExampleFunctionalConsumerMultiple("ExampleFunctionalConsumerMultiple") +consumer_input_functional = ExampleFunctionalConsumerMultiple( + "ExampleFunctionalConsumerMultiple", + Offset=0, +) consumer_input_algorithm = k4FWCoreTest_CheckExampleEventData("CheckExampleEventData") consumer_input_algorithm.mcparticles = "MCParticles1" consumer_input_algorithm.keepEventNumberZero = True -# We only care about the new FunctionalMCParticles collection in this example +############################### + producer_functional = ExampleFunctionalProducerMultiple( "ProducerFunctional", - OutputCollectionFloat="VectorFloat_", - OutputCollectionParticles1="FunctionalMCParticles", - OutputCollectionParticles2="MCParticles2_", - OutputCollectionSimTrackerHits="SimTrackerHits_", - OutputCollectionTrackerHits="TrackerHits_", - OutputCollectionTracks="Tracks_", + OutputCollectionFloat=["FunctionalVectorFloat"], + OutputCollectionParticles1=["FunctionalMCParticles"], + OutputCollectionParticles2=["FunctionalMCParticles2"], + OutputCollectionSimTrackerHits=["FunctionalSimTrackerHits"], + OutputCollectionTrackerHits=["FunctionalTrackerHits"], + OutputCollectionTracks=["FunctionalTracks"], ExampleInt=5, ) +# Check the functional-produced collections with functional and old algorithms + +# Here we check the new FunctionalMCParticles and the others that are +# read from the file consumer_producerfun_functional = ExampleFunctionalConsumerMultiple( - "FunctionalConsumerFunctional", - InputCollectionParticles="FunctionalMCParticles", + "FunctionalConsumerFromFunctional", + InputCollectionParticles=["FunctionalMCParticles"], + Offset=0, ) consumer_producerfun_algorithm = k4FWCoreTest_CheckExampleEventData("CheckFunctional") consumer_producerfun_algorithm.mcparticles = "FunctionalMCParticles" consumer_producerfun_algorithm.keepEventNumberZero = True +############################### + producer_algorithm = k4FWCoreTest_CreateExampleEventData("CreateExampleEventData") # We only care about the MCParticles collection -producer_algorithm.mcparticles = "AlgorithmMCParticles" -producer_algorithm.simtrackhits = "SimTrackerHits__" -producer_algorithm.trackhits = "TrackerHits__" -producer_algorithm.tracks = "Tracks__" -producer_algorithm.vectorfloat = "VectorFloat__" +producer_algorithm.mcparticles = "OldAlgorithmMCParticles" +producer_algorithm.simtrackhits = "OldAlgorithmSimTrackerHits" +producer_algorithm.trackhits = "OldAlgorithmTrackerHits" +producer_algorithm.tracks = "OldAlgorithmTracks" +producer_algorithm.vectorfloat = "OldAlgorithmVectorFloat" -consumer_produceralg_functional = ExampleFunctionalConsumerMultiple("FunctionalConsumerAlgorithm") +# Check the functional-produced collections with functional and old algorithms + +consumer_produceralg_functional = ExampleFunctionalConsumerMultiple( + "FunctionalConsumerFromAlgorithm", + InputCollectionParticles=["OldAlgorithmMCParticles"], + Offset=0, +) consumer_produceralg_algorithm = k4FWCoreTest_CheckExampleEventData("CheckAlgorithm") -consumer_produceralg_algorithm.mcparticles = "FunctionalMCParticles" -consumer_produceralg_algorithm.keepEventNumberZero = True +consumer_produceralg_algorithm.mcparticles = "OldAlgorithmMCParticles" -# Let's also run the transformer, why not -transformer_functional = ExampleFunctionalTransformerMultiple("FunctionalTransformerMultiple") +############################### + +# Let's also run the transformer on collections that are either read, produced by a functional or an algorithm +transformer_functional = ExampleFunctionalTransformerMultiple( + "FunctionalTransformerMultiple", + InputCollectionFloat=["VectorFloat"], + InputCollectionParticles=["FunctionalMCParticles"], + InputCollectionSimTrackerHits=["OldAlgorithmSimTrackerHits"], + InputCollectionTrackerHits=["TrackerHits"], + OutputCollectionCounter=["Counter"], + OutputCollectionParticles=["TransformedFunctionalMCParticles1"], +) -out = PodioOutput("out") -out.filename = "output_k4test_exampledata_functional_mix.root" ApplicationMgr( - TopAlg=[ - inp, + TopAlg=([inp] if not args.iosvc else []) + + [ # Check we can read input consumer_input_functional, consumer_input_algorithm, @@ -100,10 +155,10 @@ consumer_produceralg_functional, consumer_produceralg_algorithm, transformer_functional, - out, - ], + ] + + ([out] if not args.iosvc else []), EvtSel="NONE", EvtMax=10, - ExtSvc=[podioevent], - OutputLevel=INFO, + ExtSvc=[iosvc if args.iosvc else podioevent], + OutputLevel=DEBUG, ) diff --git a/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py b/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py index 82c3e576..09e24533 100644 --- a/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py +++ b/test/k4FWCoreTest/scripts/check_KeepDropSwitch.py @@ -17,10 +17,11 @@ # limitations under the License. # import ROOT +import sys ROOT.gSystem.Load("libedm4hepDict") -file = ROOT.TFile.Open("output_k4test_exampledata_2.root") +file = ROOT.TFile.Open(sys.argv[1]) tree = file.Get("events") tree.GetEntry(0) @@ -29,5 +30,5 @@ raise Exception("podio::CollectionBase read from file did not saved properly") status = tree.GetBranchStatus("MCParticles") -if status == True: +if status: raise Exception("KeepDropSwitch did not drop the collection") diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp index 7f1f6d70..0915fa38 100644 --- a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp @@ -20,7 +20,7 @@ #include "edm4hep/Constants.h" #include "edm4hep/EventHeaderCollection.h" -#include "k4FWCore/BaseClass.h" +#include "k4FWCore/Consumer.h" #include #include @@ -33,10 +33,9 @@ #include #include -struct ExampleEventHeaderConsumer final - : Gaudi::Functional::Consumer { +struct ExampleEventHeaderConsumer final : k4FWCore::Consumer { ExampleEventHeaderConsumer(const std::string& name, ISvcLocator* svcLoc) - : Consumer(name, svcLoc, {KeyValue("EventHeaderName", edm4hep::EventHeaderName)}) {} + : Consumer(name, svcLoc, {KeyValues("EventHeaderName", {edm4hep::EventHeaderName})}) {} void operator()(const edm4hep::EventHeaderCollection& evtHeaderColl) const { if (evtHeaderColl.empty()) { diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp index 1f5e9458..a81fb702 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumer.cpp @@ -18,47 +18,47 @@ */ #include "Gaudi/Property.h" -#include "GaudiAlg/Consumer.h" #include "edm4hep/MCParticleCollection.h" -// Define BaseClass_t -#include "k4FWCore/BaseClass.h" +#include "k4FWCore/Consumer.h" #include #include -struct ExampleFunctionalConsumer final - : Gaudi::Functional::Consumer { +struct ExampleFunctionalConsumer final : k4FWCore::Consumer { // The pair in KeyValue can be changed from python and it corresponds // to the name of the input collection ExampleFunctionalConsumer(const std::string& name, ISvcLocator* svcLoc) - : Consumer(name, svcLoc, KeyValue("InputCollection", "MCParticles")) {} + : Consumer(name, svcLoc, KeyValues("InputCollection", {"MCParticles"})) {} // This is the function that will be called to transform the data // Note that the function has to be const, as well as the collections // we get from the input void operator()(const edm4hep::MCParticleCollection& input) const override { + debug() << "Received MCParticle collection with " << input.size() << " elements" << endmsg; + if (input.size() != 2) { + fatal() << "Wrong size of MCParticle collection, expected 2 got " << input.size() << endmsg; + throw std::runtime_error("Wrong size of MCParticle collection"); + } int i = 0; for (const auto& particle : input) { - if ((particle.getPDG() != 1 + i + m_possibleOffset) || - (particle.getGeneratorStatus() != 2 + i + m_possibleOffset) || - (particle.getSimulatorStatus() != 3 + i + m_possibleOffset) || - (particle.getCharge() != 4 + i + m_possibleOffset) || (particle.getTime() != 5 + i + m_possibleOffset) || - (particle.getMass() != 6 + i + m_possibleOffset)) { + if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) || + (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) || + (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) { std::stringstream error; - error << "Wrong data in MCParticle collection, expected " << 1 + i + m_possibleOffset << ", " - << 2 + i + m_possibleOffset << ", " << 3 + i + m_possibleOffset << ", " << 4 + i + m_possibleOffset - << ", " << 5 + i + m_possibleOffset << ", " << 6 + i + m_possibleOffset << " got " << particle.getPDG() - << ", " << particle.getGeneratorStatus() << ", " << particle.getSimulatorStatus() << ", " - << particle.getCharge() << ", " << particle.getTime() << ", " << particle.getMass() << ""; + error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset + << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", " + << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", " + << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", " + << particle.getMass() << ""; throw std::runtime_error(error.str()); } i++; } } - Gaudi::Property m_possibleOffset{this, "PossibleOffset", 0, "Possible offset in the values data"}; + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; }; DECLARE_COMPONENT(ExampleFunctionalConsumer) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp index 440d8f85..d21efb94 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerMultiple.cpp @@ -18,23 +18,14 @@ */ #include "Gaudi/Property.h" -#include "GaudiAlg/Consumer.h" #include "edm4hep/MCParticleCollection.h" #include "edm4hep/SimTrackerHitCollection.h" #include "edm4hep/TrackCollection.h" -#if __has_include("edm4hep/TrackerHit3DCollection.h") #include "edm4hep/TrackerHit3DCollection.h" -#else -#include "edm4hep/TrackerHitCollection.h" -namespace edm4hep { - using TrackerHit3DCollection = edm4hep::TrackerHitCollection; -} // namespace edm4hep -#endif #include "podio/UserDataCollection.h" -// Define BaseClass_t -#include "k4FWCore/BaseClass.h" +#include "k4FWCore/Consumer.h" #include #include @@ -48,19 +39,18 @@ using TrackerHitColl = edm4hep::TrackerHit3DCollection; using TrackColl = edm4hep::TrackCollection; struct ExampleFunctionalConsumerMultiple final - : Gaudi::Functional::Consumer { + : k4FWCore::Consumer { // The pairs in KeyValue can be changed from python and they correspond // to the names of the input collection ExampleFunctionalConsumerMultiple(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc, { - KeyValue("InputCollectionFloat", "VectorFloat"), - KeyValue("InputCollectionParticles", "MCParticles1"), - KeyValue("InputCollectionSimTrackerHits", "SimTrackerHits"), - KeyValue("InputCollectionTrackerHits", "TrackerHits"), - KeyValue("InputCollectionTracks", "Tracks"), + KeyValues("InputCollectionFloat", {"VectorFloat"}), + KeyValues("InputCollectionParticles", {"MCParticles1"}), + KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}), + KeyValues("InputCollectionTrackerHits", {"TrackerHits"}), + KeyValues("InputCollectionTracks", {"Tracks"}), }) {} // This is the function that will be called to transform the data @@ -79,14 +69,20 @@ struct ExampleFunctionalConsumerMultiple final throw std::runtime_error(error.str()); } - auto p4 = particles.momentum()[0]; - if ((p4.x != m_magicNumberOffset + 5) || (p4.y != m_magicNumberOffset + 6) || (p4.z != m_magicNumberOffset + 7) || - (particles[0].getMass() != m_magicNumberOffset + 8)) { - std::stringstream error; - error << "Wrong data in particles collection, expected " << m_magicNumberOffset + 5 << ", " - << m_magicNumberOffset + 6 << ", " << m_magicNumberOffset + 7 << ", " << m_magicNumberOffset + 8 << " got " - << p4.x << ", " << p4.y << ", " << p4.z << ", " << particles[0].getMass() << ""; - throw std::runtime_error(error.str()); + int i = 0; + for (const auto& particle : particles) { + if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) || + (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) || + (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) { + std::stringstream error; + error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset + << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", " + << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", " + << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", " + << particle.getMass() << ""; + throw std::runtime_error(error.str()); + } + i++; } if ((simTrackerHits[0].getPosition()[0] != 3) || (simTrackerHits[0].getPosition()[1] != 4) || @@ -117,9 +113,7 @@ struct ExampleFunctionalConsumerMultiple final } private: - // integer to add to the dummy values written to the edm - Gaudi::Property m_magicNumberOffset{this, "magicNumberOffset", 0, - "Integer to add to the dummy values written to the edm"}; + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; }; DECLARE_COMPONENT(ExampleFunctionalConsumerMultiple) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp new file mode 100644 index 00000000..dd016eab --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollections.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Property.h" + +#include "edm4hep/MCParticleCollection.h" + +#include "k4FWCore/Consumer.h" + +#include + +struct ExampleFunctionalConsumerRuntimeCollections final + : k4FWCore::Consumer& input)> { + // The pair in KeyValue can be changed from python and it corresponds + // to the name of the output collection + ExampleFunctionalConsumerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, KeyValues("InputCollection", {"DefaultValue"})) {} + + // This is the function that will be called to produce the data + void operator()(const std::map& input) const override { + if (input.size() != 3) { + throw std::runtime_error("Wrong size of the input map, expected 3, got " + std::to_string(input.size())); + } + for (auto& [key, val] : input) { + int i = 0; + for (const auto& particle : val) { + if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) || + (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) || + (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) { + std::stringstream error; + error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset + << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", " + << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", " + << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", " + << particle.getMass() << ""; + throw std::runtime_error(error.str()); + } + i++; + } + } + } + +private: + // We can define any property we want that can be set from python + // and use it inside operator() + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalConsumerRuntimeCollections) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp new file mode 100644 index 00000000..5b2ca067 --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalConsumerRuntimeCollectionsMultiple.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Property.h" + +#include "edm4hep/MCParticleCollection.h" +#include "edm4hep/SimTrackerHitCollection.h" +#include "edm4hep/TrackCollection.h" + +#include "k4FWCore/Consumer.h" + +#include + +struct ExampleFunctionalConsumerRuntimeCollectionsMultiple final + : k4FWCore::Consumer& particleMap, + const std::map& trackMap, + const edm4hep::SimTrackerHitCollection& simTrackerHits)> { + // The pair in KeyValue can be changed from python and it corresponds + // to the name of the output collection + ExampleFunctionalConsumerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, + {KeyValues("Particles", {"MCParticles"}), KeyValues("Tracks", {"MCParticles"}), + KeyValues("SimTrackerHits", {"MCParticles"})}) {} + // : Consumer(name, svcLoc, {KeyValue("Particles", ("MCParticles")), KeyValue("Tracks", ("MCParticles")), KeyValue("SimTrackerHits", ("MCParticles"))}) {} + + // This is the function that will be called to produce the data + void operator()(const std::map& particleMap, + const std::map& trackMap, + const edm4hep::SimTrackerHitCollection& simTrackerHits) const override { + info() << "Received " << particleMap.size() << " particle collections and " << trackMap.size() + << " track collections" << endmsg; + if (particleMap.size() != 5) { + throw std::runtime_error("Wrong size of the particleMap map, expected 5, got " + + std::to_string(particleMap.size())); + } + for (auto& [key, particles] : particleMap) { + int i = 0; + for (const auto& particle : particles) { + if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) || + (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) || + (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) { + std::stringstream error; + error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset + << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", " + << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", " + << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", " + << particle.getMass() << ""; + throw std::runtime_error(error.str()); + } + i++; + } + } + if (trackMap.size() != 3) { + fatal() << "Wrong size of the tracks map, expected 3, got " << trackMap.size() << endmsg; + } + for (auto& [key, tracks] : trackMap) { + if ((tracks[0].getType() != 1) || (std::abs(tracks[0].getChi2() - 2.1) > 1e-6) || (tracks[0].getNdf() != 3) || + (std::abs(tracks[0].getDEdx() - 4.1) > 1e-6) || (std::abs(tracks[0].getDEdxError() - 5.1) > 1e-6) || + (std::abs(tracks[0].getRadiusOfInnermostHit() - 6.1) > 1e-6)) { + std::stringstream error; + error << "Wrong data in tracks collection, expected 1, 2.1, 3, 4.1, 5.1, 6.1 got " << tracks[0].getType() + << ", " << tracks[0].getChi2() << ", " << tracks[0].getNdf() << ", " << tracks[0].getDEdx() << ", " + << tracks[0].getDEdxError() << ", " << tracks[0].getRadiusOfInnermostHit() << ""; + throw std::runtime_error(error.str()); + } + } + } + +private: + // We can define any property we want that can be set from python + // and use it inside operator() + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalConsumerRuntimeCollectionsMultiple) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp index fbd319b3..bda86666 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducer.cpp @@ -18,18 +18,18 @@ */ #include "Gaudi/Property.h" -#include "GaudiAlg/Producer.h" -#include "k4FWCore/BaseClass.h" #include "edm4hep/MCParticleCollection.h" +#include "k4FWCore/Producer.h" + #include -struct ExampleFunctionalProducer final : Gaudi::Functional::Producer { +struct ExampleFunctionalProducer final : k4FWCore::Producer { // The pair in KeyValue can be changed from python and it corresponds // to the name of the output collection ExampleFunctionalProducer(const std::string& name, ISvcLocator* svcLoc) - : Producer(name, svcLoc, KeyValue("OutputCollection", "MCParticles")) {} + : Producer(name, svcLoc, {}, KeyValues("OutputCollection", {"MCParticles"})) {} // This is the function that will be called to produce the data edm4hep::MCParticleCollection operator()() const override { diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp index 70659cce..40ef1331 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerMultiple.cpp @@ -18,8 +18,8 @@ */ #include "Gaudi/Property.h" -#include "GaudiAlg/Producer.h" -#include "k4FWCore/BaseClass.h" + +#include "k4FWCore/Producer.h" #include "edm4hep/MCParticleCollection.h" #include "edm4hep/SimTrackerHitCollection.h" @@ -37,27 +37,25 @@ namespace edm4hep { #include // Which type of collections we are producing -using Float = podio::UserDataCollection; -using Particle = edm4hep::MCParticleCollection; -using SimTrackerHit = edm4hep::SimTrackerHitCollection; -using TrackerHit = edm4hep::TrackerHit3DCollection; -using Track = edm4hep::TrackCollection; - -struct ExampleFunctionalProducerMultiple final - : Gaudi::Functional::Producer(), - BaseClass_t> { + +using retType = + std::tuple, edm4hep::MCParticleCollection, edm4hep::MCParticleCollection, + edm4hep::SimTrackerHitCollection, edm4hep::TrackerHit3DCollection, edm4hep::TrackCollection>; + +struct ExampleFunctionalProducerMultiple final : k4FWCore::Producer { // The pairs in KeyValue can be changed from python and they correspond // to the names of the output collections ExampleFunctionalProducerMultiple(const std::string& name, ISvcLocator* svcLoc) - : Producer( - name, svcLoc, - {KeyValue("OutputCollectionFloat", "VectorFloat"), KeyValue("OutputCollectionParticles1", "MCParticles1"), - KeyValue("OutputCollectionParticles2", "MCParticles2"), - KeyValue("OutputCollectionSimTrackerHits", "SimTrackerHits"), - KeyValue("OutputCollectionTrackerHits", "TrackerHits"), KeyValue("OutputCollectionTracks", "Tracks")}) {} + : Producer(name, svcLoc, {}, + {KeyValues("OutputCollectionFloat", {"VectorFloat"}), + KeyValues("OutputCollectionParticles1", {"MCParticles1"}), + KeyValues("OutputCollectionParticles2", {"MCParticles2"}), + KeyValues("OutputCollectionSimTrackerHits", {"SimTrackerHits"}), + KeyValues("OutputCollectionTrackerHits", {"TrackerHits"}), + KeyValues("OutputCollectionTracks", {"Tracks"})}) {} // This is the function that will be called to produce the data - std::tuple operator()() const override { + retType operator()() const override { // The following was copied and adapted from the // k4FWCoreTest_CreateExampleEventData test @@ -66,10 +64,10 @@ struct ExampleFunctionalProducerMultiple final floatVector.push_back(25.); floatVector.push_back(m_event); - auto particles = edm4hep::MCParticleCollection(); - auto particle = particles.create(); - particle.setMomentum({m_magicNumberOffset + m_event + 5.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0}); - particle.setMass(m_magicNumberOffset + m_event + 8); + auto particles = edm4hep::MCParticleCollection(); + edm4hep::Vector3d v{0, 0, 0}; + particles.create(1, 2, 3, 4.f, 5.f, 6.f, v, v, v); + particles.create(2, 3, 4, 5.f, 6.f, 7.f); auto simTrackerHits = edm4hep::SimTrackerHitCollection(); auto hit = simTrackerHits.create(); @@ -96,8 +94,8 @@ struct ExampleFunctionalProducerMultiple final track.addToTrackerHits(trackerHit); track.addToTracks(track2); - return std::make_tuple(std::move(floatVector), std::move(particles), Particle(), std::move(simTrackerHits), - std::move(trackerHits), std::move(tracks)); + return std::make_tuple(std::move(floatVector), std::move(particles), edm4hep::MCParticleCollection(), + std::move(simTrackerHits), std::move(trackerHits), std::move(tracks)); } private: diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp new file mode 100644 index 00000000..7579844e --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalProducerRuntimeCollections.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Property.h" + +#include "edm4hep/MCParticleCollection.h" + +#include "k4FWCore/Producer.h" + +#include + +struct ExampleFunctionalProducerRuntimeCollections final + : k4FWCore::Producer()> { + // The pair in KeyValue can be changed from python and it corresponds + // to the name of the output collection + ExampleFunctionalProducerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc) + : Producer(name, svcLoc, {}, {KeyValues("OutputCollections", {"MCParticles"})}) {} + + // This is the function that will be called to produce the data + std::map operator()() const override { + std::map m_outputCollections; + for (int i = 0; i < m_numberOfCollections; ++i) { + std::string name = "MCParticles" + std::to_string(i); + auto coll = edm4hep::MCParticleCollection(); + coll->create(1, 2, 3, 4.f, 5.f, 6.f); + coll->create(2, 3, 4, 5.f, 6.f, 7.f); + m_outputCollections[name] = std::move(coll); + } + return m_outputCollections; + } + +private: + // We can define any property we want that can be set from python + // and use it inside operator() + Gaudi::Property m_numberOfCollections{this, "NumberOfCollections", 3, + "Example int that can be used in the algorithm"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalProducerRuntimeCollections) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp index 0abe3c31..54946940 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformer.cpp @@ -18,43 +18,42 @@ */ #include "Gaudi/Property.h" -#include "GaudiAlg/Transformer.h" #include "edm4hep/MCParticleCollection.h" #include "edm4hep/MutableMCParticle.h" -// Define BaseClass_t -#include "k4FWCore/BaseClass.h" +#include "k4FWCore/Transformer.h" #include -// Which type of collection we are reading and writing -using colltype_in = edm4hep::MCParticleCollection; -using colltype_out = edm4hep::MCParticleCollection; - struct ExampleFunctionalTransformer final - : Gaudi::Functional::Transformer { + : k4FWCore::Transformer { ExampleFunctionalTransformer(const std::string& name, ISvcLocator* svcLoc) - : Transformer(name, svcLoc, KeyValue("InputCollection", "MCParticles"), - KeyValue("OutputCollection", "NewMCParticles")) {} + : Transformer(name, svcLoc, {KeyValues("InputCollection", {"MCParticles"})}, + {KeyValues("OutputCollection", {"NewMCParticles"})}) {} // This is the function that will be called to transform the data // Note that the function has to be const, as well as all pointers to collections // we get from the input - colltype_out operator()(const colltype_in& input) const override { + edm4hep::MCParticleCollection operator()(const edm4hep::MCParticleCollection& input) const override { + info() << "Transforming " << input.size() << " particles" << endmsg; auto coll_out = edm4hep::MCParticleCollection(); for (const auto& particle : input) { auto new_particle = edm4hep::MutableMCParticle(); - new_particle.setPDG(particle.getPDG() + 10); - new_particle.setGeneratorStatus(particle.getGeneratorStatus() + 10); - new_particle.setSimulatorStatus(particle.getSimulatorStatus() + 10); - new_particle.setCharge(particle.getCharge() + 10); - new_particle.setTime(particle.getTime() + 10); - new_particle.setMass(particle.getMass() + 10); + new_particle.setPDG(particle.getPDG() + m_offset); + new_particle.setGeneratorStatus(particle.getGeneratorStatus() + m_offset); + new_particle.setSimulatorStatus(particle.getSimulatorStatus() + m_offset); + new_particle.setCharge(particle.getCharge() + m_offset); + new_particle.setTime(particle.getTime() + m_offset); + new_particle.setMass(particle.getMass() + m_offset); coll_out->push_back(new_particle); } return coll_out; } + +private: + // integer to add to the dummy values written to the edm + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; }; DECLARE_COMPONENT(ExampleFunctionalTransformer) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp index e84587fa..a27d2b71 100644 --- a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerMultiple.cpp @@ -18,7 +18,6 @@ */ #include "Gaudi/Property.h" -#include "GaudiAlg/Transformer.h" #include "edm4hep/MCParticleCollection.h" #include "edm4hep/SimTrackerHitCollection.h" @@ -34,8 +33,7 @@ namespace edm4hep { #include "edm4hep/TrackerHit3DCollection.h" #include "podio/UserDataCollection.h" -// Define BaseClass_t -#include "k4FWCore/BaseClass.h" +#include "k4FWCore/Transformer.h" #include @@ -51,18 +49,17 @@ using Counter = podio::UserDataCollection; using Particle = edm4hep::MCParticleCollection; struct ExampleFunctionalTransformerMultiple final - : Gaudi::Functional::MultiTransformer(const FloatColl&, const ParticleColl&, - const SimTrackerHitColl&, const TrackerHitColl&, - const TrackColl&), - BaseClass_t> { + : k4FWCore::MultiTransformer( + const FloatColl&, const ParticleColl&, const SimTrackerHitColl&, const TrackerHitColl&, const TrackColl&)> { ExampleFunctionalTransformerMultiple(const std::string& name, ISvcLocator* svcLoc) : MultiTransformer( name, svcLoc, - {KeyValue("InputCollectionFloat", "VectorFloat"), KeyValue("InputCollectionParticles", "MCParticles1"), - KeyValue("InputCollectionSimTrackerHits", "SimTrackerHits"), - KeyValue("InputCollectionTrackerHits", "TrackerHits"), KeyValue("InputCollectionTracks", "Tracks")}, - {KeyValue("OutputCollectionCounter", "Counter"), KeyValue("OutputCollectionParticles", "NewMCParticles")}) { - } + {KeyValues("InputCollectionFloat", {"VectorFloat"}), + KeyValues("InputCollectionParticles", {"MCParticles1"}), + KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}), + KeyValues("InputCollectionTrackerHits", {"TrackerHits"}), KeyValues("InputCollectionTracks", {"Tracks"})}, + {KeyValues("OutputCollectionCounter", {"Counter"}), + KeyValues("OutputCollectionParticles", {"NewMCParticles"})}) {} // This is the function that will be called to transform the data // Note that the function has to be const, as well as the collections @@ -80,23 +77,22 @@ struct ExampleFunctionalTransformerMultiple final // We could create a new one auto newParticle = newParticlesColl->create(); - newParticle.setPDG(p.getPDG()); - newParticle.setGeneratorStatus(p.getGeneratorStatus() + 1); - newParticle.setSimulatorStatus(p.getSimulatorStatus() + 1); - newParticle.setCharge(p.getCharge() + 2); - newParticle.setTime(p.getTime() + 3); - newParticle.setMass(p.getMass() + 4); + newParticle.setPDG(p.getPDG() + m_offset); + newParticle.setGeneratorStatus(p.getGeneratorStatus() + m_offset); + newParticle.setSimulatorStatus(p.getSimulatorStatus() + m_offset); + newParticle.setCharge(p.getCharge() + m_offset); + newParticle.setTime(p.getTime() + m_offset); + newParticle.setMass(p.getMass() + m_offset); } counter.push_back(particles.size()); - counter.push_back(simTrackerHits.size()); - counter.push_back(trackerHits.size()); - counter.push_back(tracks.size()); return std::make_tuple(std::move(counter), std::move(newParticlesColl)); } + + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; }; DECLARE_COMPONENT(ExampleFunctionalTransformerMultiple) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp new file mode 100644 index 00000000..23e07962 --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollections.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Property.h" + +#include "edm4hep/MCParticleCollection.h" + +#include "k4FWCore/Transformer.h" + +#include + +/* ExampleFunctionalTransformerRuntimeCollections + * + * This is an example of a functional transformer that takes an arbitrary number + * of input collections and produces an arbitrary number of output collections + * (in this example the same number of input collections but it can be different) + */ + +using mapType = std::map; + +struct ExampleFunctionalTransformerRuntimeCollections final + : k4FWCore::Transformer& input)> { + // The pair in KeyValue can be changed from python and it corresponds + // to the name of the output collection + ExampleFunctionalTransformerRuntimeCollections(const std::string& name, ISvcLocator* svcLoc) + : Transformer(name, svcLoc, {KeyValues("InputCollections", {"MCParticles"})}, + {KeyValues("OutputCollections", {"MCParticles"})}) {} + + // This is the function that will be called to produce the data + mapType operator()(const std::map& input) const override { + std::map outputCollections; + for (int i = 0; i < input.size(); ++i) { + std::string name = "NewMCParticles" + std::to_string(i); + auto& old_coll = input.at("MCParticles" + std::to_string(i)); + auto coll = edm4hep::MCParticleCollection(); + coll->push_back(old_coll.at(0).clone()); + coll->push_back(old_coll.at(1).clone()); + outputCollections[name] = std::move(coll); + } + return outputCollections; + } + +private: + // We can define any property we want that can be set from python + // and use it inside operator() + Gaudi::Property m_numberOfCollections{this, "NumberOfCollections", 3, + "Example int that can be used in the algorithm"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeCollections) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp new file mode 100644 index 00000000..0cf79e30 --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeCollectionsMultiple.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Property.h" + +#include "edm4hep/MCParticleCollection.h" +#include "edm4hep/SimTrackerHitCollection.h" +#include "edm4hep/TrackCollection.h" +#include "edm4hep/TrackerHit3DCollection.h" +#include "podio/UserDataCollection.h" + +#include "k4FWCore/Transformer.h" + +#include +#include + +// Which type of collection we are reading +using FloatColl = std::map&>; +using ParticleColl = std::map; +using SimTrackerHitColl = std::map; +using TrackerHitColl = std::map; +using TrackColl = std::map; + +using retType = std::tuple< + std::map>, std::map, + std::map, std::map, + std::map, std::map>; + +struct ExampleFunctionalTransformerRuntimeCollectionsMultiple final + : k4FWCore::MultiTransformer { + // The pairs in KeyValue can be changed from python and they correspond + // to the names of the input collection + ExampleFunctionalTransformerRuntimeCollectionsMultiple(const std::string& name, ISvcLocator* svcLoc) + : MultiTransformer(name, svcLoc, + { + KeyValues("InputCollectionFloat", {"VectorFloat"}), + KeyValues("InputCollectionParticles", {"MCParticles1"}), + KeyValues("InputCollectionSimTrackerHits", {"SimTrackerHits"}), + KeyValues("InputCollectionTrackerHits", {"TrackerHits"}), + KeyValues("InputCollectionTracks", {"Tracks"}), + }, + { + KeyValues("OutputCollectionFloat", {"VectorFloat"}), + KeyValues("OutputCollectionParticles1", {"MCParticles1"}), + KeyValues("OutputCollectionParticles2", {"MCParticles2"}), + KeyValues("OutputCollectionSimTrackerHits", {"SimTrackerHits"}), + KeyValues("OutputCollectionTrackerHits", {"TrackerHits"}), + KeyValues("OutputCollectionTracks", {"Tracks"}), + }) {} + + // This is the function that will be called to transform the data + // Note that the function has to be const, as well as the collections + // we get from the input + retType operator()(const FloatColl& floatMap, const ParticleColl& particlesMap, + const SimTrackerHitColl& simTrackerHitMap, const TrackerHitColl& trackerHitMap, + const TrackColl& trackMap) const override { + auto floatMapOut = std::map>(); + auto particleMapOut = std::map(); + auto particle2MapOut = std::map(); + auto simTrackerHitMapOut = std::map(); + auto trackerHitMapOut = std::map(); + auto trackMapOut = std::map(); + + if (floatMap.size() != 3) { + throw std::runtime_error("Wrong size of the floatVector collection map, expected 3, got " + + std::to_string(floatMap.size()) + ""); + } + for (const auto& [key, floatVector] : floatMap) { + if (floatVector.size() != 3) { + throw std::runtime_error("Wrong size of floatVector collection, expected 3, got " + + std::to_string(floatVector.size()) + ""); + } + if ((floatVector.vec()[0] != 125) || (floatVector.vec()[1] != 25) || (floatVector.vec()[2] != 0)) { + std::stringstream error; + error << "Wrong data in floatVector collection, expected 125, 25, " << 0 << " got " << floatVector.vec()[0] + << ", " << floatVector.vec()[1] << ", " << floatVector.vec()[2] << ""; + throw std::runtime_error(error.str()); + } + auto coll = podio::UserDataCollection(); + coll.push_back(floatVector.vec()[0]); + coll.push_back(floatVector.vec()[1]); + coll.push_back(floatVector.vec()[2]); + floatMapOut["New" + key] = std::move(coll); + } + + if (particlesMap.size() != 3) { + throw std::runtime_error("Wrong size of the particleMap map, expected 3, got " + + std::to_string(particleMapOut.size()) + ""); + } + + for (auto& [key, particles] : particlesMap) { + auto coll = edm4hep::MCParticleCollection(); + int i = 0; + for (const auto& particle : particles) { + if ((particle.getPDG() != 1 + i + m_offset) || (particle.getGeneratorStatus() != 2 + i + m_offset) || + (particle.getSimulatorStatus() != 3 + i + m_offset) || (particle.getCharge() != 4 + i + m_offset) || + (particle.getTime() != 5 + i + m_offset) || (particle.getMass() != 6 + i + m_offset)) { + std::stringstream error; + error << "Wrong data in MCParticle collection, expected " << 1 + i + m_offset << ", " << 2 + i + m_offset + << ", " << 3 + i + m_offset << ", " << 4 + i + m_offset << ", " << 5 + i + m_offset << ", " + << 6 + i + m_offset << " got " << particle.getPDG() << ", " << particle.getGeneratorStatus() << ", " + << particle.getSimulatorStatus() << ", " << particle.getCharge() << ", " << particle.getTime() << ", " + << particle.getMass() << ""; + throw std::runtime_error(error.str()); + coll.push_back(particle.clone()); + } + i++; + particleMapOut["New" + key] = std::move(coll); + } + } + + if (simTrackerHitMap.size() != 3) { + throw std::runtime_error("Wrong size of the simTrackerHitMap map, expected 3, got " + + std::to_string(simTrackerHitMapOut.size()) + ""); + } + + for (auto& [key, simTrackerHits] : simTrackerHitMap) { + auto coll = edm4hep::SimTrackerHitCollection(); + if ((simTrackerHits.at(0).getPosition()[0] != 3) || (simTrackerHits.at(0).getPosition()[1] != 4) || + (simTrackerHits.at(0).getPosition()[2] != 5)) { + std::stringstream error; + error << "Wrong data in simTrackerHits collection, expected 3, 4, 5 got " + << simTrackerHits.at(0).getPosition()[0] << ", " << simTrackerHits.at(0).getPosition()[1] << ", " + << simTrackerHits.at(0).getPosition()[2] << ""; + throw std::runtime_error(error.str()); + } + coll.push_back(simTrackerHits.at(0).clone()); + simTrackerHitMapOut["New" + key] = std::move(coll); + } + + if (trackerHitMap.size() != 3) { + throw std::runtime_error("Wrong size of the trackerHitMap map, expected 3, got " + + std::to_string(trackerHitMapOut.size()) + ""); + } + + for (auto& [key, trackerHits] : trackerHitMap) { + auto coll = edm4hep::TrackerHit3DCollection(); + if ((trackerHits.at(0).getPosition()[0] != 3) || (trackerHits.at(0).getPosition()[1] != 4) || + (trackerHits.at(0).getPosition()[2] != 5)) { + std::stringstream error; + error << "Wrong data in trackerHits collection, expected 3, 4, 5 got " << trackerHits.at(0).getPosition()[0] + << ", " << trackerHits.at(0).getPosition()[1] << ", " << trackerHits.at(0).getPosition()[2] << ""; + throw std::runtime_error(error.str()); + } + coll.push_back(trackerHits.at(0).clone()); + trackerHitMapOut["New" + key] = std::move(coll); + } + + if (trackMap.size() != 3) { + throw std::runtime_error("Wrong size of the trackMap map, expected 3, got " + std::to_string(trackMapOut.size()) + + ""); + } + + for (auto& [key, tracks] : trackMap) { + auto coll = edm4hep::TrackCollection(); + if ((tracks.at(0).getType() != 1) || (std::abs(tracks.at(0).getChi2() - 2.1) > 1e-6) || + (tracks.at(0).getNdf() != 3) || (std::abs(tracks.at(0).getDEdx() - 4.1) > 1e-6) || + (std::abs(tracks.at(0).getDEdxError() - 5.1) > 1e-6) || + (std::abs(tracks.at(0).getRadiusOfInnermostHit() - 6.1) > 1e-6)) { + std::stringstream error; + error << "Wrong data in tracks collection, expected 1, 2.1, 3, 4.1, 5.1, 6.1 got " << tracks.at(0).getType() + << ", " << tracks.at(0).getChi2() << ", " << tracks.at(0).getNdf() << ", " << tracks.at(0).getDEdx() + << ", " << tracks.at(0).getDEdxError() << ", " << tracks.at(0).getRadiusOfInnermostHit() << ""; + throw std::runtime_error(error.str()); + } + coll->push_back(tracks.at(0).clone()); + trackMapOut["New" + key] = std::move(coll); + } + + return std::make_tuple(std::move(floatMapOut), std::move(particleMapOut), std::move(particle2MapOut), + std::move(simTrackerHitMapOut), std::move(trackerHitMapOut), std::move(trackMapOut)); + } + +private: + Gaudi::Property m_offset{this, "Offset", 10, "Integer to add to the dummy values written to the edm"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeCollectionsMultiple) diff --git a/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp new file mode 100644 index 00000000..f5c0ba86 --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleFunctionalTransformerRuntimeEmpty.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Gaudi/Property.h" + +#include "edm4hep/MCParticleCollection.h" + +#include "k4FWCore/Transformer.h" + +#include + +/* ExampleFunctionalTransformerRuntimeEmpty + * + * This is an example of a functional transformer that takes an arbitrary number + * of input collections and produces an arbitrary number of output collections + * but it's returning an empty map (no output collections) + */ + +using mapType = std::map; + +struct ExampleFunctionalTransformerRuntimeEmpty final + : k4FWCore::Transformer(const mapType& input)> { + // The pair in KeyValue can be changed from python and it corresponds + // to the name of the output collection + ExampleFunctionalTransformerRuntimeEmpty(const std::string& name, ISvcLocator* svcLoc) + : Transformer(name, svcLoc, {KeyValues("InputCollections", {"MCParticles"})}, + {KeyValues("OutputCollections", {"MCParticles"})}) {} + + // This is the function that will be called to produce the data + std::map operator()(const mapType& input) const override { + // We just return an empty map to make sure it works fine + std::map outputCollections; + return outputCollections; + } + +private: + // We can define any property we want that can be set from python + // and use it inside operator() + Gaudi::Property m_numberOfCollections{this, "NumberOfCollections", 3, + "Example int that can be used in the algorithm"}; +}; + +DECLARE_COMPONENT(ExampleFunctionalTransformerRuntimeEmpty) diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp index f53615cb..172d51f1 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CheckExampleEventData.cpp @@ -55,12 +55,13 @@ StatusCode k4FWCoreTest_CheckExampleEventData::execute(const EventContext&) cons auto particles = m_mcParticleHandle.get(); auto particle = (*particles)[0]; - if ((particle.getMomentum().x != m_magicNumberOffset + m_event + 5) || - (particle.getMass() != m_magicNumberOffset + m_event + 8)) { - fatal() << "Contents of mcparticles collection is not as expected: momentum.x = " << particle.getMomentum().x - << " (expected " << m_magicNumberOffset + m_event + 5 << "), mass = " << particle.getMass() << " (expected " - << m_magicNumberOffset + m_event + 8 << ")" << endmsg; - // return StatusCode::FAILURE; + if ((particle.getMomentum().x != m_magicNumberOffset + m_event + 0) || + (particle.getMass() != m_magicNumberOffset + 6)) { + std::stringstream error; + error << "Contents of mcparticles collection is not as expected: momentum.x = " << particle.getMomentum().x + << " (expected " << m_magicNumberOffset + m_event + 0 << "), mass = " << particle.getMass() << " (expected " + << m_magicNumberOffset + 6 << ")"; + throw std::runtime_error(error.str()); } if (!m_keepEventNumberZero) { diff --git a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp index 36e0aa84..f52bbf75 100644 --- a/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp +++ b/test/k4FWCoreTest/src/components/k4FWCoreTest_CreateExampleEventData.cpp @@ -64,8 +64,15 @@ StatusCode k4FWCoreTest_CreateExampleEventData::execute(const EventContext&) con edm4hep::MCParticleCollection* particles = m_mcParticleHandle.createAndPut(); auto particle = particles->create(); - particle.setMomentum({m_magicNumberOffset + m_event + 5.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0}); - particle.setMass(m_magicNumberOffset + m_event + 8); + particle.setPDG(1); + particle.setGeneratorStatus(2); + particle.setSimulatorStatus(3); + particle.setCharge(4); + particle.setTime(5); + particle.setMass(m_magicNumberOffset + 6); + particle.setMomentum({m_magicNumberOffset + m_event + 0.0, m_magicNumberOffset + 6.0, m_magicNumberOffset + 7.0}); + + particles->create(2, 3, 4, 5.f, 6.f, 7.f); edm4hep::SimTrackerHitCollection* simTrackerHits = m_simTrackerHitHandle.createAndPut(); auto hit = simTrackerHits->create();