From 3330439b79bb806aeaf9fc085c30627e90d01310 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 16 Nov 2023 20:25:55 +0100 Subject: [PATCH 1/6] Implement minimal consumer to exhibit failure --- test/k4FWCoreTest/CMakeLists.txt | 1 + .../options/runEventHeaderCheck.py | 41 +++++++++++++++++ .../components/ExampleEventHeaderConsumer.cpp | 45 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 test/k4FWCoreTest/options/runEventHeaderCheck.py create mode 100644 test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index 49ad4607..bf806747 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -112,3 +112,4 @@ add_test_with_env(ExampleFunctionalConsumerMultiple options/runExampleFunctional add_test_with_env(ExampleFunctionalTransformerMultiple options/runExampleFunctionalTransformerMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) add_test_with_env(FunctionalChain options/runFunctionalChain.py) add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) +add_test_with_env(EventHeaderCheck options/runEventHeaderCheck.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py new file mode 100644 index 00000000..f4e5fd44 --- /dev/null +++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2014-2023 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from Gaudi.Configuration import INFO +from Configurables import k4DataSvc +from Configurables import PodioInput +from Configurables import ExampleEventHeaderConsumer +from Configurables import ApplicationMgr + +podioevent = k4DataSvc("EventDataSvc") +podioevent.input = "output_k4test_exampledata_producer_multiple.root" + +inp = PodioInput() +inp.collections = [] + +consumer = ExampleEventHeaderConsumer() + +ApplicationMgr( + TopAlg=[inp, consumer], + EvtSel="NONE", + EvtMax=-1, + ExtSvc=[podioevent], + OutputLevel=INFO, +) diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp new file mode 100644 index 00000000..f6e1be78 --- /dev/null +++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2014-2023 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "GaudiAlg/Consumer.h" + +#include "edm4hep/Constants.h" +#include "edm4hep/EventHeaderCollection.h" + +#include "k4FWCore/BaseClass.h" + +#include +#include + +struct ExampleEventHeaderConsumer final + : Gaudi::Functional::Consumer { + ExampleEventHeaderConsumer(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, {KeyValue("EventHeaderName", edm4hep::EventHeaderName)}) {} + + void operator()(const edm4hep::EventHeaderCollection& evtHeaderColl) const { + if (evtHeaderColl.empty()) { + throw std::runtime_error("EventHeader collection is empty"); + } + if (!evtHeaderColl[0].isAvailable()) { + throw std::runtime_error("Cannot get a valid EventHeader"); + } + } +}; + +DECLARE_COMPONENT(ExampleEventHeaderConsumer) From ef501613074eec7bebd0c18fb261ababa9dfce97 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 17 Nov 2023 09:15:09 +0100 Subject: [PATCH 2/6] Check event header produced by EventHeaderFiller --- test/k4FWCoreTest/CMakeLists.txt | 2 +- .../options/runEventHeaderCheck.py | 6 ++-- .../components/ExampleEventHeaderConsumer.cpp | 30 +++++++++++++++++-- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index bf806747..19f40204 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -95,6 +95,7 @@ add_test_with_env(TestDataHandleUniquePtr options/TestDataHandleUniquePtr.py) add_test_with_env(TestUniqueIDGenSvc options/TestUniqueIDGenSvc.py) add_test_with_env(CreateLegacyExampleEventData options/createLegacyExampleEventData.py) add_test_with_env(TestEventHeaderFiller options/createEventHeader.py) +add_test_with_env(EventHeaderCheck options/runEventHeaderCheck.py PROPERTIES DEPENDS TestEventHeaderFiller) add_test(NAME TestExec WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND python options/TestExec.py) set_test_env(TestExec) @@ -112,4 +113,3 @@ add_test_with_env(ExampleFunctionalConsumerMultiple options/runExampleFunctional add_test_with_env(ExampleFunctionalTransformerMultiple options/runExampleFunctionalTransformerMultiple.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) add_test_with_env(FunctionalChain options/runFunctionalChain.py) add_test_with_env(FunctionalMix options/runFunctionalMix.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) -add_test_with_env(EventHeaderCheck options/runEventHeaderCheck.py PROPERTIES DEPENDS ExampleFunctionalProducerMultiple) diff --git a/test/k4FWCoreTest/options/runEventHeaderCheck.py b/test/k4FWCoreTest/options/runEventHeaderCheck.py index f4e5fd44..64f673c7 100644 --- a/test/k4FWCoreTest/options/runEventHeaderCheck.py +++ b/test/k4FWCoreTest/options/runEventHeaderCheck.py @@ -25,12 +25,14 @@ from Configurables import ApplicationMgr podioevent = k4DataSvc("EventDataSvc") -podioevent.input = "output_k4test_exampledata_producer_multiple.root" +podioevent.input = "eventHeader.root" inp = PodioInput() inp.collections = [] -consumer = ExampleEventHeaderConsumer() +consumer = ExampleEventHeaderConsumer( + "EventHeaderCheck", runNumber=42, eventNumberOffset=42 +) ApplicationMgr( TopAlg=[inp, consumer], diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp index f6e1be78..0f18a24d 100644 --- a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp @@ -16,16 +16,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include -#include "GaudiAlg/Consumer.h" #include "edm4hep/Constants.h" #include "edm4hep/EventHeaderCollection.h" #include "k4FWCore/BaseClass.h" +#include +#include +#include + +#include +#include + +#include #include #include +#include struct ExampleEventHeaderConsumer final : Gaudi::Functional::Consumer { @@ -36,10 +43,27 @@ struct ExampleEventHeaderConsumer final if (evtHeaderColl.empty()) { throw std::runtime_error("EventHeader collection is empty"); } - if (!evtHeaderColl[0].isAvailable()) { + const auto evtHeader = evtHeaderColl[0]; + if (!evtHeader.isAvailable()) { throw std::runtime_error("Cannot get a valid EventHeader"); } + + if (evtHeader.getRunNumber() != m_runNumber) { + throw std::runtime_error(fmt::format("Run number is not set correctly (expected {}, actual {})", + m_runNumber.value(), evtHeader.getRunNumber())); + } + + const auto expectedEvent = m_evtCounter++ + m_eventNumberOffset; + if (evtHeader.getEventNumber() != expectedEvent) { + throw std::runtime_error(fmt::format("Event number is not set correctly (expected {}, actual {})", expectedEvent, + evtHeader.getEventNumber())); + } } + + Gaudi::Property m_runNumber{this, "runNumber", 0, "The expected run number"}; + Gaudi::Property m_eventNumberOffset{this, "eventNumberOffset", 0, + "The event number offset where events will start counting from"}; + mutable std::atomic m_evtCounter{0}; }; DECLARE_COMPONENT(ExampleEventHeaderConsumer) From 5f782293d1b3a46aebc89ed5eda56b20f2c163dc Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 17 Nov 2023 09:18:40 +0100 Subject: [PATCH 3/6] Check and read the EventHeader every event --- k4FWCore/components/PodioInput.cpp | 11 ++++++----- k4FWCore/include/k4FWCore/PodioDataSvc.h | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/k4FWCore/components/PodioInput.cpp b/k4FWCore/components/PodioInput.cpp index 12ce8a88..2d336ae6 100644 --- a/k4FWCore/components/PodioInput.cpp +++ b/k4FWCore/components/PodioInput.cpp @@ -184,14 +184,15 @@ PodioInput::PodioInput(const std::string& name, ISvcLocator* svcLoc) : Consumer( error() << "Could not get PodioDataSvc" << endmsg; } fillReaders(); - - auto key = edm4hep::EventHeaderName; - if (std::find(m_collectionNames.begin(), m_collectionNames.end(), key) == m_collectionNames.end()) { - m_collectionNames.value().push_back(key); - } } void PodioInput::operator()() const { + if (m_podioDataSvc->hasCollection(edm4hep::EventHeaderName)) { + m_readers[edm4hep::EventHeaderCollection::typeName](edm4hep::EventHeaderName); + } else { + info() << "No EventHeader collection found in the event. Not reading it" << endmsg; + } + for (auto& collName : m_collectionNames) { debug() << "Registering collection to read " << collName << endmsg; auto type = m_podioDataSvc->getCollectionType(collName); diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index 5343a034..ab53165a 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -88,6 +88,8 @@ class PodioDataSvc : public DataSvc { /// TODO: Make this private again after conversions have been properly solved podio::Frame& getMetaDataFrame() { return m_metadataframe; } + bool hasCollection(const std::string& name) const { return m_eventframe.get(name); } + private: /// PODIO reader for ROOT files podio::ROOTFrameReader m_reader; From 8a1aa927cf880964d69cacea8627a320e83dd322 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 17 Nov 2023 11:07:08 +0100 Subject: [PATCH 4/6] Make sure to only read and register collections once --- k4FWCore/include/k4FWCore/PodioDataSvc.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index ab53165a..af5a892e 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -69,6 +69,11 @@ class PodioDataSvc : public DataSvc { const std::string_view getCollectionType(const std::string& collName); template StatusCode readCollection(const std::string& collName) { + DataObject* objectPtr = nullptr; + if (DataSvc::findObject("/Event", "/" + collName, objectPtr)) { + debug() << "Collection " << collName << " already read, not reading it again" << endmsg; + return StatusCode::SUCCESS; + } const T* collection(nullptr); collection = static_cast(m_eventframe.get(collName)); if (collection == nullptr) { From c58424017c8fd031d8cdba553eb53c3144f154d7 Mon Sep 17 00:00:00 2001 From: Thomas Madlener Date: Tue, 21 Nov 2023 14:40:15 +0100 Subject: [PATCH 5/6] Remove unnecessary include --- test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp index 0f18a24d..487f10c1 100644 --- a/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp +++ b/test/k4FWCoreTest/src/components/ExampleEventHeaderConsumer.cpp @@ -30,7 +30,6 @@ #include #include -#include #include #include From 0e55c1b69457648a0fff79fbbb770a9bbc8a2f5e Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 21 Nov 2023 14:46:35 +0100 Subject: [PATCH 6/6] Remove method that is only used once --- k4FWCore/components/PodioInput.cpp | 2 +- k4FWCore/include/k4FWCore/PodioDataSvc.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/k4FWCore/components/PodioInput.cpp b/k4FWCore/components/PodioInput.cpp index 2d336ae6..2356ee46 100644 --- a/k4FWCore/components/PodioInput.cpp +++ b/k4FWCore/components/PodioInput.cpp @@ -187,7 +187,7 @@ PodioInput::PodioInput(const std::string& name, ISvcLocator* svcLoc) : Consumer( } void PodioInput::operator()() const { - if (m_podioDataSvc->hasCollection(edm4hep::EventHeaderName)) { + if (m_podioDataSvc->getEventFrame().get(edm4hep::EventHeaderName)) { m_readers[edm4hep::EventHeaderCollection::typeName](edm4hep::EventHeaderName); } else { info() << "No EventHeader collection found in the event. Not reading it" << endmsg; diff --git a/k4FWCore/include/k4FWCore/PodioDataSvc.h b/k4FWCore/include/k4FWCore/PodioDataSvc.h index af5a892e..f349be43 100644 --- a/k4FWCore/include/k4FWCore/PodioDataSvc.h +++ b/k4FWCore/include/k4FWCore/PodioDataSvc.h @@ -93,8 +93,6 @@ class PodioDataSvc : public DataSvc { /// TODO: Make this private again after conversions have been properly solved podio::Frame& getMetaDataFrame() { return m_metadataframe; } - bool hasCollection(const std::string& name) const { return m_eventframe.get(name); } - private: /// PODIO reader for ROOT files podio::ROOTFrameReader m_reader;