Replace the Legacy DataSvc and use the new Frame based one (#120)

* Add clangd tooling and spack to gitignore

* Switch to MetaDataHandles

* Switch to new input file for tests

* Update expected anajob outputs for new input file

* Fix documentation to handle non-zero exit codes

.gitignore

@@ -47,4 +47,11 @@ Testing
 # Test input files
 test/gaudi_opts/testConverterConstants.py
 test/inputFiles/muons.slcio
-test/inputFiles/testSimulation.slcio
+test/inputFiles/testSimulation.slcio
+
+# Tooling
+/.clangd/
+
+# Spack artifacts
+spack-*/
+spack-*.txt

doc/starterkit/k4MarlinWrapperCLIC/CEDViaWrapper.md

@@ -102,13 +102,17 @@ ApplicationMgr( TopAlg = algList,
               )
 ```
 
-With all these pieces put together (as in `examples/event_display.py`) it is no possible to run the event display. In order to run the event display we first have to start the `glced` server program to which the wrapped `CEDViewer` processor will then connect. Starting the server and running the wrapped processor can be done via
+With all these pieces put together (as in `examples/event_display.py`) it is now possible to run the event display. In order to run the event display we first have to start the `glced` server program to which the wrapped `CEDViewer` processor will then connect. Starting the server and running the wrapped processor can be done via
 ```bash
 glced &
 
-k4run $K4MARLINWRAPPER/examples/event_display.py --EventDataSvc.input=gamma_10GeV_edm4hep.root
+k4run $K4MARLINWRAPPER/examples/event_display.py --EventDataSvc.input=gamma_10GeV_edm4hep.root || true
 ```
 
+Note that we silence the exit code of `k4run` to work around a [framework
+issue](https://github.com/key4hep/k4FWCore/issues/125). This is usually not
+necessary if you run things interactively.
+
 ## Creating a Gaudi options file
 
 The `event_display.py` options file that is used above and that is present in the examples has been produced following these steps:

doc/starterkit/k4MarlinWrapperCLIC/Readme.md

@@ -124,14 +124,18 @@ included with k4MarlinWrapper. Note that:
   + Change line `evtsvc.input = '$TEST_DIR/inputFiles/ttbar1_edm4hep.root'` to point to the location of your input file.
   + At the bottom of the file, in the `ApplicationMgr` parameters, change `EvtMax   = 3,` to the number of events to run.
 
-
-This can be run with:
+This can be run in the following way. Note that we show the usage here in a way
+that also works in scripts with `-e` enabled. This is necessary because of a
+[framework issue](https://github.com/key4hep/k4FWCore/issues/125) that makes
+`k4run` exit with a non-zero exit code even though nothing went wrong.
 
 ```bash
 cd CLICPerformance/clicConfig
 
 cp ../../../test/gaudi_opts/clicRec_e4h_input.py .
 
 k4run clicRec_e4h_input.py --EventDataSvc.input ttbar_edm4hep.root
+result=$?
+[ $result = "0" ] || [ $result = "4" ] && true
 ```
 

k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h

@@ -7,7 +7,7 @@
 #include "k4MarlinWrapper/util/k4MarlinWrapperUtil.h"
 
 // FWCore
-#include <k4FWCore/DataHandle.h>
+#include <k4FWCore/PodioDataSvc.h>
 
 //k4EDM4hep2LcioConv
 #include "k4EDM4hep2LcioConv/k4EDM4hep2LcioConv.h"
@@ -35,7 +35,7 @@ class EDM4hep2LcioTool : public GaudiTool, virtual public IEDMConverter {
   Gaudi::Property<std::map<std::string, std::string>> m_collNames{this, "collNameMapping", {}};
   Gaudi::Property<bool>                               m_convertAll{this, "convertAll", true};
 
-  PodioLegacyDataSvc*             m_podioDataSvc;
+  PodioDataSvc*                   m_podioDataSvc;
   ServiceHandle<IDataProviderSvc> m_eventDataSvc;
 
   void convertTracks(vec_pair<lcio::TrackImpl*, edm4hep::Track>&           tracks_vec,

k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h

@@ -5,8 +5,7 @@
 #include <GaudiAlg/GaudiTool.h>
 
 // k4FWCore
-#include <k4FWCore/DataHandle.h>
-#include <k4FWCore/PodioLegacyDataSvc.h>
+#include <k4FWCore/PodioDataSvc.h>
 
 // Converter Interface
 #include "k4MarlinWrapper/converters/IEDMConverter.h"
@@ -48,7 +47,7 @@ class Lcio2EDM4hepTool : public GaudiTool, virtual public IEDMConverter {
   Gaudi::Property<bool>                               m_convertAll{this, "convertAll", true};
 
   ServiceHandle<IDataProviderSvc> m_eds;
-  PodioLegacyDataSvc*             m_podioDataSvc;
+  PodioDataSvc*                   m_podioDataSvc;
 
   // **********************************
   // Check if a collection was already registered to skip it

k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp

@@ -1,4 +1,8 @@
 #include "k4MarlinWrapper/converters/EDM4hep2Lcio.h"
+
+#include "k4FWCore/DataHandle.h"
+#include "k4FWCore/MetaDataHandle.h"
+
 #include <unordered_map>
 
 DECLARE_COMPONENT(EDM4hep2LcioTool);
@@ -12,7 +16,7 @@ EDM4hep2LcioTool::~EDM4hep2LcioTool() { ; }
 
 StatusCode EDM4hep2LcioTool::initialize() {
   StatusCode sc  = m_eventDataSvc.retrieve();
-  m_podioDataSvc = dynamic_cast<PodioLegacyDataSvc*>(m_eventDataSvc.get());
+  m_podioDataSvc = dynamic_cast<PodioDataSvc*>(m_eventDataSvc.get());
 
   if (sc == StatusCode::FAILURE) {
     error() << "Error retrieving Event Data Service" << endmsg;
@@ -49,10 +53,9 @@ void EDM4hep2LcioTool::convertTrackerHits(vec_pair<lcio::TrackerHitImpl*, edm4he
   DataHandle<edm4hep::TrackerHitCollection> trackerhits_handle{e4h_coll_name, Gaudi::DataHandle::Reader, this};
   const auto                                trackerhits_coll = trackerhits_handle.get();
 
-  const auto collID    = trackerhits_coll->getID();
-  const auto cellIDstr = trackerhits_handle.getCollMetadataCellID(collID);
+  MetaDataHandle<std::string> cellIDStrHandle{trackerhits_handle, "CellIDEncoding", Gaudi::DataHandle::Reader};
 
-  auto* conv_trackerhits = convTrackerHits(trackerhits_coll, cellIDstr, trackerhits_vec);
+  auto* conv_trackerhits = convTrackerHits(trackerhits_coll, cellIDStrHandle.get(), trackerhits_vec);
 
   // Add all trackerhits to event
   lcio_event->addCollection(conv_trackerhits, lcio_coll_name);
@@ -68,8 +71,8 @@ void EDM4hep2LcioTool::convertSimTrackerHits(
   DataHandle<edm4hep::SimTrackerHitCollection> simtrackerhits_handle{e4h_coll_name, Gaudi::DataHandle::Reader, this};
   const auto                                   simtrackerhits_coll = simtrackerhits_handle.get();
 
-  const auto collID    = simtrackerhits_coll->getID();
-  const auto cellIDstr = simtrackerhits_handle.getCollMetadataCellID(collID);
+  MetaDataHandle<std::string> cellIDHandle{simtrackerhits_handle, "CellIDEncoding", Gaudi::DataHandle::Reader};
+  const auto                  cellIDstr = cellIDHandle.get();
 
   auto* conv_simtrackerhits = convSimTrackerHits(simtrackerhits_coll, cellIDstr, simtrackerhits_vec, mcparticles_vec);
 
@@ -86,8 +89,8 @@ void EDM4hep2LcioTool::convertCalorimeterHits(
   DataHandle<edm4hep::CalorimeterHitCollection> calohit_handle{e4h_coll_name, Gaudi::DataHandle::Reader, this};
   const auto                                    calohit_coll = calohit_handle.get();
 
-  const auto collID    = calohit_coll->getID();
-  const auto cellIDstr = calohit_handle.getCollMetadataCellID(collID);
+  MetaDataHandle<std::string> cellIDHandle{calohit_handle, "CellIDEncoding", Gaudi::DataHandle::Reader};
+  const auto                  cellIDstr = cellIDHandle.get();
 
   auto* conv_calohits = convCalorimeterHits(calohit_coll, cellIDstr, calo_hits_vec);
 
@@ -120,8 +123,8 @@ void EDM4hep2LcioTool::convertSimCalorimeterHits(
   DataHandle<edm4hep::SimCalorimeterHitCollection> sim_calohit_handle{e4h_coll_name, Gaudi::DataHandle::Reader, this};
   const auto                                       simcalohit_coll = sim_calohit_handle.get();
 
-  auto       collID    = simcalohit_coll->getID();
-  const auto cellIDstr = sim_calohit_handle.getCollMetadataCellID(collID);
+  MetaDataHandle<std::string> cellIDHandle{sim_calohit_handle, "CellIDEncoding", Gaudi::DataHandle::Reader};
+  const auto                  cellIDstr = cellIDHandle.get();
 
   // TODO mcparticles_vdc
   auto* conv_simcalohits = convSimCalorimeterHits(simcalohit_coll, cellIDstr, sim_calo_hits_vec, mcparticles);
@@ -227,22 +230,12 @@ void EDM4hep2LcioTool::convertEventHeader(const std::string& e4h_coll_name, lcio
 // Select the appropiate method to convert a collection given its type
 void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::string& lcio_coll_name,
                                   lcio::LCEventImpl* lcio_event, CollectionsPairVectors& collection_pairs) {
-  // Get the associated type to the collection name
-  auto evt_store = m_podioDataSvc->getCollections();
-
-  std::string fulltype = "";
-
-  for (auto& [name, coll] : evt_store) {
-    if (name == e4h_coll_name) {
-      fulltype = coll->getValueTypeName();
-      break;
-    }
-  }
-
-  if (fulltype == "") {
-    error() << "Could not get type from collection name: " << e4h_coll_name << endmsg;
-    return;
+  const auto& evtFrame = m_podioDataSvc->getEventFrame();
+  const auto  collPtr  = evtFrame.get(e4h_coll_name);
+  if (!collPtr) {
+    error() << "No collection with name: " << e4h_coll_name << " available for conversion" << endmsg;
   }
+  const auto fulltype = collPtr->getValueTypeName();
 
   if (fulltype == "edm4hep::Track") {
     convertTracks(collection_pairs.tracks, collection_pairs.trackerhits, e4h_coll_name, lcio_coll_name, lcio_event);
@@ -272,6 +265,9 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s
                                   collection_pairs.clusters, e4h_coll_name, lcio_coll_name, lcio_event);
   } else if (fulltype == "edm4hep::EventHeader") {
     convertEventHeader(e4h_coll_name, lcio_event);
+  } else if (fulltype == "edm4hep::CaloHitContribution") {
+    debug() << "CaloHitContribution collection cannot be converted standalone. "
+            << "SimCalorimeterHit collection to be converted in order to be able to attach to them" << endmsg;
   } else {
     warning() << "Error trying to convert requested " << fulltype << " with name " << e4h_coll_name << endmsg;
     warning() << "List of supported types: "
@@ -285,14 +281,14 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s
 // Parse property parameters and convert the indicated collections.
 // Use the collection names in the parameters to read and write them
 StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
-  const auto collections = m_podioDataSvc->getCollections();
+  const auto collections = m_podioDataSvc->getEventFrame().getAvailableCollections();
   // Start off with the pre-defined collection name mappings
   auto collsToConvert{m_collNames.value()};
   if (m_convertAll) {
     info() << "Converting all collections from EDM4hep to LCIO" << endmsg;
     // And simply add the rest, exploiting the fact that emplace will not
     // replace existing entries with the same key
-    for (const auto& [name, _] : collections) {
+    for (const auto& name : collections) {
       collsToConvert.emplace(name, name);
     }
   }

k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp

@@ -18,6 +18,9 @@
 
 #include "k4EDM4hep2LcioConv/k4Lcio2EDM4hepConv.h"
 
+#include <k4FWCore/DataHandle.h>
+#include <k4FWCore/MetaDataHandle.h>
+
 DECLARE_COMPONENT(Lcio2EDM4hepTool);
 
 Lcio2EDM4hepTool::Lcio2EDM4hepTool(const std::string& type, const std::string& name, const IInterface* parent)
@@ -30,7 +33,7 @@ Lcio2EDM4hepTool::Lcio2EDM4hepTool(const std::string& type, const std::string& n
 Lcio2EDM4hepTool::~Lcio2EDM4hepTool() { ; }
 
 StatusCode Lcio2EDM4hepTool::initialize() {
-  m_podioDataSvc = dynamic_cast<PodioLegacyDataSvc*>(m_eds.get());
+  m_podioDataSvc = dynamic_cast<PodioDataSvc*>(m_eds.get());
   if (nullptr == m_podioDataSvc)
     return StatusCode::FAILURE;
 
@@ -39,9 +42,12 @@ StatusCode Lcio2EDM4hepTool::initialize() {
 
 StatusCode Lcio2EDM4hepTool::finalize() { return GaudiTool::finalize(); }
 
+// **********************************
+// Check if a collection was already registered to skip it
+// **********************************
 bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) {
-  auto collections = m_podioDataSvc->getCollections();
-  for (const auto& [name, coll] : collections) {
+  auto collections = m_podioDataSvc->getEventFrame().getAvailableCollections();
+  for (const auto& name : collections) {
     if (collection_name == name) {
       debug() << "Collection named " << name << " already registered, skipping conversion." << endmsg;
       return true;
@@ -68,24 +74,18 @@ void Lcio2EDM4hepTool::registerCollection(
     error() << "Could not register collection " << name << endmsg;
   }
 
-  auto handle = DataHandle<podio::CollectionBase>{name, Gaudi::DataHandle::Reader, this};
-
   // Convert metadata
   if (lcioColl != nullptr) {
-    const auto acollid = handle.get()->getID();
-
     std::vector<std::string> string_keys = {};
     lcioColl->getParameters().getStringKeys(string_keys);
 
-    auto& e4h_coll_md = m_podioDataSvc->getProvider().getCollectionMetaData(acollid);
-
     for (auto& elem : string_keys) {
       if (elem == "CellIDEncoding") {
-        std::string lcio_coll_cellid_str = lcioColl->getParameters().getStringVal(lcio::LCIO::CellIDEncoding);
-        e4h_coll_md.setValue("CellIDEncodingString", lcio_coll_cellid_str);
+        const auto& lcio_coll_cellid_str = lcioColl->getParameters().getStringVal(lcio::LCIO::CellIDEncoding);
+        auto&       mdFrame              = m_podioDataSvc->getMetaDataFrame();
+        mdFrame.putParameter(podio::collMetadataParamName(name, "CellIDEncoding"), lcio_coll_cellid_str);
       } else {
-        std::string lcio_coll_value = lcioColl->getParameters().getStringVal(elem);
-        e4h_coll_md.setValue(elem, lcio_coll_value);
+        // TODO: figure out where this actually needs to go
       }
     }
   }

test/gaudi_opts/clicRec_e4h_input.py

@@ -23,12 +23,12 @@
 # algList.append(read)
 
 
-from Configurables import k4LegacyDataSvc, PodioLegacyInput
-evtsvc = k4LegacyDataSvc('EventDataSvc')
-evtsvc.input = '$TEST_DIR/inputFiles/ttbar1_edm4hep.root'
+from Configurables import k4DataSvc, PodioInput
+evtsvc = k4DataSvc('EventDataSvc')
+evtsvc.input = '$TEST_DIR/inputFiles/ttbar_edm4hep_frame.root'
 
 
-inp = PodioLegacyInput('InputReader')
+inp = PodioInput('InputReader')
 inp.collections = [
   'EventHeader',
   'MCParticles',
@@ -1765,8 +1765,8 @@
 
 
 # Write output to EDM4hep
-from Configurables import PodioLegacyOutput
-out = PodioLegacyOutput("PodioOutput", filename = "my_output.root")
+from Configurables import PodioOutput
+out = PodioOutput("PodioOutput", filename = "my_output.root")
 out.outputCommands = ["keep *"]
 
 

test/gaudi_opts/edm_converters.py

@@ -1,12 +1,12 @@
 from Gaudi.Configuration import *
 
-from Configurables import k4LegacyDataSvc, TestE4H2L, EDM4hep2LcioTool, Lcio2EDM4hepTool
+from Configurables import k4DataSvc, TestE4H2L, EDM4hep2LcioTool, Lcio2EDM4hepTool
 
 algList = []
 
 END_TAG = "END_TAG"
 
-evtsvc = k4LegacyDataSvc("EventDataSvc")
+evtsvc = k4DataSvc("EventDataSvc")
 
 # EDM4hep2lcio Tool
 edmConvTool = EDM4hep2LcioTool("EDM4hep2lcio")