From 5967d0b59c36f080cfe8a4bbcf93720d7e0b4117 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 14 Jan 2025 09:12:02 +0100 Subject: [PATCH 01/30] Add support for running with IOSvc --- k4MarlinWrapper/examples/clicRec_e4h_input.py | 64 +++++--- .../k4MarlinWrapper/converters/EDM4hep2Lcio.h | 4 + .../k4MarlinWrapper/converters/Lcio2EDM4hep.h | 2 +- .../src/components/EDM4hep2Lcio.cpp | 146 ++++++++++++++++-- .../src/components/Lcio2EDM4hep.cpp | 51 ++++-- test/CMakeLists.txt | 10 +- test/gaudi_opts/test_global_converter_maps.py | 36 ++++- .../test_link_conversion_edm4hep.py | 31 ++-- test/scripts/clicRec_e4h_input.sh | 23 ++- 9 files changed, 289 insertions(+), 78 deletions(-) diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index ee802f4e..9d8021fa 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -19,10 +19,16 @@ import os -from Gaudi.Configuration import * +from Gaudi.Configuration import DEBUG, WARNING -from Configurables import LcioEvent, MarlinProcessorWrapper -from k4MarlinWrapper.parseConstants import * +from Configurables import MarlinProcessorWrapper +from k4MarlinWrapper.parseConstants import parseConstants + +from Configurables import Lcio2EDM4hepTool, EDM4hep2LcioTool +from Configurables import k4DataSvc, PodioInput, PodioOutput, EventDataSvc + +from k4FWCore import ApplicationMgr, IOSvc +from k4FWCore.parseArgs import parser algList = [] @@ -33,22 +39,33 @@ parseConstants(CONSTANTS) - -# For converters -from Configurables import ToolSvc, Lcio2EDM4hepTool, EDM4hep2LcioTool - - -from Configurables import k4DataSvc, PodioInput - -evtsvc = k4DataSvc("EventDataSvc") -evtsvc.input = os.path.join( - "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") +parser.add_argument( + "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" ) - - -inp = PodioInput("InputReader") -inp.OutputLevel = DEBUG - +parser.add_argument("--rec-output", help="Output file name for the REC file") +parser.add_argument("--dst-output", help="Output file name for the DST file") + +args = parser.parse_known_args()[0] + +if args.iosvc: + evtsvc = EventDataSvc("EventDataSvc") + iosvc = IOSvc() + iosvc.Input = os.path.join( + "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") + ) + iosvc.Output = "my_output.root" + iosvc.outputDstName = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"] +else: + evtsvc = k4DataSvc("EventDataSvc") + evtsvc.input = os.path.join( + "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") + ) + + inp = PodioInput("InputReader") + inp.OutputLevel = DEBUG + + out = PodioOutput("PodioOutput", filename="my_output.root") + out.outputCommands = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"] MyAIDAProcessor = MarlinProcessorWrapper("MyAIDAProcessor") MyAIDAProcessor.OutputLevel = WARNING @@ -1167,7 +1184,7 @@ "DropCollectionTypes": [], "FullSubsetCollections": ["EfficientMCParticles", "InefficientMCParticles"], "KeepCollectionNames": [], - "LCIOOutputFile": ["Output_REC_e4h_input.slcio"], + "LCIOOutputFile": [args.rec_output], "LCIOWriteMode": ["WRITE_NEW"], } @@ -1233,7 +1250,7 @@ "RefinedVertices", "RefinedVertices_RP", ], - "LCIOOutputFile": ["Output_DST_e4h_input.slcio"], + "LCIOOutputFile": [args.dst_output], "LCIOWriteMode": ["WRITE_NEW"], } @@ -2429,6 +2446,7 @@ } +<<<<<<< HEAD # Write output to EDM4hep from Configurables import PodioOutput @@ -2437,6 +2455,8 @@ algList.append(inp) +======= +>>>>>>> cb3c7a3 (Add support for running with IOSvc) algList.append(MyAIDAProcessor) algList.append(EventNumber) algList.append(InitDD4hep) @@ -2486,8 +2506,8 @@ # # algList.append(VertexFinderUnconstrained) # Config.VertexUnconstrainedON algList.append(Output_REC) algList.append(Output_DST) -algList.append(out) -from Configurables import ApplicationMgr +if not args.iosvc: + algList = [inp] + algList + [out] ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=3, ExtSvc=[evtsvc], OutputLevel=WARNING) diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h index 4d281004..c0d7658a 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h @@ -64,6 +64,8 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { PodioDataSvc* m_podioDataSvc; ServiceHandle m_eventDataSvc; + std::vector m_collectionNames; + std::map m_idToName; void convertTracks(TrackMap& tracks_vec, const std::string& e4h_coll_name, const std::string& lcio_coll_name, lcio::LCEventImpl* lcio_event); @@ -110,6 +112,8 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { std::vector& pidCollections, std::vector& dQdxCollections); + StatusCode getAvailableCollectionsFromStore(); + /// Get an EDM4hep collection by name, consulting either the podio based data /// svc or the IOSvc podio::CollectionBase* getEDM4hepCollection(const std::string& name) const; diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h b/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h index a72a11f3..79d825d5 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h @@ -58,7 +58,7 @@ class Lcio2EDM4hepTool : public AlgTool, virtual public IEDMConverter { Gaudi::Property> m_collNames{this, "collNameMapping", {}}; Gaudi::Property m_convertAll{this, "convertAll", true}; - ServiceHandle m_eds; + ServiceHandle m_eventDataSvc; PodioDataSvc* m_podioDataSvc; // ********************************** diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index 30b81a25..cf5249c1 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -17,16 +17,24 @@ * limitations under the License. */ #include "k4MarlinWrapper/converters/EDM4hep2Lcio.h" +#include #include "GlobalConvertedObjectsMap.h" +#include "UTIL/PIDHandler.h" + #include "edm4hep/Constants.h" #include "k4FWCore/DataHandle.h" +#include "k4FWCore/FunctionalUtils.h" #include "k4FWCore/MetaDataHandle.h" #include "k4FWCore/PodioDataSvc.h" #include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/IDataManagerSvc.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include "GaudiKernel/SmartDataPtr.h" +#include #include DECLARE_COMPONENT(EDM4hep2LcioTool); @@ -55,14 +63,14 @@ EDM4hep2LcioTool::EDM4hep2LcioTool(const std::string& type, const std::string& n } StatusCode EDM4hep2LcioTool::initialize() { - StatusCode sc = m_eventDataSvc.retrieve(); - m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); - + StatusCode sc = m_eventDataSvc.retrieve(); if (sc == StatusCode::FAILURE) { - error() << "Error retrieving Event Data Service" << endmsg; + error() << "Could not retrieve the EventDataSvc;" << endmsg; return StatusCode::FAILURE; } + m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); + return AlgTool::initialize(); } @@ -277,7 +285,7 @@ void EDM4hep2LcioTool::convertEventHeader(const std::string& e4h_coll_name, lcio podio::CollectionBase* EDM4hep2LcioTool::getEDM4hepCollection(const std::string& collName) const { DataObject* p; - auto sc = m_podioDataSvc->retrieveObject(collName, p); + auto sc = m_eventDataSvc->retrieveObject(collName, p); if (sc.isFailure()) { throw GaudiException("Collection not found", name(), StatusCode::FAILURE); } @@ -302,14 +310,59 @@ podio::CollectionBase* EDM4hep2LcioTool::getEDM4hepCollection(const std::string& throw GaudiException("Collection could not be casted to the expected type", name(), StatusCode::FAILURE); } +StatusCode EDM4hep2LcioTool::getAvailableCollectionsFromStore() { + SmartIF mgr; + mgr = evtSvc(); + + SmartDataPtr root(evtSvc(), "/Event"); + if (!root) { + error() << "Failed to retrieve root object /Event" << endmsg; + return StatusCode::FAILURE; + } + + auto pObj = root->registry(); + if (!pObj) { + error() << "Failed to retrieve the root registry object" << endmsg; + return StatusCode::FAILURE; + } + std::vector leaves; + StatusCode sc = mgr->objectLeaves(pObj, leaves); + if (!sc.isSuccess()) { + error() << "Failed to retrieve object leaves" << endmsg; + return StatusCode::FAILURE; + } + for (const auto& pReg : leaves) { + if (pReg->name() == k4FWCore::frameLocation) { + continue; + } + DataObject* p; + sc = m_eventDataSvc->retrieveObject("/Event" + pReg->name(), p); + if (sc.isFailure()) { + error() << "Could not retrieve object " << pReg->name() << " from the EventStore" << endmsg; + return sc; + } + auto wrapper = dynamic_cast>*>(p); + if (!wrapper) { + continue; + } + // Remove the leading / + m_collectionNames.push_back(pReg->name().substr(1, pReg->name().size() - 1)); + m_idToName.emplace(wrapper->getData()->getID(), pReg->name()); + } + return StatusCode::SUCCESS; +} + // Select the appropriate method to convert a collection given its type void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::string& lcio_coll_name, lcio::LCEventImpl* lcio_event, CollectionPairMappings& collection_pairs, std::vector& pidCollections, std::vector& dQdxCollections) { - const auto& metadata = m_podioDataSvc->getMetaDataFrame(); - const auto collPtr = getEDM4hepCollection(e4h_coll_name); - const auto fulltype = collPtr->getValueTypeName(); + std::optional> metadata; + if (m_podioDataSvc) { + metadata = m_podioDataSvc->getMetaDataFrame(); + } + const auto collPtr = getEDM4hepCollection(e4h_coll_name); + const auto fulltype = collPtr->getValueTypeName(); debug() << "Converting type " << fulltype << " from input " << e4h_coll_name << endmsg; @@ -340,8 +393,32 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s } else if (fulltype == "edm4hep::EventHeader") { convertEventHeader(e4h_coll_name, lcio_event); } else if (fulltype == "edm4hep::ParticleID") { - pidCollections.emplace_back(e4h_coll_name, static_cast(collPtr), - edm4hep::utils::PIDHandler::getAlgoInfo(metadata, e4h_coll_name)); + std::optional maybeAlgoName; + std::optional maybeAlgoType; + std::optional> maybeParamNames; + + if (m_podioDataSvc) { + const auto& frame = metadata.value().get(); + maybeAlgoName = + frame.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName)); + maybeAlgoType = + frame.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType)); + maybeParamNames = frame.getParameter>( + podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames)); + + } else { + maybeAlgoName = k4FWCore::getParameter( + podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName)); + maybeAlgoType = + k4FWCore::getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType)); + maybeParamNames = k4FWCore::getParameter>( + podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames)); + } + + edm4hep::utils::ParticleIDMeta pidInfo{std::move(maybeAlgoName.value()), maybeAlgoType.value(), + maybeParamNames.value()}; + + pidCollections.emplace_back(e4h_coll_name, static_cast(collPtr), pidInfo); } else if (fulltype == "edm4hep::RecDqDx") { dQdxCollections.emplace_back(e4h_coll_name, static_cast(collPtr)); } @@ -362,8 +439,17 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s // Parse property parameters and convert the indicated collections. // Use the collection names in the parameters to read and write them StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { - const auto& edmEvent = m_podioDataSvc->getEventFrame(); - const auto collections = edmEvent.getAvailableCollections(); + std::optional> edmEvent; + if (m_collectionNames.empty() && m_podioDataSvc) { + edmEvent = m_podioDataSvc->getEventFrame(); + m_collectionNames = edmEvent.value().get().getAvailableCollections(); + } else if (m_collectionNames.empty()) { + auto sc = getAvailableCollectionsFromStore(); + if (sc.isFailure()) { + warning() << "Could not retrieve available collections from the EventStore" << endmsg; + return sc; + } + } // Start off with the pre-defined collection name mappings auto collsToConvert{m_collNames.value()}; // We *always* want to convert the EventHeader @@ -372,7 +458,7 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { info() << "Converting all collections from EDM4hep to LCIO" << endmsg; // And simply add the rest, exploiting the fact that emplace will not // replace existing entries with the same key - for (const auto& name : collections) { + for (const auto& name : m_collectionNames) { collsToConvert.emplace(name, name); } } @@ -399,13 +485,39 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { } debug() << "Event: " << lcio_event->getEventNumber() << " Run: " << lcio_event->getRunNumber() << endmsg; - // Deal with EDM4hep2LCIOConv::sortParticleIDs(pidCollections); + if (!m_podioDataSvc) { + DataObject* p; + StatusCode code = m_eventDataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p); + if (code.isSuccess()) { + auto* frame = dynamic_cast*>(p); + edmEvent = std::cref(frame->getData()); + } else { + auto frame = podio::Frame{}; + edmEvent = frame; + } + } for (const auto& pidCollMeta : pidCollections) { - const auto algoId = attachParticleIDMetaData(lcio_event, edmEvent, pidCollMeta); + auto algoId = attachParticleIDMetaData(lcio_event, edmEvent.value(), pidCollMeta); if (!algoId.has_value()) { - warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name - << " for setting consistent metadata" << endmsg; + // Now go over the collections that have been produced in a functional algorithm (if any) + bool found = false; + if (!m_podioDataSvc) { + const auto id = (*pidCollMeta.coll)[0].getParticle().id().collectionID; + if (auto it = m_idToName.find(id); it != m_idToName.end()) { + auto name = it->second; + if (pidCollMeta.metadata.has_value()) { + UTIL::PIDHandler pidHandler(lcio_event->getCollection(name)); + algoId = + pidHandler.addAlgorithm(pidCollMeta.metadata.value().algoName, pidCollMeta.metadata.value().paramNames); + found = true; + } + } + } + if (!found) { + warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name + << " for setting consistent metadata" << endmsg; + } } convertParticleIDs(collection_pairs.particleIDs, pidCollMeta.name, algoId.value_or(-1)); } diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 19bc2462..b57c80de 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -32,6 +32,7 @@ #include "GaudiKernel/AnyDataWrapper.h" +#include #include DECLARE_COMPONENT(Lcio2EDM4hepTool); @@ -39,19 +40,17 @@ DECLARE_COMPONENT(Lcio2EDM4hepTool); using namespace k4MarlinWrapper; Lcio2EDM4hepTool::Lcio2EDM4hepTool(const std::string& type, const std::string& name, const IInterface* parent) - : AlgTool(type, name, parent), m_eds("EventDataSvc", "Lcio2EDM4hepTool") { + : AlgTool(type, name, parent), m_eventDataSvc("EventDataSvc", "Lcio2EDM4hepTool") { declareInterface(this); - StatusCode sc = m_eds.retrieve(); + StatusCode sc = m_eventDataSvc.retrieve(); if (sc.isFailure()) { error() << "Could not retrieve EventDataSvc" << endmsg; } } StatusCode Lcio2EDM4hepTool::initialize() { - m_podioDataSvc = dynamic_cast(m_eds.get()); - if (!m_podioDataSvc) - return StatusCode::FAILURE; + m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); return AlgTool::initialize(); } @@ -62,9 +61,13 @@ StatusCode Lcio2EDM4hepTool::finalize() { return AlgTool::finalize(); } // Check if a collection was already registered to skip it // ********************************** bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { - auto collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); + std::vector collections; + //TODO: + if (m_podioDataSvc) { + collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); + } for (const auto& name : collections) { - if (collection_name == name) { + if (name == collection_name) { debug() << "Collection named " << name << " already registered, skipping conversion." << endmsg; return true; } @@ -85,7 +88,7 @@ void Lcio2EDM4hepTool::registerCollection( // No need to check for pre-existing collections, since we only ever end up // here if that is not the case - auto sc = m_podioDataSvc->registerObject("/Event", "/" + std::string(name), wrapper); + auto sc = m_eventDataSvc->registerObject("/Event", "/" + std::string(name), wrapper); if (sc == StatusCode::FAILURE) { error() << "Could not register collection " << name << endmsg; } @@ -98,8 +101,14 @@ void Lcio2EDM4hepTool::registerCollection( for (auto& elem : string_keys) { if (elem == edm4hep::labels::CellIDEncoding) { const auto& lcio_coll_cellid_str = lcioColl->getParameters().getStringVal(lcio::LCIO::CellIDEncoding); - auto& mdFrame = m_podioDataSvc->getMetaDataFrame(); - mdFrame.putParameter(podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding), lcio_coll_cellid_str); + if (m_podioDataSvc) { + auto& mdFrame = m_podioDataSvc->getMetaDataFrame(); + mdFrame.putParameter(podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding), + lcio_coll_cellid_str); + } else { + k4FWCore::putParameter(podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding), + lcio_coll_cellid_str); + } debug() << "Storing CellIDEncoding " << podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding) << " value: " << lcio_coll_cellid_str << endmsg; } else { @@ -131,8 +140,10 @@ namespace { StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { // Convert event parameters - // auto& frame = const_cast(m_podioDataSvc->getEventFrame()); - auto& frame = m_podioDataSvc->m_eventframe; + std::optional> frame; + if (m_podioDataSvc) { + frame = m_podioDataSvc->m_eventframe; + } LCIO2EDM4hepConv::convertObjectParameters(the_event, frame); // Convert Event Header outside the collections loop @@ -208,9 +219,19 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { } // Set the ParticleID meta information - auto& metadataFrame = m_podioDataSvc->getMetaDataFrame(); - for (const auto& [collName, pidInfo] : pidInfos) { - edm4hep::utils::PIDHandler::setAlgoInfo(metadataFrame, collName, pidInfo); + // TODO: Clean up + if (m_podioDataSvc) { + auto& metadataFrame = m_podioDataSvc->getMetaDataFrame(); + for (const auto& [collName, pidInfo] : pidInfos) { + edm4hep::utils::PIDHandler::setAlgoInfo(metadataFrame, collName, pidInfo); + } + } else { + for (const auto& [collName, pidInfo] : pidInfos) { + k4FWCore::putParameter(podio::collMetadataParamName(collName, edm4hep::labels::PIDAlgoName), pidInfo.algoName); + k4FWCore::putParameter(podio::collMetadataParamName(collName, edm4hep::labels::PIDAlgoType), pidInfo.algoType()); + k4FWCore::putParameter(podio::collMetadataParamName(collName, edm4hep::labels::PIDParameterNames), + pidInfo.paramNames); + } } // We want one "global" map that is created the first time it is used in the event. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 358357f2..5ca42b8a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -77,7 +77,8 @@ set_tests_properties ( same_num_io PASS_REGULAR_EXPRESSION "Input and output have same number of events") # Test clicReconstruction with EDM4hep input and output -ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}") +ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --no-iosvc") +ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input_iosvc COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --iosvc") # Run clicReconstruction sequence with LCIO input and output, no converters, with inter-event parallelism ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_lcio_mt COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_lcio_mt.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/testSimulation.slcio}") @@ -88,11 +89,16 @@ add_test( clic_geo_test ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/geoTest_ # Test for checking whether the converters can resolve relations across # multiple processors ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --EventDataSvc.input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) +ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps_iosvc COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --iosvc --IOSvc.Input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) ExternalData_Add_Test( marlinwrapper_tests NAME link_conversion_edm4hep_to_lcio COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_link_conversion_edm4hep.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} ) +ExternalData_Add_Test( marlinwrapper_tests + NAME link_conversion_edm4hep_to_lcio_iosvc + COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_link_conversion_edm4hep.py --iosvc --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} +) add_test( event_header_conversion bash -c "k4run ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/createEventHeader.py && anajob test.slcio | grep 'EVENT: 42'" ) @@ -104,7 +110,7 @@ set_tests_properties(${test_names} PROPERTIES ENVIRONMENT ) set_tests_properties( - clicRec clicRec_lcio_mt clicRec_edm4hep_input + clicRec clicRec_lcio_mt clicRec_edm4hep_input clicRec_edm4hep_input_iosvc PROPERTIES DEPENDS CLICPerformance_setup ) diff --git a/test/gaudi_opts/test_global_converter_maps.py b/test/gaudi_opts/test_global_converter_maps.py index 2746c3f8..846734d0 100644 --- a/test/gaudi_opts/test_global_converter_maps.py +++ b/test/gaudi_opts/test_global_converter_maps.py @@ -23,20 +23,40 @@ from Configurables import ( PodioInput, + PodioOutput, MarlinProcessorWrapper, k4DataSvc, Lcio2EDM4hepTool, EDM4hep2LcioTool, MCRecoLinkChecker, - ApplicationMgr, PseudoRecoAlgorithm, + EventDataSvc, ) -evtsvc = k4DataSvc("EventDataSvc") +from k4FWCore import ApplicationMgr, IOSvc -podioInput = PodioInput("InputReader") -podioInput.collections = ["MCParticles"] -podioInput.OutputLevel = INFO +from k4FWCore.parseArgs import parser + +parser.add_argument( + "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" +) + +args = parser.parse_known_args()[0] + +if args.iosvc: + evtsvc = EventDataSvc("EventDataSvc") +else: + evtsvc = k4DataSvc("EventDataSvc") + +if args.iosvc: + iosvc = IOSvc() + iosvc.Output = "global_converter_maps_iosvc.root" +else: + podioInput = PodioInput("InputReader") + podioInput.collections = ["MCParticles"] + podioInput.OutputLevel = INFO + podioOutput = PodioOutput("OutputWriter") + podioOutput.filename = "global_converter_maps.root" PseudoRecoAlg = PseudoRecoAlgorithm( "PseudoRecoAlgorithm", InputMCs=["MCParticles"], OutputRecos=["PseudoRecoParticles"] @@ -74,10 +94,12 @@ mcLinkChecker.OutputLevel = DEBUG algList = [ - podioInput, PseudoRecoAlg, TrivialMCTruthLinkerProc, mcLinkChecker, ] -ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=3, ExtSvc=[evtsvc], OutputLevel=DEBUG) +if not args.iosvc: + algList = [podioInput] + algList + [podioOutput] + +ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=1, ExtSvc=[evtsvc], OutputLevel=DEBUG) diff --git a/test/gaudi_opts/test_link_conversion_edm4hep.py b/test/gaudi_opts/test_link_conversion_edm4hep.py index 92080f7e..26c22cef 100644 --- a/test/gaudi_opts/test_link_conversion_edm4hep.py +++ b/test/gaudi_opts/test_link_conversion_edm4hep.py @@ -23,24 +23,33 @@ from Configurables import ( PodioInput, k4DataSvc, - ApplicationMgr, PseudoRecoAlgorithm, TrivialMCRecoLinker, MarlinProcessorWrapper, EDM4hep2LcioTool, + EventDataSvc, ) +from k4FWCore import ApplicationMgr, IOSvc + from k4FWCore.parseArgs import parser parser.add_argument("--inputfile", help="Input file") -my_args = parser.parse_known_args()[0] - -evtsvc = k4DataSvc("EventDataSvc") -evtsvc.input = my_args.inputfile +parser.add_argument( + "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" +) +args = parser.parse_known_args()[0] -podioInput = PodioInput("InputReader") -podioInput.collections = ["MCParticles"] -podioInput.OutputLevel = INFO +if args.iosvc: + evtsvc = EventDataSvc("EventDataSvc") + iosvc = IOSvc() + iosvc.Input = args.inputfile +else: + evtsvc = k4DataSvc("EventDataSvc") + evtsvc.input = args.inputfile + podioInput = PodioInput("InputReader") + podioInput.collections = ["MCParticles"] + podioInput.OutputLevel = INFO PseudoRecoAlg = PseudoRecoAlgorithm( @@ -74,9 +83,13 @@ mcLinkConverter.OutputLevel = DEBUG MarlinMCLinkChecker.EDM4hep2LcioTool = mcLinkConverter +algList = [PseudoRecoAlg, MCRecoLinker, MarlinMCLinkChecker] + +if not args.iosvc: + algList = [podioInput] + algList ApplicationMgr( - TopAlg=[podioInput, PseudoRecoAlg, MCRecoLinker, MarlinMCLinkChecker], + TopAlg=algList, ExtSvc=[evtsvc], EvtMax=-1, EvtSel="NONE", diff --git a/test/scripts/clicRec_e4h_input.sh b/test/scripts/clicRec_e4h_input.sh index 7b67072a..5c9d6916 100755 --- a/test/scripts/clicRec_e4h_input.sh +++ b/test/scripts/clicRec_e4h_input.sh @@ -23,7 +23,19 @@ set -eu cd CLICPerformance/clicConfig -k4run $EXAMPLE_DIR/clicRec_e4h_input.py --EventDataSvc.input=$1 +if [ "$2" = "--iosvc" ]; then + iosvc="_iosvc" + echo "Running with IO service" + file_arg="--iosvc --IOSvc.Input=$1" +elif [ "$2" = "--no-iosvc" ]; then + iosvc="" + echo "Running without IO service" + file_arg="--EventDataSvc.input=$1" +else + echo "Wrong argument $2" + return 1 +fi +k4run $EXAMPLE_DIR/clicRec_e4h_input.py ${file_arg} --rec-output Output_REC_e4h_input$iosvc.slcio --dst-output Output_DST_e4h_input$iosvc.slcio input_num_events=$(python $TEST_DIR/python/root_num_events.py $1) output_num_events=$(python $TEST_DIR/python/root_num_events.py my_output.root) @@ -35,14 +47,15 @@ if [ "$input_num_events" != "$output_num_events" ]; then fi # Second check: contents (at least superficially) -echo "Comparing contents of Output_REC_e4h_input.slcio" -if ! diff <(anajob Output_REC_e4h_input.slcio) $TEST_DIR/inputFiles/anajob_Output_REC.expected; then +# Exclude the file name since it is different when using the IOSvc +echo "Comparing contents of Output_REC_e4h_input$iosvc.slcio" +if ! diff -I "Output_REC_e4h_input.*\.slcio" <(anajob Output_REC_e4h_input$iosvc.slcio) $TEST_DIR/inputFiles/anajob_Output_REC.expected; then echo "File contents of REC slcio file are not as expected" exit 1 fi -echo "Comparing contents of Output_DST_e4h_input.slcio" -if ! diff <(anajob Output_DST_e4h_input.slcio) $TEST_DIR/inputFiles/anajob_Output_DST.expected; then +echo "Comparing contents of Output_DST_e4h_input$iosvc.slcio" +if ! diff -I "Output_DST_e4h_input.*\.slcio" <(anajob Output_DST_e4h_input$iosvc.slcio) $TEST_DIR/inputFiles/anajob_Output_DST.expected; then echo "File contents of DST slcio file are not as expected" exit 1 fi From 80e92fc262a13c97b6f6635643fcb9a98b4bd710 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Tue, 14 Jan 2025 09:12:37 +0100 Subject: [PATCH 02/30] Fix rebase --- k4MarlinWrapper/examples/clicRec_e4h_input.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index 9d8021fa..d2639ffb 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -2445,18 +2445,6 @@ "UseMCP": ["0"], } - -<<<<<<< HEAD -# Write output to EDM4hep -from Configurables import PodioOutput - -out = PodioOutput("PodioOutput", filename="my_output.root") -out.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] - - -algList.append(inp) -======= ->>>>>>> cb3c7a3 (Add support for running with IOSvc) algList.append(MyAIDAProcessor) algList.append(EventNumber) algList.append(InitDD4hep) From b6cee813250000e741cb581548c1dfaa5c3b759a Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 17 Jan 2025 11:22:18 +0100 Subject: [PATCH 03/30] Use the new overload of convertObjectParameters in k4EDM4hep2LcioConv --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index b57c80de..7f9bd8ef 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -140,11 +140,12 @@ namespace { StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { // Convert event parameters - std::optional> frame; if (m_podioDataSvc) { - frame = m_podioDataSvc->m_eventframe; + LCIO2EDM4hepConv::convertObjectParameters(the_event, m_podioDataSvc->m_eventframe); + } else { + LCIO2EDM4hepConv::convertObjectParameters( + the_event, [](const std::string& key, const auto& value) { k4FWCore::putParameter(key, value); }); } - LCIO2EDM4hepConv::convertObjectParameters(the_event, frame); // Convert Event Header outside the collections loop if (!collectionExist(edm4hep::labels::EventHeader)) { From 3c45aebf0acbda56daf06b4fc400217d0888d405 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 17 Jan 2025 11:23:35 +0100 Subject: [PATCH 04/30] Remove unused header --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 7f9bd8ef..ca59aecc 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -32,7 +32,6 @@ #include "GaudiKernel/AnyDataWrapper.h" -#include #include DECLARE_COMPONENT(Lcio2EDM4hepTool); From add8c80035a7f6fce05339bdfb9760b7d81901a9 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Jan 2025 16:46:31 +0100 Subject: [PATCH 05/30] Use the metadata service to put the PID parameters --- .../k4MarlinWrapper/converters/Lcio2EDM4hep.h | 3 +++ k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h b/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h index 79d825d5..3f3b7538 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h @@ -22,6 +22,8 @@ #include #include +#include "k4FWCore/IMetadataSvc.h" + #include "k4MarlinWrapper/converters/IEDMConverter.h" #include @@ -59,6 +61,7 @@ class Lcio2EDM4hepTool : public AlgTool, virtual public IEDMConverter { Gaudi::Property m_convertAll{this, "convertAll", true}; ServiceHandle m_eventDataSvc; + SmartIF m_metadataSvc; PodioDataSvc* m_podioDataSvc; // ********************************** diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index ca59aecc..ecfaf4d0 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -51,6 +51,12 @@ Lcio2EDM4hepTool::Lcio2EDM4hepTool(const std::string& type, const std::string& n StatusCode Lcio2EDM4hepTool::initialize() { m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); + m_metadataSvc = service("MetadataSvc", false); + if(!m_metadataSvc) { + error() << "Could not retrieve MetadataSvc" << endmsg; + return StatusCode::FAILURE; + } + return AlgTool::initialize(); } @@ -227,10 +233,7 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { } } else { for (const auto& [collName, pidInfo] : pidInfos) { - k4FWCore::putParameter(podio::collMetadataParamName(collName, edm4hep::labels::PIDAlgoName), pidInfo.algoName); - k4FWCore::putParameter(podio::collMetadataParamName(collName, edm4hep::labels::PIDAlgoType), pidInfo.algoType()); - k4FWCore::putParameter(podio::collMetadataParamName(collName, edm4hep::labels::PIDParameterNames), - pidInfo.paramNames); + m_metadataSvc->put(collName, pidInfo); } } From f7c9223f26c96f8cc6d93718e9ff0909d32f584a Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Jan 2025 16:49:08 +0100 Subject: [PATCH 06/30] Don't store the metadata frame in an optional --- k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index cf5249c1..cc09a90a 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -357,10 +357,7 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s lcio::LCEventImpl* lcio_event, CollectionPairMappings& collection_pairs, std::vector& pidCollections, std::vector& dQdxCollections) { - std::optional> metadata; - if (m_podioDataSvc) { - metadata = m_podioDataSvc->getMetaDataFrame(); - } + const auto& metadata = m_podioDataSvc->getMetaDataFrame(); const auto collPtr = getEDM4hepCollection(e4h_coll_name); const auto fulltype = collPtr->getValueTypeName(); @@ -398,12 +395,11 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s std::optional> maybeParamNames; if (m_podioDataSvc) { - const auto& frame = metadata.value().get(); maybeAlgoName = - frame.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName)); + metadata.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName)); maybeAlgoType = - frame.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType)); - maybeParamNames = frame.getParameter>( + metadata.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType)); + maybeParamNames = metadata.getParameter>( podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames)); } else { From 2f2504ef4155eecc758d880ea248263354b29cef Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Jan 2025 16:57:37 +0100 Subject: [PATCH 07/30] Fix pre-commit --- k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index cc09a90a..4718a233 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -358,8 +358,8 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s std::vector& pidCollections, std::vector& dQdxCollections) { const auto& metadata = m_podioDataSvc->getMetaDataFrame(); - const auto collPtr = getEDM4hepCollection(e4h_coll_name); - const auto fulltype = collPtr->getValueTypeName(); + const auto collPtr = getEDM4hepCollection(e4h_coll_name); + const auto fulltype = collPtr->getValueTypeName(); debug() << "Converting type " << fulltype << " from input " << e4h_coll_name << endmsg; From 4897a60f05695b4937b6f503366fa0db8a924b62 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Jan 2025 18:30:38 +0100 Subject: [PATCH 08/30] Use the metadata svc in EDM4hep2LCIO --- .../k4MarlinWrapper/converters/EDM4hep2Lcio.h | 4 +++ .../src/components/EDM4hep2Lcio.cpp | 30 +++++++------------ .../src/components/Lcio2EDM4hep.cpp | 10 +++---- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h index c0d7658a..c579e327 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h @@ -21,6 +21,8 @@ #include "k4MarlinWrapper/converters/IEDMConverter.h" +#include "k4FWCore/IMetadataSvc.h" + #include "k4EDM4hep2LcioConv/k4EDM4hep2LcioConv.h" #include @@ -31,6 +33,7 @@ #include class PodioDataSvc; +class MetadataSvc; template using ObjMapT = k4EDM4hep2LcioConv::VecMapT; @@ -64,6 +67,7 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { PodioDataSvc* m_podioDataSvc; ServiceHandle m_eventDataSvc; + SmartIF m_metadataSvc; std::vector m_collectionNames; std::map m_idToName; diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index 4718a233..5992691b 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -23,6 +23,7 @@ #include "UTIL/PIDHandler.h" #include "edm4hep/Constants.h" +#include "edm4hep/utils/ParticleIDUtils.h" #include "k4FWCore/DataHandle.h" #include "k4FWCore/FunctionalUtils.h" @@ -71,6 +72,12 @@ StatusCode EDM4hep2LcioTool::initialize() { m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); + m_metadataSvc = service("MetadataSvc", false); + if (!m_metadataSvc) { + error() << "Could not retrieve MetadataSvc" << endmsg; + return StatusCode::FAILURE; + } + return AlgTool::initialize(); } @@ -390,30 +397,13 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s } else if (fulltype == "edm4hep::EventHeader") { convertEventHeader(e4h_coll_name, lcio_event); } else if (fulltype == "edm4hep::ParticleID") { - std::optional maybeAlgoName; - std::optional maybeAlgoType; - std::optional> maybeParamNames; - + std::optional pidInfo; if (m_podioDataSvc) { - maybeAlgoName = - metadata.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName)); - maybeAlgoType = - metadata.getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType)); - maybeParamNames = metadata.getParameter>( - podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames)); - + pidInfo = edm4hep::utils::PIDHandler::getAlgoInfo(metadata, e4h_coll_name); } else { - maybeAlgoName = k4FWCore::getParameter( - podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName)); - maybeAlgoType = - k4FWCore::getParameter(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType)); - maybeParamNames = k4FWCore::getParameter>( - podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames)); + pidInfo = m_metadataSvc->get(e4h_coll_name); } - edm4hep::utils::ParticleIDMeta pidInfo{std::move(maybeAlgoName.value()), maybeAlgoType.value(), - maybeParamNames.value()}; - pidCollections.emplace_back(e4h_coll_name, static_cast(collPtr), pidInfo); } else if (fulltype == "edm4hep::RecDqDx") { dQdxCollections.emplace_back(e4h_coll_name, static_cast(collPtr)); diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index ecfaf4d0..024e31a3 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -71,11 +71,11 @@ bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { if (m_podioDataSvc) { collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); } - for (const auto& name : collections) { - if (name == collection_name) { - debug() << "Collection named " << name << " already registered, skipping conversion." << endmsg; - return true; - } + else { + } + if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) { + debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg; + return true; } return false; } From 27baccb1106269c0d7f412f4af42a66e577723fa Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 22 Jan 2025 20:31:24 +0100 Subject: [PATCH 09/30] Fix pre-commit --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 024e31a3..8e031350 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -52,7 +52,7 @@ StatusCode Lcio2EDM4hepTool::initialize() { m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); m_metadataSvc = service("MetadataSvc", false); - if(!m_metadataSvc) { + if (!m_metadataSvc) { error() << "Could not retrieve MetadataSvc" << endmsg; return StatusCode::FAILURE; } @@ -70,8 +70,7 @@ bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { //TODO: if (m_podioDataSvc) { collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); - } - else { + } else { } if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) { debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg; From 8cbbdc6d6b85d13e13ae47403842fc0f49fcdb8f Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Jan 2025 09:35:59 +0100 Subject: [PATCH 10/30] Only complain about the metadata service when using IOSvc --- k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp | 2 +- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index 5992691b..01478c95 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -73,7 +73,7 @@ StatusCode EDM4hep2LcioTool::initialize() { m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); m_metadataSvc = service("MetadataSvc", false); - if (!m_metadataSvc) { + if (!m_podioDataSvc && !m_metadataSvc) { error() << "Could not retrieve MetadataSvc" << endmsg; return StatusCode::FAILURE; } diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 8e031350..abc47772 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -52,7 +52,7 @@ StatusCode Lcio2EDM4hepTool::initialize() { m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); m_metadataSvc = service("MetadataSvc", false); - if (!m_metadataSvc) { + if (!m_podioDataSvc && !m_metadataSvc) { error() << "Could not retrieve MetadataSvc" << endmsg; return StatusCode::FAILURE; } From 64627007c35c6bf3c5dcbc33042e5bbeea87dabb Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Jan 2025 09:36:31 +0100 Subject: [PATCH 11/30] Add shared utilities to retrieve collections from the store --- k4MarlinWrapper/CMakeLists.txt | 9 +++ .../k4MarlinWrapper/converters/EDM4hep2Lcio.h | 2 - .../src/components/EDM4hep2Lcio.cpp | 52 ++------------- .../src/components/Lcio2EDM4hep.cpp | 3 + k4MarlinWrapper/src/components/StoreUtils.cpp | 66 +++++++++++++++++++ k4MarlinWrapper/src/components/StoreUtils.h | 8 +++ 6 files changed, 91 insertions(+), 49 deletions(-) create mode 100644 k4MarlinWrapper/src/components/StoreUtils.cpp create mode 100644 k4MarlinWrapper/src/components/StoreUtils.h diff --git a/k4MarlinWrapper/CMakeLists.txt b/k4MarlinWrapper/CMakeLists.txt index 7d2a3784..33e1029a 100644 --- a/k4MarlinWrapper/CMakeLists.txt +++ b/k4MarlinWrapper/CMakeLists.txt @@ -54,6 +54,13 @@ target_include_directories(MarlinWrapper PUBLIC ${LCIO_INCLUDE_DIRS} ) +# k4MarlinWrapperUtils +gaudi_add_library(k4MarlinWrapperUtils SHARED + SOURCES + src/components/StoreUtils.cpp + LINK k4FWCore::k4FWCore +) + # EDM4hep2lcio gaudi_add_module(EDM4hep2Lcio SOURCES @@ -64,6 +71,7 @@ gaudi_add_module(EDM4hep2Lcio ${LCIO_LIBRARIES} ${Marlin_LIBRARIES} EDM4HEP::edm4hep + k4MarlinWrapperUtils ) target_include_directories(EDM4hep2Lcio PUBLIC @@ -80,6 +88,7 @@ gaudi_add_module(Lcio2EDM4hep EDM4HEP::edm4hep k4FWCore::k4FWCore k4EDM4hep2LcioConv::k4EDM4hep2LcioConv + k4MarlinWrapperUtils ) target_include_directories(Lcio2EDM4hep PUBLIC diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h index c579e327..520a4da4 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h @@ -116,8 +116,6 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { std::vector& pidCollections, std::vector& dQdxCollections); - StatusCode getAvailableCollectionsFromStore(); - /// Get an EDM4hep collection by name, consulting either the podio based data /// svc or the IOSvc podio::CollectionBase* getEDM4hepCollection(const std::string& name) const; diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index 01478c95..c698a784 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -19,6 +19,7 @@ #include "k4MarlinWrapper/converters/EDM4hep2Lcio.h" #include #include "GlobalConvertedObjectsMap.h" +#include "StoreUtils.h" #include "UTIL/PIDHandler.h" @@ -317,48 +318,6 @@ podio::CollectionBase* EDM4hep2LcioTool::getEDM4hepCollection(const std::string& throw GaudiException("Collection could not be casted to the expected type", name(), StatusCode::FAILURE); } -StatusCode EDM4hep2LcioTool::getAvailableCollectionsFromStore() { - SmartIF mgr; - mgr = evtSvc(); - - SmartDataPtr root(evtSvc(), "/Event"); - if (!root) { - error() << "Failed to retrieve root object /Event" << endmsg; - return StatusCode::FAILURE; - } - - auto pObj = root->registry(); - if (!pObj) { - error() << "Failed to retrieve the root registry object" << endmsg; - return StatusCode::FAILURE; - } - std::vector leaves; - StatusCode sc = mgr->objectLeaves(pObj, leaves); - if (!sc.isSuccess()) { - error() << "Failed to retrieve object leaves" << endmsg; - return StatusCode::FAILURE; - } - for (const auto& pReg : leaves) { - if (pReg->name() == k4FWCore::frameLocation) { - continue; - } - DataObject* p; - sc = m_eventDataSvc->retrieveObject("/Event" + pReg->name(), p); - if (sc.isFailure()) { - error() << "Could not retrieve object " << pReg->name() << " from the EventStore" << endmsg; - return sc; - } - auto wrapper = dynamic_cast>*>(p); - if (!wrapper) { - continue; - } - // Remove the leading / - m_collectionNames.push_back(pReg->name().substr(1, pReg->name().size() - 1)); - m_idToName.emplace(wrapper->getData()->getID(), pReg->name()); - } - return StatusCode::SUCCESS; -} - // Select the appropriate method to convert a collection given its type void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::string& lcio_coll_name, lcio::LCEventImpl* lcio_event, CollectionPairMappings& collection_pairs, @@ -430,11 +389,10 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { edmEvent = m_podioDataSvc->getEventFrame(); m_collectionNames = edmEvent.value().get().getAvailableCollections(); } else if (m_collectionNames.empty()) { - auto sc = getAvailableCollectionsFromStore(); - if (sc.isFailure()) { - warning() << "Could not retrieve available collections from the EventStore" << endmsg; - return sc; - } + std::optional> idToNameOpt(std::move(m_idToName)); + auto collections = getAvailableCollectionsFromStore(this, idToNameOpt); + m_idToName = std::move(idToNameOpt.value()); + m_collectionNames.insert(m_collectionNames.end(), collections.begin(), collections.end()); } // Start off with the pre-defined collection name mappings auto collsToConvert{m_collNames.value()}; diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index abc47772..f92136b5 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -18,6 +18,7 @@ */ #include "k4MarlinWrapper/converters/Lcio2EDM4hep.h" #include "GlobalConvertedObjectsMap.h" +#include "StoreUtils.h" #include #include @@ -71,6 +72,8 @@ bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { if (m_podioDataSvc) { collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); } else { + std::optional> dummy = std::nullopt; + getAvailableCollectionsFromStore(this, dummy, true); } if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) { debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg; diff --git a/k4MarlinWrapper/src/components/StoreUtils.cpp b/k4MarlinWrapper/src/components/StoreUtils.cpp new file mode 100644 index 00000000..b06f9d0b --- /dev/null +++ b/k4MarlinWrapper/src/components/StoreUtils.cpp @@ -0,0 +1,66 @@ +#include "GaudiKernel/AlgTool.h" +#include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/IDataManagerSvc.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include "GaudiKernel/SmartDataPtr.h" + +#include "StoreUtils.h" +#include "podio/Frame.h" + +#include "k4FWCore/FunctionalUtils.h" + +#include +#include + +std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, + std::optional>& idToName, + bool returnFrameCollections + ) { + std::vector collectionNames; + + SmartIF mgr; + mgr = thisClass->evtSvc(); + + SmartDataPtr root(thisClass->evtSvc(), "/Event"); + if (!root) { + thisClass->error() << "Failed to retrieve root object /Event" << endmsg; + } + + auto pObj = root->registry(); + if (!pObj) { + thisClass->error() << "Failed to retrieve the root registry object" << endmsg; + } + std::vector leaves; + StatusCode sc = mgr->objectLeaves(pObj, leaves); + if (!sc.isSuccess()) { + thisClass->error() << "Failed to retrieve object leaves" << endmsg; + } + for (const auto& pReg : leaves) { + if (pReg->name() == k4FWCore::frameLocation) { + if (!returnFrameCollections) + continue; + auto wrapper = dynamic_cast*>(pReg->object()); + if (!wrapper) { + throw std::runtime_error("Could not cast object to Frame"); + } + for (const auto& name : wrapper->getData().getAvailableCollections()) { + collectionNames.push_back(name); + } + } + DataObject* p; + sc = thisClass->evtSvc()->retrieveObject("/Event" + pReg->name(), p); + if (sc.isFailure()) { + thisClass->error() << "Could not retrieve object " << pReg->name() << " from the EventStore" << endmsg; + } + auto wrapper = dynamic_cast>*>(p); + if (!wrapper) { + continue; + } + // Remove the leading / + collectionNames.push_back(pReg->name().substr(1, pReg->name().size() - 1)); + if (idToName) { + idToName->emplace(wrapper->getData()->getID(), pReg->name()); + } + } + return collectionNames; +} diff --git a/k4MarlinWrapper/src/components/StoreUtils.h b/k4MarlinWrapper/src/components/StoreUtils.h new file mode 100644 index 00000000..7f993326 --- /dev/null +++ b/k4MarlinWrapper/src/components/StoreUtils.h @@ -0,0 +1,8 @@ +#include "GaudiKernel/AlgTool.h" + +#include +#include + +std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, + std::optional>& idToName, + bool returnFrameCollections = false); From 7b41e442e5d481456e5ff8bdc07fe65b2ab82238 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Jan 2025 09:49:34 +0100 Subject: [PATCH 12/30] Fix --- k4MarlinWrapper/examples/clicRec_e4h_input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index d2639ffb..e6d4f880 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -54,7 +54,7 @@ "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") ) iosvc.Output = "my_output.root" - iosvc.outputDstName = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"] + iosvc.outputCommands = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"] else: evtsvc = k4DataSvc("EventDataSvc") evtsvc.input = os.path.join( From ce365c8f6deacb2e6391c6faba071f1bb033510f Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Jan 2025 09:53:35 +0100 Subject: [PATCH 13/30] Fix commands --- k4MarlinWrapper/examples/clicRec_e4h_input.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index e6d4f880..15841efa 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -54,7 +54,7 @@ "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") ) iosvc.Output = "my_output.root" - iosvc.outputCommands = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"] + iosvc.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] else: evtsvc = k4DataSvc("EventDataSvc") evtsvc.input = os.path.join( @@ -65,7 +65,7 @@ inp.OutputLevel = DEBUG out = PodioOutput("PodioOutput", filename="my_output.root") - out.outputCommands = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"] + out.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] MyAIDAProcessor = MarlinProcessorWrapper("MyAIDAProcessor") MyAIDAProcessor.OutputLevel = WARNING From 7764db16310e8c0bb82ba947001f92329c720049 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Jan 2025 09:58:10 +0100 Subject: [PATCH 14/30] Fix pre-commit --- .../src/components/EDM4hep2Lcio.cpp | 4 ++-- k4MarlinWrapper/src/components/StoreUtils.cpp | 23 ++++++++++++++++--- k4MarlinWrapper/src/components/StoreUtils.h | 18 +++++++++++++++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index c698a784..c1a36587 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -390,8 +390,8 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { m_collectionNames = edmEvent.value().get().getAvailableCollections(); } else if (m_collectionNames.empty()) { std::optional> idToNameOpt(std::move(m_idToName)); - auto collections = getAvailableCollectionsFromStore(this, idToNameOpt); - m_idToName = std::move(idToNameOpt.value()); + auto collections = getAvailableCollectionsFromStore(this, idToNameOpt); + m_idToName = std::move(idToNameOpt.value()); m_collectionNames.insert(m_collectionNames.end(), collections.begin(), collections.end()); } // Start off with the pre-defined collection name mappings diff --git a/k4MarlinWrapper/src/components/StoreUtils.cpp b/k4MarlinWrapper/src/components/StoreUtils.cpp index b06f9d0b..cf36f76e 100644 --- a/k4MarlinWrapper/src/components/StoreUtils.cpp +++ b/k4MarlinWrapper/src/components/StoreUtils.cpp @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "GaudiKernel/AlgTool.h" #include "GaudiKernel/AnyDataWrapper.h" #include "GaudiKernel/IDataManagerSvc.h" @@ -9,13 +27,12 @@ #include "k4FWCore/FunctionalUtils.h" -#include #include +#include std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, std::optional>& idToName, - bool returnFrameCollections - ) { + bool returnFrameCollections) { std::vector collectionNames; SmartIF mgr; diff --git a/k4MarlinWrapper/src/components/StoreUtils.h b/k4MarlinWrapper/src/components/StoreUtils.h index 7f993326..9bad7d0a 100644 --- a/k4MarlinWrapper/src/components/StoreUtils.h +++ b/k4MarlinWrapper/src/components/StoreUtils.h @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "GaudiKernel/AlgTool.h" #include From b4794a7d6721311049d8a774dfb052abb83f2d5e Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 24 Jan 2025 10:02:00 +0100 Subject: [PATCH 15/30] Remove TODO and add a comment --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 1 - k4MarlinWrapper/src/components/StoreUtils.h | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index f92136b5..5512de88 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -68,7 +68,6 @@ StatusCode Lcio2EDM4hepTool::finalize() { return AlgTool::finalize(); } // ********************************** bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { std::vector collections; - //TODO: if (m_podioDataSvc) { collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); } else { diff --git a/k4MarlinWrapper/src/components/StoreUtils.h b/k4MarlinWrapper/src/components/StoreUtils.h index 9bad7d0a..c039b4af 100644 --- a/k4MarlinWrapper/src/components/StoreUtils.h +++ b/k4MarlinWrapper/src/components/StoreUtils.h @@ -21,6 +21,9 @@ #include #include +// This functionality is used in the Writer from k4FWCore and is reimplemented +// here with some additions to make it useful for converting both from EDM4hep +// to LCIO and vice versa std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, std::optional>& idToName, bool returnFrameCollections = false); From 5e85df2e1c818360d26e85e8b80568d8ef445ff1 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 08:45:05 +0100 Subject: [PATCH 16/30] Address comments in the PR --- k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h | 7 ++++--- k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp | 1 - k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 4 ++-- k4MarlinWrapper/src/components/StoreUtils.cpp | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h index 520a4da4..55ceafde 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h @@ -21,8 +21,6 @@ #include "k4MarlinWrapper/converters/IEDMConverter.h" -#include "k4FWCore/IMetadataSvc.h" - #include "k4EDM4hep2LcioConv/k4EDM4hep2LcioConv.h" #include @@ -33,7 +31,8 @@ #include class PodioDataSvc; -class MetadataSvc; +class IDataProviderSvc; +class IMetadataSvc; template using ObjMapT = k4EDM4hep2LcioConv::VecMapT; @@ -66,7 +65,9 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { Gaudi::Property m_convertAll{this, "convertAll", true}; PodioDataSvc* m_podioDataSvc; + // EventDataSvc that is used together with IOSvc ServiceHandle m_eventDataSvc; + // Metadata service from k4FWCore that is used together with IOSvc SmartIF m_metadataSvc; std::vector m_collectionNames; std::map m_idToName; diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index c1a36587..7eae7e78 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -34,7 +34,6 @@ #include "GaudiKernel/AnyDataWrapper.h" #include "GaudiKernel/IDataManagerSvc.h" #include "GaudiKernel/IDataProviderSvc.h" -#include "GaudiKernel/SmartDataPtr.h" #include #include diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 5512de88..6369bb6b 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -147,9 +147,9 @@ namespace { StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { // Convert event parameters if (m_podioDataSvc) { - LCIO2EDM4hepConv::convertObjectParameters(the_event, m_podioDataSvc->m_eventframe); + LCIO2EDM4hepConv::convertObjectParameters(the_event, m_podioDataSvc->m_eventframe); } else { - LCIO2EDM4hepConv::convertObjectParameters( + LCIO2EDM4hepConv::convertObjectParameters( the_event, [](const std::string& key, const auto& value) { k4FWCore::putParameter(key, value); }); } diff --git a/k4MarlinWrapper/src/components/StoreUtils.cpp b/k4MarlinWrapper/src/components/StoreUtils.cpp index cf36f76e..33758a54 100644 --- a/k4MarlinWrapper/src/components/StoreUtils.cpp +++ b/k4MarlinWrapper/src/components/StoreUtils.cpp @@ -16,13 +16,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "StoreUtils.h" + #include "GaudiKernel/AlgTool.h" #include "GaudiKernel/AnyDataWrapper.h" #include "GaudiKernel/IDataManagerSvc.h" #include "GaudiKernel/IDataProviderSvc.h" #include "GaudiKernel/SmartDataPtr.h" -#include "StoreUtils.h" #include "podio/Frame.h" #include "k4FWCore/FunctionalUtils.h" From db7165ffeee50443c5458ccbae4cad7777f57671 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 08:56:18 +0100 Subject: [PATCH 17/30] Fix pre-commit --- k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h index 55ceafde..ea5b6699 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h @@ -64,7 +64,7 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { Gaudi::Property> m_collNames{this, "collNameMapping", {}}; Gaudi::Property m_convertAll{this, "convertAll", true}; - PodioDataSvc* m_podioDataSvc; + PodioDataSvc* m_podioDataSvc; // EventDataSvc that is used together with IOSvc ServiceHandle m_eventDataSvc; // Metadata service from k4FWCore that is used together with IOSvc From 38b167ededcfc00c8c9727f57bf9e008f668330a Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 09:02:05 +0100 Subject: [PATCH 18/30] Use a different Gaudi output file with IOSvc and without --- k4MarlinWrapper/examples/clicRec_e4h_input.py | 5 +++-- test/scripts/clicRec_e4h_input.sh | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index 15841efa..6d3e9767 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -44,6 +44,7 @@ ) parser.add_argument("--rec-output", help="Output file name for the REC file") parser.add_argument("--dst-output", help="Output file name for the DST file") +parser.add_argument("--gaudi-output", help="Output file name for the Gaudi file") args = parser.parse_known_args()[0] @@ -53,7 +54,7 @@ iosvc.Input = os.path.join( "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") ) - iosvc.Output = "my_output.root" + iosvc.Output = args.gaudi_output iosvc.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] else: evtsvc = k4DataSvc("EventDataSvc") @@ -64,7 +65,7 @@ inp = PodioInput("InputReader") inp.OutputLevel = DEBUG - out = PodioOutput("PodioOutput", filename="my_output.root") + out = PodioOutput("PodioOutput", filename=args.gaudi_output) out.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] MyAIDAProcessor = MarlinProcessorWrapper("MyAIDAProcessor") diff --git a/test/scripts/clicRec_e4h_input.sh b/test/scripts/clicRec_e4h_input.sh index 5c9d6916..07a4d3ee 100755 --- a/test/scripts/clicRec_e4h_input.sh +++ b/test/scripts/clicRec_e4h_input.sh @@ -35,10 +35,10 @@ else echo "Wrong argument $2" return 1 fi -k4run $EXAMPLE_DIR/clicRec_e4h_input.py ${file_arg} --rec-output Output_REC_e4h_input$iosvc.slcio --dst-output Output_DST_e4h_input$iosvc.slcio +k4run $EXAMPLE_DIR/clicRec_e4h_input.py ${file_arg} --rec-output Output_REC_e4h_input$iosvc.slcio --dst-output Output_DST_e4h_input$iosvc.slcio --gaudi-output my_output$iosvc.root input_num_events=$(python $TEST_DIR/python/root_num_events.py $1) -output_num_events=$(python $TEST_DIR/python/root_num_events.py my_output.root) +output_num_events=$(python $TEST_DIR/python/root_num_events.py my_output$iosvc.root) # First check do we have the same number of events in input and output if [ "$input_num_events" != "$output_num_events" ]; then From 7f7e4deb9a33503db440aeacdad21c7504ee0e10 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 09:33:18 +0100 Subject: [PATCH 19/30] Add default arguments for clicRec_e4h_input --- k4MarlinWrapper/examples/clicRec_e4h_input.py | 12 +++++++++--- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 1 - 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index 6d3e9767..1eabb227 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -42,9 +42,15 @@ parser.add_argument( "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" ) -parser.add_argument("--rec-output", help="Output file name for the REC file") -parser.add_argument("--dst-output", help="Output file name for the DST file") -parser.add_argument("--gaudi-output", help="Output file name for the Gaudi file") +parser.add_argument( + "--rec-output", default="Output_REC_e4h_input.slcio", help="Output file name for the REC file" +) +parser.add_argument( + "--dst-output", default="Output_DST_e4h_input.slcio", help="Output file name for the DST file" +) +parser.add_argument( + "--gaudi-output", default="my_output.root", help="Output file name for the Gaudi file" +) args = parser.parse_known_args()[0] diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 6369bb6b..5be1d5c4 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -226,7 +226,6 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { } // Set the ParticleID meta information - // TODO: Clean up if (m_podioDataSvc) { auto& metadataFrame = m_podioDataSvc->getMetaDataFrame(); for (const auto& [collName, pidInfo] : pidInfos) { From 62689ede53b12254170aa63e1c702fa5638d07f3 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 10:54:47 +0100 Subject: [PATCH 20/30] Fix parameter conversion --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 5be1d5c4..aafbb155 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include "GaudiKernel/AnyDataWrapper.h" @@ -149,8 +150,14 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { if (m_podioDataSvc) { LCIO2EDM4hepConv::convertObjectParameters(the_event, m_podioDataSvc->m_eventframe); } else { - LCIO2EDM4hepConv::convertObjectParameters( - the_event, [](const std::string& key, const auto& value) { k4FWCore::putParameter(key, value); }); + DataObject* p; + StatusCode code = m_eventDataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p); + if (code.isSuccess()) { + auto* frameWrapper = dynamic_cast*>(p); + LCIO2EDM4hepConv::convertObjectParameters(the_event, frameWrapper->getData()); + } else { + warning() << "Could not retrieve the event frame; event parameters will not be converted" << endmsg; + } } // Convert Event Header outside the collections loop From 9ac303b455fe5527fb5ceaa8450b438f0432c7cd Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 11:04:32 +0100 Subject: [PATCH 21/30] Fix pre-commit and add a new test --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 2 +- test/CMakeLists.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index aafbb155..14ccdea9 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -28,9 +28,9 @@ #include #include +#include #include #include -#include #include "GaudiKernel/AnyDataWrapper.h" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5ca42b8a..cc8b0d02 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -79,6 +79,7 @@ set_tests_properties ( same_num_io # Test clicReconstruction with EDM4hep input and output ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --no-iosvc") ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input_iosvc COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --iosvc") +add_test( clicRec_edm4hep_input_compare_output bash -c "diff <(podio-dump CLICPerformance/clicConfig/my_output.root | grep -v "input file:") <(podio-dump CLICPerformance/clicConfig/my_output_iosvc.root | grep -v "input file:")") # Run clicReconstruction sequence with LCIO input and output, no converters, with inter-event parallelism ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_lcio_mt COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_lcio_mt.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/testSimulation.slcio}") @@ -115,4 +116,9 @@ set_tests_properties( DEPENDS CLICPerformance_setup ) +set_tests_properties( + clicRec_edm4hep_input_compare_output + PROPERTIES + DEPENDS clicRec_edm4hep_input clicRec_edm4hep_input_iosvc +) From 1940dd23de4ba4959e1a9a9ee4e8aa0d06f6eacd Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 11:09:37 +0100 Subject: [PATCH 22/30] Fix syntax --- test/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cc8b0d02..828b2498 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -79,7 +79,7 @@ set_tests_properties ( same_num_io # Test clicReconstruction with EDM4hep input and output ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --no-iosvc") ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input_iosvc COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --iosvc") -add_test( clicRec_edm4hep_input_compare_output bash -c "diff <(podio-dump CLICPerformance/clicConfig/my_output.root | grep -v "input file:") <(podio-dump CLICPerformance/clicConfig/my_output_iosvc.root | grep -v "input file:")") +add_test( clicRec_edm4hep_input_compare_output bash -c "diff <(podio-dump CLICPerformance/clicConfig/my_output.root | grep -v 'input file:') <(podio-dump CLICPerformance/clicConfig/my_output_iosvc.root | grep -v 'input file:')") # Run clicReconstruction sequence with LCIO input and output, no converters, with inter-event parallelism ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_lcio_mt COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_lcio_mt.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/testSimulation.slcio}") @@ -119,6 +119,6 @@ set_tests_properties( set_tests_properties( clicRec_edm4hep_input_compare_output PROPERTIES - DEPENDS clicRec_edm4hep_input clicRec_edm4hep_input_iosvc + DEPENDS "clicRec_edm4hep_input;clicRec_edm4hep_input_iosvc" ) From cf1870161b9f46050091d525c38a35c1ff897218 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 11:36:25 +0100 Subject: [PATCH 23/30] Add comments, improve warning and add a throw --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 2 +- k4MarlinWrapper/src/components/StoreUtils.cpp | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 14ccdea9..7044fbb7 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -156,7 +156,7 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { auto* frameWrapper = dynamic_cast*>(p); LCIO2EDM4hepConv::convertObjectParameters(the_event, frameWrapper->getData()); } else { - warning() << "Could not retrieve the event frame; event parameters will not be converted" << endmsg; + warning() << "Could not retrieve the event frame; event parameters will not be converted. This is a known limitation when running with IOSvc without an input file." << endmsg; } } diff --git a/k4MarlinWrapper/src/components/StoreUtils.cpp b/k4MarlinWrapper/src/components/StoreUtils.cpp index 33758a54..ac7a95b5 100644 --- a/k4MarlinWrapper/src/components/StoreUtils.cpp +++ b/k4MarlinWrapper/src/components/StoreUtils.cpp @@ -28,9 +28,14 @@ #include "k4FWCore/FunctionalUtils.h" +#include #include #include +// This is a reimplementation of functionality to retrieve collections from the +// store that can be found in Writer.cpp in k4FWCore with some modifications +// that are specific to the usage of this function in the converters, like +// returning also a map from collection ID to collection name std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, std::optional>& idToName, bool returnFrameCollections) { @@ -46,12 +51,12 @@ std::vector getAvailableCollectionsFromStore(const AlgTool* auto pObj = root->registry(); if (!pObj) { - thisClass->error() << "Failed to retrieve the root registry object" << endmsg; + throw std::runtime_error("Failed to retrieve the root registry object"); } std::vector leaves; StatusCode sc = mgr->objectLeaves(pObj, leaves); if (!sc.isSuccess()) { - thisClass->error() << "Failed to retrieve object leaves" << endmsg; + throw std::runtime_error("Failed to retrieve object leaves"); } for (const auto& pReg : leaves) { if (pReg->name() == k4FWCore::frameLocation) { From f62a519f6986a5f18f4648911ec6263ed9d4bfc0 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 13:30:46 +0100 Subject: [PATCH 24/30] Improve warning --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 7044fbb7..30f395e0 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -156,7 +156,9 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { auto* frameWrapper = dynamic_cast*>(p); LCIO2EDM4hepConv::convertObjectParameters(the_event, frameWrapper->getData()); } else { - warning() << "Could not retrieve the event frame; event parameters will not be converted. This is a known limitation when running with IOSvc without an input file." << endmsg; + warning() << "Could not retrieve the event frame; event parameters will not be converted. This is a known " + "limitation when running with IOSvc without an input file." + << endmsg; } } From a1c7d66ff4d9a62740f5596e8424caa763d37b85 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Thu, 30 Jan 2025 21:42:41 +0100 Subject: [PATCH 25/30] Add a new test with a functional taking input from a processor --- test/CMakeLists.txt | 2 ++ test/gaudi_opts/test_global_converter_maps.py | 21 +++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 828b2498..a2cd86c8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,6 +20,7 @@ limitations under the License. gaudi_add_module(GaudiTestAlgorithms SOURCES src/MCRecoLinkChecker.cc + src/MCRecoLinkCheckerFunctional.cc src/PseudoRecoAlgorithm.cc src/TrivialMCRecoLinker.cc LINK @@ -91,6 +92,7 @@ add_test( clic_geo_test ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/geoTest_ # multiple processors ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --EventDataSvc.input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps_iosvc COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --iosvc --IOSvc.Input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) +ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps_iosvc_functional COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --iosvc --use-functional-checker --IOSvc.Input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) ExternalData_Add_Test( marlinwrapper_tests NAME link_conversion_edm4hep_to_lcio diff --git a/test/gaudi_opts/test_global_converter_maps.py b/test/gaudi_opts/test_global_converter_maps.py index 846734d0..5d344713 100644 --- a/test/gaudi_opts/test_global_converter_maps.py +++ b/test/gaudi_opts/test_global_converter_maps.py @@ -40,9 +40,15 @@ parser.add_argument( "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" ) +parser.add_argument( + "--use-functional-checker", action="store_true", default=False, help="Use functional checker" +) args = parser.parse_known_args()[0] +if args.use_functional_checker: + from Configurables import MCRecoLinkCheckerFunctional as MCRecoLinkChecker + if args.iosvc: evtsvc = EventDataSvc("EventDataSvc") else: @@ -50,7 +56,10 @@ if args.iosvc: iosvc = IOSvc() - iosvc.Output = "global_converter_maps_iosvc.root" + if not args.use_functional_checker: + iosvc.Output = "global_converter_maps_iosvc.root" + else: + iosvc.Output = "global_converter_maps_iosvc_functional.root" else: podioInput = PodioInput("InputReader") podioInput.collections = ["MCParticles"] @@ -88,9 +97,13 @@ TrivialMCTruthLinkerProc.Lcio2EDM4hepTool = mcTruthConverter mcLinkChecker = MCRecoLinkChecker("MCRecoLinkChecker") -mcLinkChecker.InputMCRecoLinks = "TrivialMCRecoLinks" -mcLinkChecker.InputMCs = "MCParticles" -mcLinkChecker.InputRecos = "PseudoRecoParticles" +mcLinkChecker.InputMCRecoLinks = ( + "TrivialMCRecoLinks" if not args.use_functional_checker else ["TrivialMCRecoLinks"] +) +mcLinkChecker.InputMCs = "MCParticles" if not args.use_functional_checker else ["MCParticles"] +mcLinkChecker.InputRecos = ( + "PseudoRecoParticles" if not args.use_functional_checker else ["PseudoRecoParticles"] +) mcLinkChecker.OutputLevel = DEBUG algList = [ From 0030ee8b4a74f40565b4299ecaec59d5e866a4e0 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 31 Jan 2025 08:18:03 +0100 Subject: [PATCH 26/30] Add missing files --- test/src/MCRecoLinkCheckerFunctional.cc | 68 +++++++++++++++++++++++++ test/src/MCRecoLinkCheckerFunctional.h | 38 ++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 test/src/MCRecoLinkCheckerFunctional.cc create mode 100644 test/src/MCRecoLinkCheckerFunctional.h diff --git a/test/src/MCRecoLinkCheckerFunctional.cc b/test/src/MCRecoLinkCheckerFunctional.cc new file mode 100644 index 00000000..026b85dc --- /dev/null +++ b/test/src/MCRecoLinkCheckerFunctional.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "MCRecoLinkCheckerFunctional.h" + +MCRecoLinkCheckerFunctional::MCRecoLinkCheckerFunctional(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, + {KeyValues("InputMCRecoLinks", {"InputMCRecoLinks"}), + KeyValues("InputMCs", {"InputMCs"}), + KeyValues("InputRecos", {"InputRecos"})}) {} + +void MCRecoLinkCheckerFunctional::operator()(const edm4hep::RecoMCParticleLinkCollection& relationColl, + const edm4hep::MCParticleCollection& mcColl, + const edm4hep::ReconstructedParticleCollection& recoColl) const { + if (relationColl.size() != mcColl.size()) { + error() << "The MCReco relation collection does not have the expected size (expected: " << relationColl.size() + << ", actual: " << mcColl.size() << ")" << endmsg; + throw std::runtime_error( + "MCRecoLinkCheckerFunctional: The MCReco relation collection does not have the expected size"); + } + + for (size_t i = 0; i < mcColl.size(); ++i) { + const auto mc = mcColl[i]; + const auto reco = recoColl[i]; + const auto relation = relationColl[i]; + + if (relation.getWeight() != i) { + error() << "Relation " << i << " does not not have the correct weight (expected: " << i + << ", actual: " << relation.getWeight() << ")" << endmsg; + throw std::runtime_error("The MCReco relation collection does not have the expected weight"); + } + + if (!(relation.getTo() == mc)) { + auto relMC = relation.getTo(); + error() << "Relation " << i + << " does not point to the correct MCParticle (expected: " << mc.getObjectID().collectionID << "|" + << mc.getObjectID().index << ", actual: " << relMC.getObjectID().collectionID << "|" + << relMC.getObjectID().index << ")" << endmsg; + throw std::runtime_error("The MCReco relation collection does not point to the correct MCParticle"); + } + + if (!(relation.getFrom() == reco)) { + auto relRec = relation.getFrom(); + error() << "Relation " << i + << " does not point to the correct RecoParticle (expected: " << reco.getObjectID().collectionID << "|" + << reco.getObjectID().index << ", actual: " << relRec.getObjectID().collectionID << "|" + << relRec.getObjectID().index << ")" << endmsg; + throw std::runtime_error("The MCReco relation collection does not point to the correct RecoParticle"); + } + } +} + +DECLARE_COMPONENT(MCRecoLinkCheckerFunctional) diff --git a/test/src/MCRecoLinkCheckerFunctional.h b/test/src/MCRecoLinkCheckerFunctional.h new file mode 100644 index 00000000..281d51bd --- /dev/null +++ b/test/src/MCRecoLinkCheckerFunctional.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef K4MARLINWRAPPER_TEST_MCRECOLINKCHECKERFUNCTIONAL_H +#define K4MARLINWRAPPER_TEST_MCRECOLINKCHECKERFUNCTIONAL_H + +#include "edm4hep/MCParticleCollection.h" +#include "edm4hep/RecoMCParticleLinkCollection.h" +#include "edm4hep/ReconstructedParticleCollection.h" + +#include "k4FWCore/Consumer.h" + +struct MCRecoLinkCheckerFunctional final + : k4FWCore::Consumer { + MCRecoLinkCheckerFunctional(const std::string& name, ISvcLocator* svcLoc); + + void operator()(const edm4hep::RecoMCParticleLinkCollection& relationColl, + const edm4hep::MCParticleCollection& mcColl, + const edm4hep::ReconstructedParticleCollection& recoColl) const override; +}; + +#endif // K4MARLINWRAPPER_TEST_MCRECOLINKCHECKERFUNCTIONAL_H From 61cbb2e469377c76a8ba5afb1a0990ed72fec726 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Fri, 31 Jan 2025 09:08:21 +0100 Subject: [PATCH 27/30] Fix pre-commit --- test/src/MCRecoLinkCheckerFunctional.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/src/MCRecoLinkCheckerFunctional.cc b/test/src/MCRecoLinkCheckerFunctional.cc index 026b85dc..6e98a8a7 100644 --- a/test/src/MCRecoLinkCheckerFunctional.cc +++ b/test/src/MCRecoLinkCheckerFunctional.cc @@ -20,8 +20,7 @@ MCRecoLinkCheckerFunctional::MCRecoLinkCheckerFunctional(const std::string& name, ISvcLocator* svcLoc) : Consumer(name, svcLoc, - {KeyValues("InputMCRecoLinks", {"InputMCRecoLinks"}), - KeyValues("InputMCs", {"InputMCs"}), + {KeyValues("InputMCRecoLinks", {"InputMCRecoLinks"}), KeyValues("InputMCs", {"InputMCs"}), KeyValues("InputRecos", {"InputRecos"})}) {} void MCRecoLinkCheckerFunctional::operator()(const edm4hep::RecoMCParticleLinkCollection& relationColl, From 618122f7e91641f261c2ca2bf9c295df83096a74 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 5 Feb 2025 11:13:20 +0100 Subject: [PATCH 28/30] Make sure to set the collections --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 30f395e0..1ac988f5 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -73,7 +73,7 @@ bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); } else { std::optional> dummy = std::nullopt; - getAvailableCollectionsFromStore(this, dummy, true); + collections = getAvailableCollectionsFromStore(this, dummy, true); } if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) { debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg; From 62de8ccc2ae24d3d76e32dfc9f41dcc72ed45732 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 5 Feb 2025 11:16:29 +0100 Subject: [PATCH 29/30] Fix pre-commit --- k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 1ac988f5..373e7ede 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -73,7 +73,7 @@ bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); } else { std::optional> dummy = std::nullopt; - collections = getAvailableCollectionsFromStore(this, dummy, true); + collections = getAvailableCollectionsFromStore(this, dummy, true); } if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) { debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg; From 9aacacc40fb9c30b795004a5499bbf5cccac6a42 Mon Sep 17 00:00:00 2001 From: jmcarcell Date: Wed, 5 Feb 2025 16:19:23 +0100 Subject: [PATCH 30/30] Bump the required version of k4FWCore and k4EDM4hep2LcioConv --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43f2bd81..deb98c52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,8 +36,8 @@ find_package(Gaudi REQUIRED) find_package(LCIO REQUIRED) find_package(Marlin REQUIRED) find_package(EDM4HEP 0.99 REQUIRED) -find_package(k4FWCore REQUIRED) -find_package(k4EDM4hep2LcioConv REQUIRED) +find_package(k4FWCore 1.2.0 REQUIRED) +find_package(k4EDM4hep2LcioConv 0.10 REQUIRED) include(CTest)