diff --git a/CMakeLists.txt b/CMakeLists.txt index 43f2bd81..deb98c52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,8 +36,8 @@ find_package(Gaudi REQUIRED) find_package(LCIO REQUIRED) find_package(Marlin REQUIRED) find_package(EDM4HEP 0.99 REQUIRED) -find_package(k4FWCore REQUIRED) -find_package(k4EDM4hep2LcioConv REQUIRED) +find_package(k4FWCore 1.2.0 REQUIRED) +find_package(k4EDM4hep2LcioConv 0.10 REQUIRED) include(CTest) diff --git a/k4MarlinWrapper/CMakeLists.txt b/k4MarlinWrapper/CMakeLists.txt index 7d2a3784..33e1029a 100644 --- a/k4MarlinWrapper/CMakeLists.txt +++ b/k4MarlinWrapper/CMakeLists.txt @@ -54,6 +54,13 @@ target_include_directories(MarlinWrapper PUBLIC ${LCIO_INCLUDE_DIRS} ) +# k4MarlinWrapperUtils +gaudi_add_library(k4MarlinWrapperUtils SHARED + SOURCES + src/components/StoreUtils.cpp + LINK k4FWCore::k4FWCore +) + # EDM4hep2lcio gaudi_add_module(EDM4hep2Lcio SOURCES @@ -64,6 +71,7 @@ gaudi_add_module(EDM4hep2Lcio ${LCIO_LIBRARIES} ${Marlin_LIBRARIES} EDM4HEP::edm4hep + k4MarlinWrapperUtils ) target_include_directories(EDM4hep2Lcio PUBLIC @@ -80,6 +88,7 @@ gaudi_add_module(Lcio2EDM4hep EDM4HEP::edm4hep k4FWCore::k4FWCore k4EDM4hep2LcioConv::k4EDM4hep2LcioConv + k4MarlinWrapperUtils ) target_include_directories(Lcio2EDM4hep PUBLIC diff --git a/k4MarlinWrapper/examples/clicRec_e4h_input.py b/k4MarlinWrapper/examples/clicRec_e4h_input.py index ee802f4e..1eabb227 100644 --- a/k4MarlinWrapper/examples/clicRec_e4h_input.py +++ b/k4MarlinWrapper/examples/clicRec_e4h_input.py @@ -19,10 +19,16 @@ import os -from Gaudi.Configuration import * +from Gaudi.Configuration import DEBUG, WARNING -from Configurables import LcioEvent, MarlinProcessorWrapper -from k4MarlinWrapper.parseConstants import * +from Configurables import MarlinProcessorWrapper +from k4MarlinWrapper.parseConstants import parseConstants + +from Configurables import Lcio2EDM4hepTool, EDM4hep2LcioTool +from Configurables import k4DataSvc, PodioInput, PodioOutput, EventDataSvc + +from k4FWCore import ApplicationMgr, IOSvc +from k4FWCore.parseArgs import parser algList = [] @@ -33,22 +39,40 @@ parseConstants(CONSTANTS) - -# For converters -from Configurables import ToolSvc, Lcio2EDM4hepTool, EDM4hep2LcioTool - - -from Configurables import k4DataSvc, PodioInput - -evtsvc = k4DataSvc("EventDataSvc") -evtsvc.input = os.path.join( - "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") +parser.add_argument( + "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" +) +parser.add_argument( + "--rec-output", default="Output_REC_e4h_input.slcio", help="Output file name for the REC file" +) +parser.add_argument( + "--dst-output", default="Output_DST_e4h_input.slcio", help="Output file name for the DST file" +) +parser.add_argument( + "--gaudi-output", default="my_output.root", help="Output file name for the Gaudi file" ) +args = parser.parse_known_args()[0] -inp = PodioInput("InputReader") -inp.OutputLevel = DEBUG +if args.iosvc: + evtsvc = EventDataSvc("EventDataSvc") + iosvc = IOSvc() + iosvc.Input = os.path.join( + "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") + ) + iosvc.Output = args.gaudi_output + iosvc.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] +else: + evtsvc = k4DataSvc("EventDataSvc") + evtsvc.input = os.path.join( + "$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root") + ) + inp = PodioInput("InputReader") + inp.OutputLevel = DEBUG + + out = PodioOutput("PodioOutput", filename=args.gaudi_output) + out.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] MyAIDAProcessor = MarlinProcessorWrapper("MyAIDAProcessor") MyAIDAProcessor.OutputLevel = WARNING @@ -1167,7 +1191,7 @@ "DropCollectionTypes": [], "FullSubsetCollections": ["EfficientMCParticles", "InefficientMCParticles"], "KeepCollectionNames": [], - "LCIOOutputFile": ["Output_REC_e4h_input.slcio"], + "LCIOOutputFile": [args.rec_output], "LCIOWriteMode": ["WRITE_NEW"], } @@ -1233,7 +1257,7 @@ "RefinedVertices", "RefinedVertices_RP", ], - "LCIOOutputFile": ["Output_DST_e4h_input.slcio"], + "LCIOOutputFile": [args.dst_output], "LCIOWriteMode": ["WRITE_NEW"], } @@ -2428,15 +2452,6 @@ "UseMCP": ["0"], } - -# Write output to EDM4hep -from Configurables import PodioOutput - -out = PodioOutput("PodioOutput", filename="my_output.root") -out.outputCommands = ["keep *", "drop RefinedVertexJets_PID_RefinedVertex"] - - -algList.append(inp) algList.append(MyAIDAProcessor) algList.append(EventNumber) algList.append(InitDD4hep) @@ -2486,8 +2501,8 @@ # # algList.append(VertexFinderUnconstrained) # Config.VertexUnconstrainedON algList.append(Output_REC) algList.append(Output_DST) -algList.append(out) -from Configurables import ApplicationMgr +if not args.iosvc: + algList = [inp] + algList + [out] ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=3, ExtSvc=[evtsvc], OutputLevel=WARNING) diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h index 4d281004..ea5b6699 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h @@ -31,6 +31,8 @@ #include class PodioDataSvc; +class IDataProviderSvc; +class IMetadataSvc; template using ObjMapT = k4EDM4hep2LcioConv::VecMapT; @@ -62,8 +64,13 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter { Gaudi::Property> m_collNames{this, "collNameMapping", {}}; Gaudi::Property m_convertAll{this, "convertAll", true}; - PodioDataSvc* m_podioDataSvc; + PodioDataSvc* m_podioDataSvc; + // EventDataSvc that is used together with IOSvc ServiceHandle m_eventDataSvc; + // Metadata service from k4FWCore that is used together with IOSvc + SmartIF m_metadataSvc; + std::vector m_collectionNames; + std::map m_idToName; void convertTracks(TrackMap& tracks_vec, const std::string& e4h_coll_name, const std::string& lcio_coll_name, lcio::LCEventImpl* lcio_event); diff --git a/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h b/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h index a72a11f3..3f3b7538 100644 --- a/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h +++ b/k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h @@ -22,6 +22,8 @@ #include #include +#include "k4FWCore/IMetadataSvc.h" + #include "k4MarlinWrapper/converters/IEDMConverter.h" #include @@ -58,7 +60,8 @@ class Lcio2EDM4hepTool : public AlgTool, virtual public IEDMConverter { Gaudi::Property> m_collNames{this, "collNameMapping", {}}; Gaudi::Property m_convertAll{this, "convertAll", true}; - ServiceHandle m_eds; + ServiceHandle m_eventDataSvc; + SmartIF m_metadataSvc; PodioDataSvc* m_podioDataSvc; // ********************************** diff --git a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp index 30b81a25..7eae7e78 100644 --- a/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp +++ b/k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp @@ -17,16 +17,25 @@ * limitations under the License. */ #include "k4MarlinWrapper/converters/EDM4hep2Lcio.h" +#include #include "GlobalConvertedObjectsMap.h" +#include "StoreUtils.h" + +#include "UTIL/PIDHandler.h" #include "edm4hep/Constants.h" +#include "edm4hep/utils/ParticleIDUtils.h" #include "k4FWCore/DataHandle.h" +#include "k4FWCore/FunctionalUtils.h" #include "k4FWCore/MetaDataHandle.h" #include "k4FWCore/PodioDataSvc.h" #include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/IDataManagerSvc.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include #include DECLARE_COMPONENT(EDM4hep2LcioTool); @@ -55,11 +64,17 @@ EDM4hep2LcioTool::EDM4hep2LcioTool(const std::string& type, const std::string& n } StatusCode EDM4hep2LcioTool::initialize() { - StatusCode sc = m_eventDataSvc.retrieve(); + StatusCode sc = m_eventDataSvc.retrieve(); + if (sc == StatusCode::FAILURE) { + error() << "Could not retrieve the EventDataSvc;" << endmsg; + return StatusCode::FAILURE; + } + m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); - if (sc == StatusCode::FAILURE) { - error() << "Error retrieving Event Data Service" << endmsg; + m_metadataSvc = service("MetadataSvc", false); + if (!m_podioDataSvc && !m_metadataSvc) { + error() << "Could not retrieve MetadataSvc" << endmsg; return StatusCode::FAILURE; } @@ -277,7 +292,7 @@ void EDM4hep2LcioTool::convertEventHeader(const std::string& e4h_coll_name, lcio podio::CollectionBase* EDM4hep2LcioTool::getEDM4hepCollection(const std::string& collName) const { DataObject* p; - auto sc = m_podioDataSvc->retrieveObject(collName, p); + auto sc = m_eventDataSvc->retrieveObject(collName, p); if (sc.isFailure()) { throw GaudiException("Collection not found", name(), StatusCode::FAILURE); } @@ -340,8 +355,14 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s } else if (fulltype == "edm4hep::EventHeader") { convertEventHeader(e4h_coll_name, lcio_event); } else if (fulltype == "edm4hep::ParticleID") { - pidCollections.emplace_back(e4h_coll_name, static_cast(collPtr), - edm4hep::utils::PIDHandler::getAlgoInfo(metadata, e4h_coll_name)); + std::optional pidInfo; + if (m_podioDataSvc) { + pidInfo = edm4hep::utils::PIDHandler::getAlgoInfo(metadata, e4h_coll_name); + } else { + pidInfo = m_metadataSvc->get(e4h_coll_name); + } + + pidCollections.emplace_back(e4h_coll_name, static_cast(collPtr), pidInfo); } else if (fulltype == "edm4hep::RecDqDx") { dQdxCollections.emplace_back(e4h_coll_name, static_cast(collPtr)); } @@ -362,8 +383,16 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s // Parse property parameters and convert the indicated collections. // Use the collection names in the parameters to read and write them StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { - const auto& edmEvent = m_podioDataSvc->getEventFrame(); - const auto collections = edmEvent.getAvailableCollections(); + std::optional> edmEvent; + if (m_collectionNames.empty() && m_podioDataSvc) { + edmEvent = m_podioDataSvc->getEventFrame(); + m_collectionNames = edmEvent.value().get().getAvailableCollections(); + } else if (m_collectionNames.empty()) { + std::optional> idToNameOpt(std::move(m_idToName)); + auto collections = getAvailableCollectionsFromStore(this, idToNameOpt); + m_idToName = std::move(idToNameOpt.value()); + m_collectionNames.insert(m_collectionNames.end(), collections.begin(), collections.end()); + } // Start off with the pre-defined collection name mappings auto collsToConvert{m_collNames.value()}; // We *always* want to convert the EventHeader @@ -372,7 +401,7 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { info() << "Converting all collections from EDM4hep to LCIO" << endmsg; // And simply add the rest, exploiting the fact that emplace will not // replace existing entries with the same key - for (const auto& name : collections) { + for (const auto& name : m_collectionNames) { collsToConvert.emplace(name, name); } } @@ -399,13 +428,39 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) { } debug() << "Event: " << lcio_event->getEventNumber() << " Run: " << lcio_event->getRunNumber() << endmsg; - // Deal with EDM4hep2LCIOConv::sortParticleIDs(pidCollections); + if (!m_podioDataSvc) { + DataObject* p; + StatusCode code = m_eventDataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p); + if (code.isSuccess()) { + auto* frame = dynamic_cast*>(p); + edmEvent = std::cref(frame->getData()); + } else { + auto frame = podio::Frame{}; + edmEvent = frame; + } + } for (const auto& pidCollMeta : pidCollections) { - const auto algoId = attachParticleIDMetaData(lcio_event, edmEvent, pidCollMeta); + auto algoId = attachParticleIDMetaData(lcio_event, edmEvent.value(), pidCollMeta); if (!algoId.has_value()) { - warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name - << " for setting consistent metadata" << endmsg; + // Now go over the collections that have been produced in a functional algorithm (if any) + bool found = false; + if (!m_podioDataSvc) { + const auto id = (*pidCollMeta.coll)[0].getParticle().id().collectionID; + if (auto it = m_idToName.find(id); it != m_idToName.end()) { + auto name = it->second; + if (pidCollMeta.metadata.has_value()) { + UTIL::PIDHandler pidHandler(lcio_event->getCollection(name)); + algoId = + pidHandler.addAlgorithm(pidCollMeta.metadata.value().algoName, pidCollMeta.metadata.value().paramNames); + found = true; + } + } + } + if (!found) { + warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name + << " for setting consistent metadata" << endmsg; + } } convertParticleIDs(collection_pairs.particleIDs, pidCollMeta.name, algoId.value_or(-1)); } diff --git a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp index 19bc2462..373e7ede 100644 --- a/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp +++ b/k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp @@ -18,6 +18,7 @@ */ #include "k4MarlinWrapper/converters/Lcio2EDM4hep.h" #include "GlobalConvertedObjectsMap.h" +#include "StoreUtils.h" #include #include @@ -27,6 +28,7 @@ #include #include +#include #include #include @@ -39,19 +41,23 @@ DECLARE_COMPONENT(Lcio2EDM4hepTool); using namespace k4MarlinWrapper; Lcio2EDM4hepTool::Lcio2EDM4hepTool(const std::string& type, const std::string& name, const IInterface* parent) - : AlgTool(type, name, parent), m_eds("EventDataSvc", "Lcio2EDM4hepTool") { + : AlgTool(type, name, parent), m_eventDataSvc("EventDataSvc", "Lcio2EDM4hepTool") { declareInterface(this); - StatusCode sc = m_eds.retrieve(); + StatusCode sc = m_eventDataSvc.retrieve(); if (sc.isFailure()) { error() << "Could not retrieve EventDataSvc" << endmsg; } } StatusCode Lcio2EDM4hepTool::initialize() { - m_podioDataSvc = dynamic_cast(m_eds.get()); - if (!m_podioDataSvc) + m_podioDataSvc = dynamic_cast(m_eventDataSvc.get()); + + m_metadataSvc = service("MetadataSvc", false); + if (!m_podioDataSvc && !m_metadataSvc) { + error() << "Could not retrieve MetadataSvc" << endmsg; return StatusCode::FAILURE; + } return AlgTool::initialize(); } @@ -62,12 +68,16 @@ StatusCode Lcio2EDM4hepTool::finalize() { return AlgTool::finalize(); } // Check if a collection was already registered to skip it // ********************************** bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) { - auto collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); - for (const auto& name : collections) { - if (collection_name == name) { - debug() << "Collection named " << name << " already registered, skipping conversion." << endmsg; - return true; - } + std::vector collections; + if (m_podioDataSvc) { + collections = m_podioDataSvc->getEventFrame().getAvailableCollections(); + } else { + std::optional> dummy = std::nullopt; + collections = getAvailableCollectionsFromStore(this, dummy, true); + } + if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) { + debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg; + return true; } return false; } @@ -85,7 +95,7 @@ void Lcio2EDM4hepTool::registerCollection( // No need to check for pre-existing collections, since we only ever end up // here if that is not the case - auto sc = m_podioDataSvc->registerObject("/Event", "/" + std::string(name), wrapper); + auto sc = m_eventDataSvc->registerObject("/Event", "/" + std::string(name), wrapper); if (sc == StatusCode::FAILURE) { error() << "Could not register collection " << name << endmsg; } @@ -98,8 +108,14 @@ void Lcio2EDM4hepTool::registerCollection( for (auto& elem : string_keys) { if (elem == edm4hep::labels::CellIDEncoding) { const auto& lcio_coll_cellid_str = lcioColl->getParameters().getStringVal(lcio::LCIO::CellIDEncoding); - auto& mdFrame = m_podioDataSvc->getMetaDataFrame(); - mdFrame.putParameter(podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding), lcio_coll_cellid_str); + if (m_podioDataSvc) { + auto& mdFrame = m_podioDataSvc->getMetaDataFrame(); + mdFrame.putParameter(podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding), + lcio_coll_cellid_str); + } else { + k4FWCore::putParameter(podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding), + lcio_coll_cellid_str); + } debug() << "Storing CellIDEncoding " << podio::collMetadataParamName(name, edm4hep::labels::CellIDEncoding) << " value: " << lcio_coll_cellid_str << endmsg; } else { @@ -131,9 +147,20 @@ namespace { StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { // Convert event parameters - // auto& frame = const_cast(m_podioDataSvc->getEventFrame()); - auto& frame = m_podioDataSvc->m_eventframe; - LCIO2EDM4hepConv::convertObjectParameters(the_event, frame); + if (m_podioDataSvc) { + LCIO2EDM4hepConv::convertObjectParameters(the_event, m_podioDataSvc->m_eventframe); + } else { + DataObject* p; + StatusCode code = m_eventDataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p); + if (code.isSuccess()) { + auto* frameWrapper = dynamic_cast*>(p); + LCIO2EDM4hepConv::convertObjectParameters(the_event, frameWrapper->getData()); + } else { + warning() << "Could not retrieve the event frame; event parameters will not be converted. This is a known " + "limitation when running with IOSvc without an input file." + << endmsg; + } + } // Convert Event Header outside the collections loop if (!collectionExist(edm4hep::labels::EventHeader)) { @@ -208,9 +235,15 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) { } // Set the ParticleID meta information - auto& metadataFrame = m_podioDataSvc->getMetaDataFrame(); - for (const auto& [collName, pidInfo] : pidInfos) { - edm4hep::utils::PIDHandler::setAlgoInfo(metadataFrame, collName, pidInfo); + if (m_podioDataSvc) { + auto& metadataFrame = m_podioDataSvc->getMetaDataFrame(); + for (const auto& [collName, pidInfo] : pidInfos) { + edm4hep::utils::PIDHandler::setAlgoInfo(metadataFrame, collName, pidInfo); + } + } else { + for (const auto& [collName, pidInfo] : pidInfos) { + m_metadataSvc->put(collName, pidInfo); + } } // We want one "global" map that is created the first time it is used in the event. diff --git a/k4MarlinWrapper/src/components/StoreUtils.cpp b/k4MarlinWrapper/src/components/StoreUtils.cpp new file mode 100644 index 00000000..ac7a95b5 --- /dev/null +++ b/k4MarlinWrapper/src/components/StoreUtils.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "StoreUtils.h" + +#include "GaudiKernel/AlgTool.h" +#include "GaudiKernel/AnyDataWrapper.h" +#include "GaudiKernel/IDataManagerSvc.h" +#include "GaudiKernel/IDataProviderSvc.h" +#include "GaudiKernel/SmartDataPtr.h" + +#include "podio/Frame.h" + +#include "k4FWCore/FunctionalUtils.h" + +#include +#include +#include + +// This is a reimplementation of functionality to retrieve collections from the +// store that can be found in Writer.cpp in k4FWCore with some modifications +// that are specific to the usage of this function in the converters, like +// returning also a map from collection ID to collection name +std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, + std::optional>& idToName, + bool returnFrameCollections) { + std::vector collectionNames; + + SmartIF mgr; + mgr = thisClass->evtSvc(); + + SmartDataPtr root(thisClass->evtSvc(), "/Event"); + if (!root) { + thisClass->error() << "Failed to retrieve root object /Event" << endmsg; + } + + auto pObj = root->registry(); + if (!pObj) { + throw std::runtime_error("Failed to retrieve the root registry object"); + } + std::vector leaves; + StatusCode sc = mgr->objectLeaves(pObj, leaves); + if (!sc.isSuccess()) { + throw std::runtime_error("Failed to retrieve object leaves"); + } + for (const auto& pReg : leaves) { + if (pReg->name() == k4FWCore::frameLocation) { + if (!returnFrameCollections) + continue; + auto wrapper = dynamic_cast*>(pReg->object()); + if (!wrapper) { + throw std::runtime_error("Could not cast object to Frame"); + } + for (const auto& name : wrapper->getData().getAvailableCollections()) { + collectionNames.push_back(name); + } + } + DataObject* p; + sc = thisClass->evtSvc()->retrieveObject("/Event" + pReg->name(), p); + if (sc.isFailure()) { + thisClass->error() << "Could not retrieve object " << pReg->name() << " from the EventStore" << endmsg; + } + auto wrapper = dynamic_cast>*>(p); + if (!wrapper) { + continue; + } + // Remove the leading / + collectionNames.push_back(pReg->name().substr(1, pReg->name().size() - 1)); + if (idToName) { + idToName->emplace(wrapper->getData()->getID(), pReg->name()); + } + } + return collectionNames; +} diff --git a/k4MarlinWrapper/src/components/StoreUtils.h b/k4MarlinWrapper/src/components/StoreUtils.h new file mode 100644 index 00000000..c039b4af --- /dev/null +++ b/k4MarlinWrapper/src/components/StoreUtils.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "GaudiKernel/AlgTool.h" + +#include +#include + +// This functionality is used in the Writer from k4FWCore and is reimplemented +// here with some additions to make it useful for converting both from EDM4hep +// to LCIO and vice versa +std::vector getAvailableCollectionsFromStore(const AlgTool* thisClass, + std::optional>& idToName, + bool returnFrameCollections = false); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 358357f2..a2cd86c8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,6 +20,7 @@ limitations under the License. gaudi_add_module(GaudiTestAlgorithms SOURCES src/MCRecoLinkChecker.cc + src/MCRecoLinkCheckerFunctional.cc src/PseudoRecoAlgorithm.cc src/TrivialMCRecoLinker.cc LINK @@ -77,7 +78,9 @@ set_tests_properties ( same_num_io PASS_REGULAR_EXPRESSION "Input and output have same number of events") # Test clicReconstruction with EDM4hep input and output -ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}") +ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --no-iosvc") +ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_edm4hep_input_iosvc COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_e4h_input.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} --iosvc") +add_test( clicRec_edm4hep_input_compare_output bash -c "diff <(podio-dump CLICPerformance/clicConfig/my_output.root | grep -v 'input file:') <(podio-dump CLICPerformance/clicConfig/my_output_iosvc.root | grep -v 'input file:')") # Run clicReconstruction sequence with LCIO input and output, no converters, with inter-event parallelism ExternalData_Add_Test( marlinwrapper_tests NAME clicRec_lcio_mt COMMAND bash -c "${CMAKE_CURRENT_SOURCE_DIR}/scripts/clicRec_lcio_mt.sh DATA{${PROJECT_SOURCE_DIR}/test/input_files/testSimulation.slcio}") @@ -88,11 +91,17 @@ add_test( clic_geo_test ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/geoTest_ # Test for checking whether the converters can resolve relations across # multiple processors ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --EventDataSvc.input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) +ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps_iosvc COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --iosvc --IOSvc.Input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) +ExternalData_Add_Test( marlinwrapper_tests NAME global_converter_maps_iosvc_functional COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_global_converter_maps.py --iosvc --use-functional-checker --IOSvc.Input DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root}) ExternalData_Add_Test( marlinwrapper_tests NAME link_conversion_edm4hep_to_lcio COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_link_conversion_edm4hep.py --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} ) +ExternalData_Add_Test( marlinwrapper_tests + NAME link_conversion_edm4hep_to_lcio_iosvc + COMMAND ${K4RUN} ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/test_link_conversion_edm4hep.py --iosvc --inputfile DATA{${PROJECT_SOURCE_DIR}/test/input_files/ttbar_20240223_edm4hep.root} +) add_test( event_header_conversion bash -c "k4run ${CMAKE_CURRENT_SOURCE_DIR}/gaudi_opts/createEventHeader.py && anajob test.slcio | grep 'EVENT: 42'" ) @@ -104,9 +113,14 @@ set_tests_properties(${test_names} PROPERTIES ENVIRONMENT ) set_tests_properties( - clicRec clicRec_lcio_mt clicRec_edm4hep_input + clicRec clicRec_lcio_mt clicRec_edm4hep_input clicRec_edm4hep_input_iosvc PROPERTIES DEPENDS CLICPerformance_setup ) +set_tests_properties( + clicRec_edm4hep_input_compare_output + PROPERTIES + DEPENDS "clicRec_edm4hep_input;clicRec_edm4hep_input_iosvc" +) diff --git a/test/gaudi_opts/test_global_converter_maps.py b/test/gaudi_opts/test_global_converter_maps.py index 2746c3f8..5d344713 100644 --- a/test/gaudi_opts/test_global_converter_maps.py +++ b/test/gaudi_opts/test_global_converter_maps.py @@ -23,20 +23,49 @@ from Configurables import ( PodioInput, + PodioOutput, MarlinProcessorWrapper, k4DataSvc, Lcio2EDM4hepTool, EDM4hep2LcioTool, MCRecoLinkChecker, - ApplicationMgr, PseudoRecoAlgorithm, + EventDataSvc, ) -evtsvc = k4DataSvc("EventDataSvc") +from k4FWCore import ApplicationMgr, IOSvc -podioInput = PodioInput("InputReader") -podioInput.collections = ["MCParticles"] -podioInput.OutputLevel = INFO +from k4FWCore.parseArgs import parser + +parser.add_argument( + "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" +) +parser.add_argument( + "--use-functional-checker", action="store_true", default=False, help="Use functional checker" +) + +args = parser.parse_known_args()[0] + +if args.use_functional_checker: + from Configurables import MCRecoLinkCheckerFunctional as MCRecoLinkChecker + +if args.iosvc: + evtsvc = EventDataSvc("EventDataSvc") +else: + evtsvc = k4DataSvc("EventDataSvc") + +if args.iosvc: + iosvc = IOSvc() + if not args.use_functional_checker: + iosvc.Output = "global_converter_maps_iosvc.root" + else: + iosvc.Output = "global_converter_maps_iosvc_functional.root" +else: + podioInput = PodioInput("InputReader") + podioInput.collections = ["MCParticles"] + podioInput.OutputLevel = INFO + podioOutput = PodioOutput("OutputWriter") + podioOutput.filename = "global_converter_maps.root" PseudoRecoAlg = PseudoRecoAlgorithm( "PseudoRecoAlgorithm", InputMCs=["MCParticles"], OutputRecos=["PseudoRecoParticles"] @@ -68,16 +97,22 @@ TrivialMCTruthLinkerProc.Lcio2EDM4hepTool = mcTruthConverter mcLinkChecker = MCRecoLinkChecker("MCRecoLinkChecker") -mcLinkChecker.InputMCRecoLinks = "TrivialMCRecoLinks" -mcLinkChecker.InputMCs = "MCParticles" -mcLinkChecker.InputRecos = "PseudoRecoParticles" +mcLinkChecker.InputMCRecoLinks = ( + "TrivialMCRecoLinks" if not args.use_functional_checker else ["TrivialMCRecoLinks"] +) +mcLinkChecker.InputMCs = "MCParticles" if not args.use_functional_checker else ["MCParticles"] +mcLinkChecker.InputRecos = ( + "PseudoRecoParticles" if not args.use_functional_checker else ["PseudoRecoParticles"] +) mcLinkChecker.OutputLevel = DEBUG algList = [ - podioInput, PseudoRecoAlg, TrivialMCTruthLinkerProc, mcLinkChecker, ] -ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=3, ExtSvc=[evtsvc], OutputLevel=DEBUG) +if not args.iosvc: + algList = [podioInput] + algList + [podioOutput] + +ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=1, ExtSvc=[evtsvc], OutputLevel=DEBUG) diff --git a/test/gaudi_opts/test_link_conversion_edm4hep.py b/test/gaudi_opts/test_link_conversion_edm4hep.py index 92080f7e..26c22cef 100644 --- a/test/gaudi_opts/test_link_conversion_edm4hep.py +++ b/test/gaudi_opts/test_link_conversion_edm4hep.py @@ -23,24 +23,33 @@ from Configurables import ( PodioInput, k4DataSvc, - ApplicationMgr, PseudoRecoAlgorithm, TrivialMCRecoLinker, MarlinProcessorWrapper, EDM4hep2LcioTool, + EventDataSvc, ) +from k4FWCore import ApplicationMgr, IOSvc + from k4FWCore.parseArgs import parser parser.add_argument("--inputfile", help="Input file") -my_args = parser.parse_known_args()[0] - -evtsvc = k4DataSvc("EventDataSvc") -evtsvc.input = my_args.inputfile +parser.add_argument( + "--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc" +) +args = parser.parse_known_args()[0] -podioInput = PodioInput("InputReader") -podioInput.collections = ["MCParticles"] -podioInput.OutputLevel = INFO +if args.iosvc: + evtsvc = EventDataSvc("EventDataSvc") + iosvc = IOSvc() + iosvc.Input = args.inputfile +else: + evtsvc = k4DataSvc("EventDataSvc") + evtsvc.input = args.inputfile + podioInput = PodioInput("InputReader") + podioInput.collections = ["MCParticles"] + podioInput.OutputLevel = INFO PseudoRecoAlg = PseudoRecoAlgorithm( @@ -74,9 +83,13 @@ mcLinkConverter.OutputLevel = DEBUG MarlinMCLinkChecker.EDM4hep2LcioTool = mcLinkConverter +algList = [PseudoRecoAlg, MCRecoLinker, MarlinMCLinkChecker] + +if not args.iosvc: + algList = [podioInput] + algList ApplicationMgr( - TopAlg=[podioInput, PseudoRecoAlg, MCRecoLinker, MarlinMCLinkChecker], + TopAlg=algList, ExtSvc=[evtsvc], EvtMax=-1, EvtSel="NONE", diff --git a/test/scripts/clicRec_e4h_input.sh b/test/scripts/clicRec_e4h_input.sh index 7b67072a..07a4d3ee 100755 --- a/test/scripts/clicRec_e4h_input.sh +++ b/test/scripts/clicRec_e4h_input.sh @@ -23,10 +23,22 @@ set -eu cd CLICPerformance/clicConfig -k4run $EXAMPLE_DIR/clicRec_e4h_input.py --EventDataSvc.input=$1 +if [ "$2" = "--iosvc" ]; then + iosvc="_iosvc" + echo "Running with IO service" + file_arg="--iosvc --IOSvc.Input=$1" +elif [ "$2" = "--no-iosvc" ]; then + iosvc="" + echo "Running without IO service" + file_arg="--EventDataSvc.input=$1" +else + echo "Wrong argument $2" + return 1 +fi +k4run $EXAMPLE_DIR/clicRec_e4h_input.py ${file_arg} --rec-output Output_REC_e4h_input$iosvc.slcio --dst-output Output_DST_e4h_input$iosvc.slcio --gaudi-output my_output$iosvc.root input_num_events=$(python $TEST_DIR/python/root_num_events.py $1) -output_num_events=$(python $TEST_DIR/python/root_num_events.py my_output.root) +output_num_events=$(python $TEST_DIR/python/root_num_events.py my_output$iosvc.root) # First check do we have the same number of events in input and output if [ "$input_num_events" != "$output_num_events" ]; then @@ -35,14 +47,15 @@ if [ "$input_num_events" != "$output_num_events" ]; then fi # Second check: contents (at least superficially) -echo "Comparing contents of Output_REC_e4h_input.slcio" -if ! diff <(anajob Output_REC_e4h_input.slcio) $TEST_DIR/inputFiles/anajob_Output_REC.expected; then +# Exclude the file name since it is different when using the IOSvc +echo "Comparing contents of Output_REC_e4h_input$iosvc.slcio" +if ! diff -I "Output_REC_e4h_input.*\.slcio" <(anajob Output_REC_e4h_input$iosvc.slcio) $TEST_DIR/inputFiles/anajob_Output_REC.expected; then echo "File contents of REC slcio file are not as expected" exit 1 fi -echo "Comparing contents of Output_DST_e4h_input.slcio" -if ! diff <(anajob Output_DST_e4h_input.slcio) $TEST_DIR/inputFiles/anajob_Output_DST.expected; then +echo "Comparing contents of Output_DST_e4h_input$iosvc.slcio" +if ! diff -I "Output_DST_e4h_input.*\.slcio" <(anajob Output_DST_e4h_input$iosvc.slcio) $TEST_DIR/inputFiles/anajob_Output_DST.expected; then echo "File contents of DST slcio file are not as expected" exit 1 fi diff --git a/test/src/MCRecoLinkCheckerFunctional.cc b/test/src/MCRecoLinkCheckerFunctional.cc new file mode 100644 index 00000000..6e98a8a7 --- /dev/null +++ b/test/src/MCRecoLinkCheckerFunctional.cc @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "MCRecoLinkCheckerFunctional.h" + +MCRecoLinkCheckerFunctional::MCRecoLinkCheckerFunctional(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, + {KeyValues("InputMCRecoLinks", {"InputMCRecoLinks"}), KeyValues("InputMCs", {"InputMCs"}), + KeyValues("InputRecos", {"InputRecos"})}) {} + +void MCRecoLinkCheckerFunctional::operator()(const edm4hep::RecoMCParticleLinkCollection& relationColl, + const edm4hep::MCParticleCollection& mcColl, + const edm4hep::ReconstructedParticleCollection& recoColl) const { + if (relationColl.size() != mcColl.size()) { + error() << "The MCReco relation collection does not have the expected size (expected: " << relationColl.size() + << ", actual: " << mcColl.size() << ")" << endmsg; + throw std::runtime_error( + "MCRecoLinkCheckerFunctional: The MCReco relation collection does not have the expected size"); + } + + for (size_t i = 0; i < mcColl.size(); ++i) { + const auto mc = mcColl[i]; + const auto reco = recoColl[i]; + const auto relation = relationColl[i]; + + if (relation.getWeight() != i) { + error() << "Relation " << i << " does not not have the correct weight (expected: " << i + << ", actual: " << relation.getWeight() << ")" << endmsg; + throw std::runtime_error("The MCReco relation collection does not have the expected weight"); + } + + if (!(relation.getTo() == mc)) { + auto relMC = relation.getTo(); + error() << "Relation " << i + << " does not point to the correct MCParticle (expected: " << mc.getObjectID().collectionID << "|" + << mc.getObjectID().index << ", actual: " << relMC.getObjectID().collectionID << "|" + << relMC.getObjectID().index << ")" << endmsg; + throw std::runtime_error("The MCReco relation collection does not point to the correct MCParticle"); + } + + if (!(relation.getFrom() == reco)) { + auto relRec = relation.getFrom(); + error() << "Relation " << i + << " does not point to the correct RecoParticle (expected: " << reco.getObjectID().collectionID << "|" + << reco.getObjectID().index << ", actual: " << relRec.getObjectID().collectionID << "|" + << relRec.getObjectID().index << ")" << endmsg; + throw std::runtime_error("The MCReco relation collection does not point to the correct RecoParticle"); + } + } +} + +DECLARE_COMPONENT(MCRecoLinkCheckerFunctional) diff --git a/test/src/MCRecoLinkCheckerFunctional.h b/test/src/MCRecoLinkCheckerFunctional.h new file mode 100644 index 00000000..281d51bd --- /dev/null +++ b/test/src/MCRecoLinkCheckerFunctional.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef K4MARLINWRAPPER_TEST_MCRECOLINKCHECKERFUNCTIONAL_H +#define K4MARLINWRAPPER_TEST_MCRECOLINKCHECKERFUNCTIONAL_H + +#include "edm4hep/MCParticleCollection.h" +#include "edm4hep/RecoMCParticleLinkCollection.h" +#include "edm4hep/ReconstructedParticleCollection.h" + +#include "k4FWCore/Consumer.h" + +struct MCRecoLinkCheckerFunctional final + : k4FWCore::Consumer { + MCRecoLinkCheckerFunctional(const std::string& name, ISvcLocator* svcLoc); + + void operator()(const edm4hep::RecoMCParticleLinkCollection& relationColl, + const edm4hep::MCParticleCollection& mcColl, + const edm4hep::ReconstructedParticleCollection& recoColl) const override; +}; + +#endif // K4MARLINWRAPPER_TEST_MCRECOLINKCHECKERFUNCTIONAL_H