diff --git a/Common/Utils/src/ShmManager.cxx b/Common/Utils/src/ShmManager.cxx index 26b30be062220..3ed863dcc96c3 100644 --- a/Common/Utils/src/ShmManager.cxx +++ b/Common/Utils/src/ShmManager.cxx @@ -123,7 +123,7 @@ bool ShmManager::createGlobalSegment(int nsegments) LOG(info) << "CREATING SIM SHARED MEM SEGMENT FOR " << nsegments << " WORKERS"; // LOG(info) << "SIZEOF ShmMetaInfo " << sizeof(ShmMetaInfo); const auto totalsize = sizeof(ShmMetaInfo) + SHMPOOLSIZE * nsegments; - if ((mShmID = shmget(IPC_PRIVATE, totalsize, IPC_CREAT | 0666)) == -1) { + if ((mShmID = shmget(IPC_PRIVATE, totalsize, IPC_CREAT | 0600)) == -1) { perror("shmget: shmget failed"); } else { // We are attaching once to determine a common virtual address under which everyone else should attach. @@ -143,6 +143,10 @@ bool ShmManager::createGlobalSegment(int nsegments) // TODO: consider using named posix shared memory segments to avoid this setenv(SHMIDNAME, std::to_string(mShmID).c_str(), 1); setenv(SHMADDRNAME, std::to_string((unsigned long long)(addr)).c_str(), 1); + + // mark the segment for removal right away: Linux still lets the workers attach by id, + // and the kernel frees it when the last process detaches, even after a crash + shmctl(mShmID, IPC_RMID, nullptr); return true; } LOG(info) << "SHARED MEM INITIALIZED AT ID " << mShmID; diff --git a/Detectors/Base/include/DetectorsBase/Detector.h b/Detectors/Base/include/DetectorsBase/Detector.h index 5856694e535a2..5f7d9679537e2 100644 --- a/Detectors/Base/include/DetectorsBase/Detector.h +++ b/Detectors/Base/include/DetectorsBase/Detector.h @@ -32,6 +32,7 @@ #include "CommonUtils/ShmManager.h" #include "CommonUtils/ShmAllocator.h" #include +#include #include #include #include @@ -186,7 +187,7 @@ class Detector : public FairDetector // and to decode it virtual void attachHits(fair::mq::Channel&, fair::mq::Parts&) = 0; virtual void fillHitBranch(TTree& tr, fair::mq::Parts& parts, int& index) = 0; - virtual void collectHits(int eventID, fair::mq::Parts& parts, int& index) = 0; + virtual void collectHits(int eventID, fair::mq::Parts& parts, int& index, bool shm) = 0; virtual void mergeHitEntriesAndFlush(int eventID, TTree& target, std::vector const& trackoffsets, @@ -269,11 +270,14 @@ inline std::string demangle(const char* name) return (status == 0) ? res.get() : name; } -void attachShmMessage(void* hitsptr, fair::mq::Channel& channel, fair::mq::Parts& parts, bool* busy_ptr); -void* decodeShmCore(fair::mq::Parts& dataparts, int index, bool*& busy); +// a flag in shared memory telling whether the hit merger still reads a hit buffer +using ShmBusyFlag = std::atomic; + +void attachShmMessage(void* hitsptr, fair::mq::Channel& channel, fair::mq::Parts& parts, ShmBusyFlag* busy_ptr); +void* decodeShmCore(fair::mq::Parts& dataparts, int index, ShmBusyFlag*& busy); template -T decodeShmMessage(fair::mq::Parts& dataparts, int index, bool*& busy) +T decodeShmMessage(fair::mq::Parts& dataparts, int index, ShmBusyFlag*& busy) { return reinterpret_cast(decodeShmCore(dataparts, index, busy)); } @@ -294,7 +298,13 @@ T decodeTMessage(fair::mq::Parts& dataparts, int index) return static_cast(decodeTMessageCore(dataparts, index)); } -void attachDetIDHeaderMessage(int id, fair::mq::Channel& channel, fair::mq::Parts& parts); +// header message preceding the hits of one detector +struct HitsHeader { + int detID; + bool shm; // whether the hits follow as shared-memory references or as TMessages +}; + +void attachHitsHeaderMessage(HitsHeader const& header, fair::mq::Channel& channel, fair::mq::Parts& parts); template TBranch* getOrMakeBranch(TTree& tree, const char* brname, T* ptr) @@ -356,10 +366,12 @@ class DetImpl : public o2::base::Detector return; } - attachDetIDHeaderMessage(GetDetId(), channel, parts); // the DetId s are universal as they come from o2::detector::DetID + // decide the transport once, so that the header and all hit messages agree + const bool shm = UseShm::value && o2::utils::ShmManager::Instance().isOperational(); + attachHitsHeaderMessage({GetDetId(), shm}, channel, parts); // the DetId s are universal as they come from o2::detector::DetID while (auto hits = static_cast(this)->Det::getHits(probe++)) { - if (!UseShm::value || !o2::utils::ShmManager::Instance().isOperational()) { + if (!shm) { attachTMessage(*hits, channel, parts); } else { // this is the shared mem variant @@ -445,7 +457,7 @@ class DetImpl : public o2::base::Detector { auto entries = hitbuffervector.size(); - auto targetdata = new T; // used to collect data inside a single container + T targetdata; // used to collect data inside a single container T* filladdress = nullptr; // pointer used for final ROOT IO if (entries == 1) { filladdress = hitbuffervector[0].get(); @@ -453,14 +465,17 @@ class DetImpl : public o2::base::Detector } else { // here we need to do merging and index adjustment int nprimTot = 0; + size_t nhits = 0; for (auto entry = 0; entry < entries; entry++) { nprimTot += nprimaries[entry]; + nhits += hitbuffervector[entry] ? hitbuffervector[entry]->size() : 0; } + targetdata.reserve(nhits); // offset for pimary track index int idelta0 = 0; // offset for secondary track index int idelta1 = nprimTot; - filladdress = targetdata; + filladdress = &targetdata; for (int entry = entries - 1; entry >= 0; --entry) { // proceed in the order of subevent Ids int index = subevtsOrdered[entry]; @@ -475,8 +490,8 @@ class DetImpl : public o2::base::Detector for (auto& hit : *incomingdata) { hit.SetTrackID(offsetTrackIndex(hit.GetTrackID(), nprim, idelta0, idelta1)); } - // this could be further generalized by using a policy for T - std::copy(incomingdata->begin(), incomingdata->end(), std::back_inserter(*targetdata)); + // move rather than copy, since hits may own memory themselves (e.g. TPC HitGroup) + targetdata.insert(targetdata.end(), std::make_move_iterator(incomingdata->begin()), std::make_move_iterator(incomingdata->end())); } // adjust offsets for next subevent idelta0 += nprim; @@ -488,10 +503,7 @@ class DetImpl : public o2::base::Detector targetbr->SetAddress(&filladdress); targetbr->Fill(); targetbr->ResetAddress(); - targetdata->clear(); - hitbuffervector.clear(); hitbuffervector = L(); // swap with empty vector to release mem - delete targetdata; } void mergeHitEntries(TTree& origin, TTree& target, std::vector const& trackoffsets, std::vector const& nprimaries, std::vector const& subevtsOrdered) final @@ -508,6 +520,17 @@ class DetImpl : public o2::base::Detector } } + // the hit containers buffered in the hit merger, per event and per hit branch + auto& hitCollector() + { + using Hit_t = typename std::remove_pointer(this)->Det::getHits(0))>::type; + using Collector_t = tbb::concurrent_unordered_map>>>; + if (!mHitCollector) { + mHitCollector = std::make_shared(); + } + return *static_cast(mHitCollector.get()); + } + void mergeHitEntriesAndFlush(int eventID, TTree& target, std::vector const& trackoffsets, std::vector const& nprimaries, std::vector const& subevtsOrdered) final { // loop over hit containers / different branches @@ -515,10 +538,9 @@ class DetImpl : public o2::base::Detector int probe = 0; using Hit_t = typename std::remove_pointer(this)->Det::getHits(0))>::type; // remove buffered event from the hit store - using Collector_t = tbb::concurrent_unordered_map>>>; - auto hitbufferPtr = reinterpret_cast(mHitCollectorBufferPtr); - auto iter = hitbufferPtr->find(eventID); - if (iter == hitbufferPtr->end()) { + auto& collector = hitCollector(); + auto iter = collector.find(eventID); + if (iter == collector.end()) { LOG(error) << "No buffered hits available for event " << eventID; return; } @@ -538,59 +560,35 @@ class DetImpl : public o2::base::Detector /// Collect Hits available as incoming message (shared mem or not) /// inside this process for later streaming to output. A function needed /// by the hit-merger process (not for direct use by users) - void collectHits(int eventID, fair::mq::Parts& parts, int& index) override + void collectHits(int eventID, fair::mq::Parts& parts, int& index, bool shm) override { using Hit_t = typename std::remove_pointer(this)->Det::getHits(0))>::type; - using Collector_t = tbb::concurrent_unordered_map>>>; - // note: we can't put this as a member because decltype type deduction doesn't seem to work for - // class members; so we use a static and communicate it to other functions via a pointer member. - // The collector must be kept *per detector instance* (keyed by 'this'): for most detectors there - // is a single instance per C++ type, but several external detectors share the same type - // (o2::ext::ExternalDetector) and would otherwise clobber/double-free each other's buffers. - // tbb::concurrent_unordered_map is node-based, so the reference stays valid across insertions. - static tbb::concurrent_unordered_map hitcollectors; - auto& hitcollector = hitcollectors[this]; - mHitCollectorBufferPtr = (char*)&hitcollector; + auto& hitcollector = hitCollector(); int probe = 0; - bool* busy = nullptr; + ShmBusyFlag* busy = nullptr; using HitPtr_t = decltype(static_cast(this)->Det::getHits(probe)); std::string name = static_cast(this)->getHitBranchNames(probe); - auto copyToBuffer = [this, eventID](HitPtr_t hitdata, Collector_t& collectbuffer, int probe) { - std::vector>>* hitvector = nullptr; - { - auto eventIter = collectbuffer.find(eventID); - if (eventIter == collectbuffer.end()) { - // key insertion and traversal are thread-safe with tbb so no need - // to protect - collectbuffer[eventID] = std::vector>>(); - } - hitvector = &(collectbuffer[eventID]); + // stores one hit container of this event and probe in the collector + auto store = [eventID, &hitcollector](std::unique_ptr hits, int probe) { + auto& hitvector = hitcollector[eventID]; // tbb insertion is thread-safe + if (probe >= hitvector.size()) { + hitvector.resize(probe + 1); } - if (probe >= hitvector->size()) { - hitvector->resize(probe + 1); - } - // add empty hit bucket to list for this event and probe - (*hitvector)[probe].emplace_back(new Hit_t()); - // copy the data into this bucket - *((*hitvector)[probe].back()) = *hitdata; + hitvector[probe].emplace_back(std::move(hits)); }; while (name.size() > 0) { - if (!UseShm::value || !o2::utils::ShmManager::Instance().isOperational()) { - // for each branch name we extract/decode hits from the message parts ... - auto hitsptr = decodeTMessage(parts, index++); - if (hitsptr) { - // ... and copy them to the buffer - copyToBuffer(hitsptr, hitcollector, probe); - delete hitsptr; + if (!shm) { + // a decoded TMessage is ours, so we adopt it + if (auto hitsptr = decodeTMessage(parts, index++)) { + store(std::unique_ptr(hitsptr), probe); } } else { - // for each branch name we extract/decode hits from the message parts ... + // hits in shared memory belong to the worker, so we copy them auto hitsptr = decodeShmMessage(parts, index++, busy); - // ... and copy them to the buffer - copyToBuffer(hitsptr, hitcollector, probe); + store(std::make_unique(*hitsptr), probe); } // next name probe++; @@ -606,7 +604,7 @@ class DetImpl : public o2::base::Detector void fillHitBranch(TTree& tr, fair::mq::Parts& parts, int& index) override { int probe = 0; - bool* busy = nullptr; + ShmBusyFlag* busy = nullptr; using Hit_t = decltype(static_cast(this)->Det::getHits(probe)); std::string name = static_cast(this)->getHitBranchNames(probe++); while (name.size() > 0) { @@ -697,8 +695,7 @@ class DetImpl : public o2::base::Detector static_cast(this)->Det::createHitBuffers(); for (int b = 0; b < NHITBUFFERS; ++b) { auto& instance = o2::utils::ShmManager::Instance(); - mShmBusy[b] = instance.hasSegment() ? (bool*)instance.getmemblock(sizeof(bool)) : new bool; - *mShmBusy[b] = false; + mShmBusy[b] = instance.hasSegment() ? new (instance.getmemblock(sizeof(ShmBusyFlag))) ShmBusyFlag(false) : new ShmBusyFlag(false); } } mInitialized = true; @@ -749,12 +746,12 @@ class DetImpl : public o2::base::Detector static constexpr int NHITBUFFERS = 3; // number of buffers for hits in order to allow async processing // in the hit merger without blocking nor copying the data // (like done in typical data aquisition systems) - bool* mShmBusy[NHITBUFFERS] = {nullptr}; //! pointer to bool in shared mem indicating of IO busy + ShmBusyFlag* mShmBusy[NHITBUFFERS] = {nullptr}; //! pointer to flag in shared mem indicating of IO busy std::vector mCachedPtr[NHITBUFFERS]; int mCurrentBuffer = 0; // holding the current buffer information int mInitialized = false; - char* mHitCollectorBufferPtr = nullptr; //! pointer to hit (collector) buffer location (strictly internal) + std::shared_ptr mHitCollector; //! type-erased hit buffers of this instance in the hit merger (see hitCollector()) ClassDefOverride(DetImpl, 0); }; diff --git a/Detectors/Base/src/Detector.cxx b/Detectors/Base/src/Detector.cxx index d2be9237f6f13..72c35e24bae3d 100644 --- a/Detectors/Base/src/Detector.cxx +++ b/Detectors/Base/src/Detector.cxx @@ -215,17 +215,17 @@ void attachMessageBufferToParts(fair::mq::Parts& parts, fair::mq::Channel& chann o2::framework::TMessageSerializer::serialize(buffer, data, cl); parts.AddPart(std::move(msg)); } -void attachDetIDHeaderMessage(int id, fair::mq::Channel& channel, fair::mq::Parts& parts) +void attachHitsHeaderMessage(HitsHeader const& header, fair::mq::Channel& channel, fair::mq::Parts& parts) { - std::unique_ptr message(channel.NewSimpleMessage(id)); + std::unique_ptr message(channel.NewSimpleMessage(header)); parts.AddPart(std::move(message)); } -void attachShmMessage(void* hits_ptr, fair::mq::Channel& channel, fair::mq::Parts& parts, bool* busy_ptr) +void attachShmMessage(void* hits_ptr, fair::mq::Channel& channel, fair::mq::Parts& parts, ShmBusyFlag* busy_ptr) { struct shmcontext { int id; void* object_ptr; - bool* busy_ptr; + ShmBusyFlag* busy_ptr; }; auto& instance = o2::utils::ShmManager::Instance(); @@ -237,13 +237,13 @@ void attachShmMessage(void* hits_ptr, fair::mq::Channel& channel, fair::mq::Part std::unique_ptr message(channel.NewSimpleMessage(info)); parts.AddPart(std::move(message)); } -void* decodeShmCore(fair::mq::Parts& dataparts, int index, bool*& busy) +void* decodeShmCore(fair::mq::Parts& dataparts, int index, ShmBusyFlag*& busy) { auto rawmessage = std::move(dataparts.At(index)); struct shmcontext { int id; void* object_ptr; - bool* busy_ptr; + ShmBusyFlag* busy_ptr; }; shmcontext* info = (shmcontext*)rawmessage->GetData(); diff --git a/Detectors/Base/test/testStack.cxx b/Detectors/Base/test/testStack.cxx index 41e66e08a9394..f6d32d3bf7157 100644 --- a/Detectors/Base/test/testStack.cxx +++ b/Detectors/Base/test/testStack.cxx @@ -117,7 +117,7 @@ class TestDetector : public o2::base::Detector std::string getHitBranchNames(int) const override { return {}; } void attachHits(fair::mq::Channel&, fair::mq::Parts&) override {} void fillHitBranch(TTree&, fair::mq::Parts&, int&) override {} - void collectHits(int, fair::mq::Parts&, int&) override {} + void collectHits(int, fair::mq::Parts&, int&, bool) override {} void mergeHitEntriesAndFlush(int, TTree&, std::vector const&, std::vector const&, std::vector const&) override {} void mergeHitEntries(TTree&, TTree&, std::vector const&, std::vector const&, diff --git a/Detectors/TPC/simulation/include/TPCSimulation/Point.h b/Detectors/TPC/simulation/include/TPCSimulation/Point.h index 1ce7fdc9f1a35..17314afdb9171 100644 --- a/Detectors/TPC/simulation/include/TPCSimulation/Point.h +++ b/Detectors/TPC/simulation/include/TPCSimulation/Point.h @@ -115,8 +115,6 @@ class HitGroup : public o2::BaseHit { } - ~HitGroup() = default; - void addHit(float x, float y, float z, float time, float e) { #ifdef HIT_AOS diff --git a/Steer/include/Steer/O2MCApplication.h b/Steer/include/Steer/O2MCApplication.h index 2ea1b9990a3f6..e43a61ec419b5 100644 --- a/Steer/include/Steer/O2MCApplication.h +++ b/Steer/include/Steer/O2MCApplication.h @@ -53,13 +53,17 @@ class O2MCApplication : public O2MCApplicationBase finishEventCommon(); + // detectors finalize their hits (e.g. sorting, summing duplicates) before these are sent + for (auto det : listActiveDetectors) { + det->FinishEvent(); + } + // This special finish event version does not fill the output tree of FairRootManager // but forwards the data to the HitMerger SendData(); // call end of event on active detectors for (auto det : listActiveDetectors) { - det->FinishEvent(); det->EndOfEvent(); } fStack->Reset(); diff --git a/run/CMakeLists.txt b/run/CMakeLists.txt index abba055cc7cca..8886bfcbaf1cc 100644 --- a/run/CMakeLists.txt +++ b/run/CMakeLists.txt @@ -55,7 +55,7 @@ add_library(internal::allsim ALIAS allsim) o2_add_executable(device-runner COMPONENT_NAME sim - SOURCES O2SimDeviceRunner.cxx + SOURCES O2SimDeviceRunner.cxx O2SimDevice.cxx PrimaryServerState.cxx PUBLIC_LINK_LIBRARIES internal::allsim) o2_add_executable(serial @@ -80,7 +80,7 @@ o2_add_executable(sim o2_add_executable(primary-server-device-runner COMPONENT_NAME sim - SOURCES O2PrimaryServerDeviceRunner.cxx + SOURCES O2PrimaryServerDeviceRunner.cxx O2PrimaryServerDevice.cxx PUBLIC_LINK_LIBRARIES internal::allsim TARGETVARNAME simexe) if(ENABLE_UPGRADES) @@ -105,7 +105,7 @@ endif() o2_add_executable(hit-merger-runner COMPONENT_NAME sim - SOURCES O2HitMergerRunner.cxx + SOURCES O2HitMergerRunner.cxx O2HitMerger.cxx PrimaryServerState.cxx PUBLIC_LINK_LIBRARIES internal::allsim) o2_add_executable(g4-determine-unknown-pdg-properties diff --git a/run/O2HitMerger.cxx b/run/O2HitMerger.cxx new file mode 100644 index 0000000000000..3fabb31193b7d --- /dev/null +++ b/run/O2HitMerger.cxx @@ -0,0 +1,963 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @author Sandro Wenzel + +#include "O2HitMerger.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "FairSystemInfo.h" + +#include "PrimaryServerState.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CommonUtils/ShmManager.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SimPublishChannelHelper.h" + +#ifdef ENABLE_UPGRADES +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include +#include +#include + +namespace o2 +{ +namespace devices +{ + +namespace +{ +// Function communicating to primary particle server that it is now safe to shutdown. +// From the perspective of o2-sim, this is the case when all configs have been propagated and the system +// is running ok: For instance after the HitMerger is initialized and got it's first data from Geant workers. +bool primaryServer_sendShutdownPermission(fair::mq::Channel& channel) +{ + std::unique_ptr request(channel.NewSimpleMessage((int)o2::O2PrimaryServerInfoRequest::AllowShutdown)); + std::unique_ptr reply(channel.NewMessage()); + + int timeoutinMS = 100; + if (channel.Send(request, timeoutinMS) > 0) { + LOG(info) << "Sending Shutdown permission to particle server"; + if (channel.Receive(reply, timeoutinMS) > 0) { + // the answer is a simple ack with a status code + LOG(info) << "Shutdown permission was acknowledged"; + } else { + LOG(error) << "No answer received within " << timeoutinMS << "ms\n"; + return false; + } + return true; + } + return false; +} +} // namespace + +O2HitMerger::O2HitMerger() +{ + mTimer.Start(); + mInitialOutputDir = std::filesystem::current_path().string(); + mCurrentOutputDir = mInitialOutputDir; +} + +O2HitMerger::~O2HitMerger() +{ + FairSystemInfo sysinfo; + LOG(info) << "TIME-STAMP " << mTimer.RealTime() << "\t"; + mTimer.Continue(); + LOG(info) << "MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " + << sysinfo.GetMaxMemory() << " MB\n"; +} + +void O2HitMerger::InitTask() +{ + LOG(info) << "INIT HIT MERGER"; + ROOT::EnableThreadSafety(); + + std::string outfilename("o2sim_merged_hits.root"); // default name + // query the sim config ... which is used to extract the filenames + if (o2::querySimConfig(GetChannels().at("o2sim-primserv-info").at(0))) { + outfilename = o2::base::NameConf::getMCKinematicsFileName(o2::conf::SimConfig::Instance().getOutPrefix().c_str()); + mNExpectedEvents = o2::conf::SimConfig::Instance().getNEvents(); + } else { + // we didn't manage to get a configuration --> better to fail + LOG(fatal) << "No configuration received. Aborting"; + } + mAsService = o2::conf::SimConfig::Instance().asService(); + mForwardKine = o2::conf::SimConfig::Instance().forwardKine(); + mWriteToDisc = o2::conf::SimConfig::Instance().writeToDisc(); + + mOutFileName = outfilename.c_str(); + if (mWriteToDisc) { + mOutFile = new TFile(outfilename.c_str(), "RECREATE"); + mOutTree = new TTree("o2sim", "o2sim"); + mOutTree->SetDirectory(mOutFile); + + mMCHeaderOnlyOutFile = new TFile(o2::base::NameConf::getMCHeadersFileName(o2::conf::SimConfig::Instance().getOutPrefix().c_str()).c_str(), "RECREATE"); + mMCHeaderTree = new TTree("o2sim", "o2sim"); + mMCHeaderTree->SetDirectory(mMCHeaderOnlyOutFile); + } + // detectors init only once + if (mDetectorInstances.size() == 0) { + initDetInstances(); + // has to be after init of Detectors + o2::utils::ShmManager::Instance().attachToGlobalSegment(); + initHitFiles(o2::conf::SimConfig::Instance().getOutPrefix()); + } + + // init pipe + auto pipeenv = getenv("ALICE_O2SIMMERGERTODRIVER_PIPE"); + if (pipeenv) { + mPipeToDriver = atoi(pipeenv); + LOG(info) << "ASSIGNED PIPE HANDLE " << mPipeToDriver; + } else { + LOG(warning) << "DID NOT FIND ENVIRONMENT VARIABLE TO INIT PIPE"; + } + + // if no data to expect we shut down the device NOW since it would otherwise hang + if (mNExpectedEvents == 0) { + if (mAsService) { + waitForControlInput(); + } else { + LOG(info) << "NOT EXPECTING ANY DATA; SHUTTING DOWN"; + raise(SIGINT); + } + } +} + +bool O2HitMerger::setWorkingDirectory(std::string const& dir) +{ + namespace fs = std::filesystem; + + // sets the output directory where simulation files are produced + // and creates it when it doesn't exist already + + // 2 possibilities: + // a) dir is relative dir. Then we interpret it as relative to the initial + // base directory + // b) or dir is itself absolut. + try { + fs::current_path(fs::path(mInitialOutputDir)); // <--- to make sure relative start is always the same + if (!dir.empty()) { + auto absolutePath = fs::absolute(fs::path(dir)); + if (!fs::exists(absolutePath)) { + if (!fs::create_directory(absolutePath)) { + LOG(error) << "Could not create directory " << absolutePath.string(); + return false; + } + } + // set the current path + fs::current_path(absolutePath.string().c_str()); + mCurrentOutputDir = fs::current_path().string(); + } + LOG(info) << "FINAL PATH " << mCurrentOutputDir; + } catch (std::exception e) { + LOG(error) << " could not change path to " << dir; + } + return true; +} + +bool O2HitMerger::ReInit(o2::conf::SimReconfigData const& reconfig) +{ + if (reconfig.stop) { + return false; + } + if (!setWorkingDirectory(reconfig.outputDir)) { + return false; + } + + std::string outfilename("o2sim_merged_hits.root"); // default name + outfilename = o2::base::NameConf::getMCKinematicsFileName(reconfig.outputPrefix); + mNExpectedEvents = reconfig.nEvents; + mOutFileName = outfilename.c_str(); + if (mWriteToDisc) { + mOutFile = new TFile(outfilename.c_str(), "RECREATE"); + mOutTree = new TTree("o2sim", "o2sim"); + mOutTree->SetDirectory(mOutFile); + + mMCHeaderOnlyOutFile = new TFile(o2::base::NameConf::getMCHeadersFileName(reconfig.outputPrefix).c_str(), "RECREATE"); + mMCHeaderTree = new TTree("o2sim", "o2sim"); + mMCHeaderTree->SetDirectory(mMCHeaderOnlyOutFile); + } + // reinit detectorInstance files (also make sure they are closed before continuing) + initHitFiles(reconfig.outputPrefix); + + // clear "counter" datastructures + mPartsCheckSum.clear(); + mEventChecksum = 0; + + // clear collector datastructures + mMCTrackBuffer.clear(); + mTrackRefBuffer.clear(); + mSubEventInfoBuffer.clear(); + mFlushableEvents.clear(); + mNextFlushID = 1; + + return true; +} + +template +V O2HitMerger::insertAdd(std::map& m, T const& key, V value) +{ + const auto iter = m.find(key); + V accum{0}; + if (iter != m.end()) { + iter->second += value; + accum = iter->second; + } else { + m.insert(std::make_pair(key, value)); + accum = value; + } + return accum; +} + +template +bool O2HitMerger::isDataComplete(T checksum, T nparts) +{ + return checksum == nparts * (nparts + 1) / 2; +} + +void O2HitMerger::consumeHits(int eventID, fair::mq::Parts& data, int& index) +{ + auto headermessage = std::move(data.At(index++)); + // this should be the header announcing the hits of one detector + if (headermessage->GetSize() == sizeof(o2::base::HitsHeader)) { + auto header = *static_cast(headermessage->GetData()); + o2::detectors::DetID id(header.detID); + LOG(debug2) << "I1 " << header.detID << " NAME " << id.getName() << " MB " + << data.At(index)->GetSize() / 1024. / 1024.; + + // get the detector that can interpret it + auto detector = mDetectorInstances[id].get(); + if (detector) { + detector->collectHits(eventID, data, index, header.shm); + } + } +} + +template +void O2HitMerger::consumeData(int eventID, fair::mq::Parts& data, int& index, BT& buffer) +{ + auto decodeddata = o2::base::decodeTMessage(data, index); + if (buffer.find(eventID) == buffer.end()) { + buffer[eventID] = typename BT::mapped_type(); + } + buffer[eventID].push_back(decodeddata); + // delete decodeddata; --> we store the pointers + index++; +} + +void O2HitMerger::fillSubEventInfoEntry(o2::data::SubEventInfo& info) +{ + if (mSubEventInfoBuffer.find(info.eventID) == mSubEventInfoBuffer.end()) { + mSubEventInfoBuffer[info.eventID] = std::list(); + } + mSubEventInfoBuffer[info.eventID].push_back(&info); +} + +bool O2HitMerger::waitForControlInput() +{ + o2::simpubsub::publishMessage(GetChannels()["merger-notifications"].at(0), o2::simpubsub::simStatusString("MERGER", "STATUS", "AWAITING INPUT")); + + auto factory = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); + auto channel = fair::mq::Channel{"o2sim-control", "sub", factory}; + auto controlsocketname = getenv("ALICE_O2SIMCONTROL"); + LOG(info) << "SOCKETNAME " << controlsocketname; + channel.Connect(std::string(controlsocketname)); + channel.Validate(); + std::unique_ptr reply(channel.NewMessage()); + + LOG(info) << "WAITING FOR INPUT"; + if (channel.Receive(reply) > 0) { + auto data = reply->GetData(); + auto size = reply->GetSize(); + + std::string command(reinterpret_cast(data), size); + LOG(info) << "message: " << command; + + o2::conf::SimReconfigData reconfig; + o2::conf::parseSimReconfigFromString(command, reconfig); + return ReInit(reconfig); + } else { + LOG(info) << "NOTHING RECEIVED"; + } + return true; +} + +bool O2HitMerger::ConditionalRun() +{ + auto& channel = GetChannels().at("simdata").at(0); + fair::mq::Parts request; + auto bytes = channel.Receive(request); + if (bytes < 0) { + LOG(error) << "Some error occurred on socket during receive on sim data"; + return true; // keep going + } + TStopwatch timer; + timer.Start(); + auto more = handleSimData(request, 0); + LOG(info) << "HitMerger processing took " << timer.RealTime(); + if (!more && mAsService) { + LOG(info) << " CONTROL "; + // if we are done treating data we may go back to init phase + // for the next batch + return waitForControlInput(); + } + + static bool initAcknowledged = false; + if (!initAcknowledged) { + primaryServer_sendShutdownPermission(GetChannels().at("o2sim-primserv-info").at(0)); + initAcknowledged = true; + } + + return more; +} + +bool O2HitMerger::handleSimData(fair::mq::Parts& data, int /*index*/) +{ + bool expectmore = true; + int index = 0; + auto infoptr = o2::base::decodeTMessage(data, index++); + o2::data::SubEventInfo& info = *infoptr; + // once a merge thread runs, the buffered info of a complete event may be freed at any time + const auto eventID = info.eventID; + const auto maxEvents = info.maxEvents; + auto accum = insertAdd(mPartsCheckSum, info.eventID, (uint32_t)info.part); + + LOG(info) << "SIMDATA channel got " << data.Size() << " parts for event " << info.eventID << " part " << info.part << " out of " << info.nparts; + + fillSubEventInfoEntry(info); + consumeData>(info.eventID, data, index, mMCTrackBuffer); + consumeData>(info.eventID, data, index, mTrackRefBuffer); + while (index < data.Size()) { + consumeHits(info.eventID, data, index); + } + + if (isDataComplete(accum, info.nparts)) { + LOG(info) << "Event " << info.eventID << " complete. Marking as flushable"; + mFlushableEvents[info.eventID] = true; + + // check if previous flush finished + // start merging only when no merging currently happening + // Like this we don't have to join/wait on the thread here and do not block the outer ConditionalRun handling + // TODO: Let this run fully asynchronously (not even triggered by ConditionalRun) + if (!mergingInProgress) { + if (mMergerIOThread.joinable()) { + mMergerIOThread.join(); + } + // start hit merging and flushing in a separate thread in order not to block + mMergerIOThread = std::thread([this]() { mergingInProgress = true; mergeAndFlushData(); mergingInProgress = false; }); + } + + mEventChecksum += eventID; + // we also need to check if we have all events + if (isDataComplete(mEventChecksum, maxEvents)) { + LOG(info) << "ALL EVENTS HERE; CHECKSUM " << mEventChecksum; + + // flush remaining data and close file + if (mMergerIOThread.joinable()) { + mMergerIOThread.join(); + } + mMergerIOThread = std::thread([this]() { mergingInProgress = true; mergeAndFlushData(); mergingInProgress = false; }); + if (mMergerIOThread.joinable()) { + mMergerIOThread.join(); + } + + expectmore = false; + } + + if (mPipeToDriver != -1) { + if (write(mPipeToDriver, &eventID, sizeof(eventID)) == -1) { + LOG(error) << "FAILED WRITING TO PIPE"; + }; + } + } + return expectmore; +} + +void O2HitMerger::cleanEvent(int eventID) +{ + auto release = [eventID](auto& buffer) { + auto iter = buffer.find(eventID); + if (iter != buffer.end()) { + for (auto ptr : iter->second) { + delete ptr; + } + iter->second = {}; + } + }; + release(mMCTrackBuffer); + release(mTrackRefBuffer); + release(mSubEventInfoBuffer); +} + +template +void O2HitMerger::backInsert(T const& from, T& to) +{ + std::copy(from.begin(), from.end(), std::back_inserter(to)); +} + +void O2HitMerger::reorderAndMergeMCTracks(int eventID, TTree* target, const std::vector& nprimaries, const std::vector& nsubevents, std::function const&)> tracks_analysis_hook, o2::dataformats::MCEventHeader const* mceventheader) +{ + // avoid doing this for trivial cases + std::vector* mcTracksPerSubEvent = nullptr; + auto targetdata = std::make_unique>(); + + auto& vectorOfSubEventMCTracks = mMCTrackBuffer[eventID]; + const auto entries = vectorOfSubEventMCTracks.size(); + + if (entries > 1) { + size_t ntracks = 0; + for (auto tracks : vectorOfSubEventMCTracks) { + ntracks += tracks->size(); + } + targetdata->reserve(ntracks); + // + // loop over subevents to store the primary events + // + int nprimTot = 0; + for (int entry = entries - 1; entry >= 0; --entry) { + int index = nsubevents[entry]; + nprimTot += nprimaries[index]; + for (int i = 0; i < nprimaries[index]; i++) { + auto& track = (*vectorOfSubEventMCTracks[index])[i]; + if (track.isTransported()) { // reset daughters only if track was transported, it will be fixed below + track.SetFirstDaughterTrackId(-1); + track.SetLastDaughterTrackId(-1); + } + targetdata->push_back(track); + } + } + // + // loop a second time to store the secondaries and fix the mother track IDs + // + Int_t idelta1 = nprimTot; + Int_t idelta0 = 0; + for (int entry = entries - 1; entry >= 0; --entry) { + int index = nsubevents[entry]; + + auto& subEventTracks = *(vectorOfSubEventMCTracks[index]); + // we need to fetch the right mctracks here!! + Int_t npart = (int)(subEventTracks.size()); + Int_t nprim = nprimaries[index]; + idelta1 -= nprim; + + for (Int_t i = nprim; i < npart; i++) { + auto& track = subEventTracks[i]; + Int_t cId = track.getMotherTrackId(); + if (cId >= nprim) { + cId += idelta1; + } else { + cId += idelta0; + } + track.SetMotherTrackId(cId); + track.SetFirstDaughterTrackId(-1); + + Int_t hwm = (int)(targetdata->size()); + auto& mother = (*targetdata)[cId]; + if (mother.getFirstDaughterTrackId() == -1) { + mother.SetFirstDaughterTrackId(hwm); + } + mother.SetLastDaughterTrackId(hwm); + + targetdata->push_back(track); + } + idelta0 += nprim; + idelta1 += npart; + } + } + // + // write to output + auto filladdr = (entries > 1) ? targetdata.get() : vectorOfSubEventMCTracks[0]; + + // we give the possibility to produce some MC track statistics + // to be saved as part of the MCHeader structure + tracks_analysis_hook(*filladdr); + + if (mWriteToDisc && target) { + auto targetbr = o2::base::getOrMakeBranch(*target, "MCTrack", &filladdr); + targetbr->SetAddress(&filladdr); + targetbr->Fill(); + targetbr->ResetAddress(); + } + // forwarding the track data to other consumers (pub/sub) + if (mForwardKine) { + auto free_tmessage = [](void* data, void* hint) { delete static_cast(hint); }; + auto& channel = GetChannels().at("kineforward").at(0); + TMessage* tmsg = new TMessage(kMESS_OBJECT); + tmsg->WriteObjectAny((void*)filladdr, TClass::GetClass("std::vector")); + std::unique_ptr trackmessage(channel.NewMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); + tmsg = new TMessage(kMESS_OBJECT); + tmsg->WriteObjectAny((void*)mceventheader, TClass::GetClass("o2::dataformats::MCEventHeader")); + std::unique_ptr headermessage(channel.NewMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); + fair::mq::Parts reply; + reply.AddPart(std::move(headermessage)); + reply.AddPart(std::move(trackmessage)); + channel.Send(reply); + LOG(info) << "Forward publish MC tracks on channel"; + } +} + +template +void O2HitMerger::remapTrackIdsAndMerge(std::string brname, int eventID, TTree& target, const std::vector& trackoffsets, const std::vector& nprimaries, const std::vector& subevOrdered, M& mapOfVectorOfTs) +{ + // + // Remap the mother track IDs by adding an offset. + // The offset calculated as the sum of the number of entries in the particle list of the previous subevents. + // This method is called by O2HitMerger::mergeAndFlushData(int) + // + T* incomingdata = nullptr; + std::unique_ptr targetdata(nullptr); + auto& vectorOfT = mapOfVectorOfTs[eventID]; + const auto entries = vectorOfT.size(); + + if (entries == 1) { + // nothing to do in case there is only one entry + incomingdata = vectorOfT[0]; + } else { + targetdata = std::make_unique(); + size_t nentries = 0; + for (auto data : vectorOfT) { + nentries += data->size(); + } + targetdata->reserve(nentries); + // loop over subevents + Int_t nprimTot = 0; + for (int entry = 0; entry < entries; entry++) { + nprimTot += nprimaries[entry]; + } + Int_t idelta0 = 0; + Int_t idelta1 = nprimTot; + for (int entry = entries - 1; entry >= 0; --entry) { + Int_t index = subevOrdered[entry]; + Int_t nprim = nprimaries[index]; + incomingdata = vectorOfT[index]; + idelta1 -= nprim; + for (auto& data : *incomingdata) { + updateTrackIdWithOffset(data, nprim, idelta0, idelta1); + targetdata->push_back(data); + } + idelta0 += nprim; + idelta1 += trackoffsets[index]; + } + } + auto dataaddr = (entries == 1) ? incomingdata : targetdata.get(); + auto targetbr = o2::base::getOrMakeBranch(target, brname.c_str(), &dataaddr); + targetbr->SetAddress(&dataaddr); + targetbr->Fill(); + targetbr->ResetAddress(); +} + +void O2HitMerger::updateTrackIdWithOffset(MCTrack& track, Int_t nprim, Int_t idelta0, Int_t idelta1) +{ + Int_t cId = track.getMotherTrackId(); + Int_t ioffset = (cId < nprim) ? idelta0 : idelta1; + if (cId != -1) { + track.SetMotherTrackId(cId + ioffset); + } +} + +void O2HitMerger::updateTrackIdWithOffset(TrackReference& ref, Int_t nprim, Int_t idelta0, Int_t idelta1) +{ + ref.setTrackID(o2::base::Detector::offsetTrackIndex(ref.getTrackID(), nprim, idelta0, idelta1)); +} + +void O2HitMerger::initHitTreeAndOutFile(std::string prefix, int detID) +{ + using o2::detectors::DetID; + if (mDetectorOutFiles.find(detID) != mDetectorOutFiles.end() && mDetectorOutFiles[detID]) { + LOG(warn) << "Hit outfile for detID " << DetID::getName(detID) << " already initialized --> Reopening"; + mDetectorOutFiles[detID]->Close(); + delete mDetectorOutFiles[detID]; + } + std::string name(o2::base::DetectorNameConf::getHitsFileName(detID, prefix)); + if (mWriteToDisc) { + mDetectorOutFiles[detID] = new TFile(name.c_str(), "RECREATE"); + mDetectorToTTreeMap[detID] = new TTree("o2sim", "o2sim"); + mDetectorToTTreeMap[detID]->SetDirectory(mDetectorOutFiles[detID]); + } else { + mDetectorOutFiles[detID] = nullptr; + mDetectorToTTreeMap[detID] = nullptr; + } +} + +bool O2HitMerger::mergeAndFlushData() +{ + auto isFlushable = [this](int eventID) { + auto iter = mFlushableEvents.find(eventID); + return iter != mFlushableEvents.end() && iter->second; + }; + + LOG(info) << "Launching merge kernel "; + if (!isFlushable(mNextFlushID)) { + return false; + } + for (; isFlushable(mNextFlushID); ++mNextFlushID) { + auto flusheventID = mNextFlushID; + LOG(info) << "Merge and flush event " << flusheventID; + auto iter = mSubEventInfoBuffer.find(flusheventID); + if (iter == mSubEventInfoBuffer.end() || iter->second.size() == 0 || mNExpectedEvents == 0) { + LOG(error) << "No data entries found for event " << flusheventID; + continue; + } + auto& subEventInfoList = iter->second; + + TStopwatch timer; + timer.Start(); + + // calculate trackoffsets + auto& confref = o2::conf::SimConfig::Instance(); + + // collecting trackoffsets (per data arrival id) to be used for global track-ID correction pass + std::vector trackoffsets; + // collecting primary particles in each subevent (data arrival id) + std::vector nprimaries; + // mapping of id to actual sub-event id (or part) + std::vector nsubevents; + + o2::dataformats::MCEventHeader* eventheader = nullptr; // The event header + + // the MC labels (trackID) for hits + for (auto info : subEventInfoList) { + assert(info->npersistenttracks >= 0); + trackoffsets.emplace_back(info->npersistenttracks); + nprimaries.emplace_back(info->nprimarytracks); + nsubevents.emplace_back(info->part); + if (eventheader == nullptr) { + eventheader = &info->mMCEventHeader; + } else { + eventheader->getMCEventStats().add(info->mMCEventHeader.getMCEventStats()); + } + } + + // now see which events can be discarded in any case due to no hits + if (confref.isFilterOutNoHitEvents()) { + if (eventheader && eventheader->getMCEventStats().getNHits() == 0) { + LOG(info) << " Taking out event " << flusheventID << " due to no hits "; + cleanEvent(flusheventID); + continue; + } + } + + // attention: We need to make sure that we write everything in the same event order + // but iteration over keys of a standard map in C++ is ordered + + // b) merge the general data + // + // for MCTrack remap the motherIds and merge at the same go + const auto entries = subEventInfoList.size(); + std::vector subevOrdered((int)(nsubevents.size())); + for (int entry = entries - 1; entry >= 0; --entry) { + subevOrdered[nsubevents[entry] - 1] = entry; + } + + // This is a hook that collects some useful statistics/properties on the event + // for use by other components; + // Properties are attached making use of the extensible "Info" feature which is already + // part of MCEventHeader. In such a way, one can also do this pass outside and attach arbitrary + // metadata to MCEventHeader without needing to change the data layout or API of the class itself. + // NOTE: This function might also be called directly in the primary server!? + auto mcheaderhook = [eventheader](std::vector const& tracks) { + int eta1Point2Counter = 0; + int eta1Point0Counter = 0; + int eta0Point8Counter = 0; + int eta1Point2CounterPi = 0; + int eta1Point0CounterPi = 0; + int eta0Point8CounterPi = 0; + int prims = 0; + for (auto& tr : tracks) { + if (tr.isPrimary()) { + prims++; + const auto eta = tr.GetEta(); + if (eta < 1.2) { + eta1Point2Counter++; + if (std::abs(tr.GetPdgCode()) == 211) { + eta1Point2CounterPi++; + } + } + if (eta < 1.0) { + eta1Point0Counter++; + if (std::abs(tr.GetPdgCode()) == 211) { + eta1Point0CounterPi++; + } + } + if (eta < 0.8) { + eta0Point8Counter++; + if (std::abs(tr.GetPdgCode()) == 211) { + eta0Point8CounterPi++; + } + } + } else { + break; // track layout is such that all prims are first anyway + } + } + // attach these properties to eventheader + // we only need to make the names standard + eventheader->putInfo("prims_eta_1.2", eta1Point2Counter); + eventheader->putInfo("prims_eta_1.0", eta1Point0Counter); + eventheader->putInfo("prims_eta_0.8", eta0Point8Counter); + eventheader->putInfo("prims_eta_1.2_pi", eta1Point2CounterPi); + eventheader->putInfo("prims_eta_1.0_pi", eta1Point0CounterPi); + eventheader->putInfo("prims_eta_0.8_pi", eta0Point8CounterPi); + eventheader->putInfo("prims_total", prims); + }; + // the kinematics and each detector go to separate files, so we merge and flush them concurrently + tbb::task_group tasks; + tasks.run([&]() { + reorderAndMergeMCTracks(flusheventID, mOutTree, nprimaries, subevOrdered, mcheaderhook, eventheader); + + if (mOutTree) { + // adjusting and merging track references + remapTrackIdsAndMerge>("TrackRefs", flusheventID, *mOutTree, trackoffsets, nprimaries, subevOrdered, mTrackRefBuffer); + + // write MC event headers + for (auto tree : {mOutTree, mMCHeaderTree}) { + auto headerbr = o2::base::getOrMakeBranch(*tree, "MCEventHeader.", &eventheader); + headerbr->SetAddress(&eventheader); + headerbr->Fill(); + headerbr->ResetAddress(); + } + + // increase the entry count in the trees + mOutTree->SetEntries(mOutTree->GetEntries() + 1); + mMCHeaderTree->SetEntries(mMCHeaderTree->GetEntries() + 1); + } + }); + + // c) do the merge procedure for all hits ... delegate this to detector specific functions + // since they know about types; number of branches; etc. + // this will also fix the trackIDs inside the hits + for (int id = 0; id < mDetectorInstances.size(); ++id) { + auto& det = mDetectorInstances[id]; + auto hittree = det ? mDetectorToTTreeMap[id] : nullptr; + if (hittree) { + tasks.run([&, det = det.get(), hittree]() { + det->mergeHitEntriesAndFlush(flusheventID, *hittree, trackoffsets, nprimaries, subevOrdered); + hittree->SetEntries(hittree->GetEntries() + 1); + }); + } + } + tasks.wait(); + + cleanEvent(flusheventID); + LOG(info) << "Merge/flush for event " << flusheventID << " took " << timer.RealTime(); + } + if (mWriteToDisc && mOutFile) { + LOG(info) << "Writing TTrees"; + std::vector files{mOutFile, mMCHeaderOnlyOutFile}; + for (int id = 0; id < mDetectorInstances.size(); ++id) { + if (mDetectorInstances[id] && mDetectorOutFiles[id]) { + files.push_back(mDetectorOutFiles[id]); + } + } + tbb::parallel_for_each(files, [](TFile* file) { file->Write("", TObject::kOverwrite); }); + } + return true; +} + +void O2HitMerger::initHitFiles(std::string prefix) +{ + using o2::detectors::DetID; + + // a little helper lambda + auto isActivated = [](std::string s) -> bool { + // access user configuration for list of wanted modules + auto& modulelist = o2::conf::SimConfig::Instance().getReadoutDetectors(); + auto active = std::find(modulelist.begin(), modulelist.end(), s) != modulelist.end(); + return active; }; + + for (int i = DetID::First; i <= DetID::Last; ++i) { + if (!isActivated(DetID::getName(i))) { + continue; + } + // init the detector specific output files + initHitTreeAndOutFile(prefix, i); + } + + // external (CAD) detectors are not part of the readout-detector list (their module names + // are not DetID names); their slots were determined in initDetInstances() + for (auto detID : mExternalDetIDs) { + initHitTreeAndOutFile(prefix, detID); + } +} + +// init detector instances used to write hit data to a TTree +void O2HitMerger::initDetInstances() +{ + using o2::detectors::DetID; + + // a little helper lambda + auto isActivated = [](std::string s) -> bool { + // access user configuration for list of wanted modules + auto& modulelist = o2::conf::SimConfig::Instance().getReadoutDetectors(); + auto active = std::find(modulelist.begin(), modulelist.end(), s) != modulelist.end(); + return active; }; + + // readout-only detector instances able to interpret and write the hits of each detector + using Factory = std::function()>; + const std::map factories{ + {DetID::TPC, [] { return std::make_unique(true); }}, + {DetID::ITS, [] { return std::make_unique(true); }}, + {DetID::MFT, [] { return std::make_unique(true); }}, + {DetID::TRD, [] { return std::make_unique(true); }}, + {DetID::PHS, [] { return std::make_unique(true); }}, + {DetID::CPV, [] { return std::make_unique(true); }}, + {DetID::EMC, [] { return std::make_unique(true); }}, + {DetID::HMP, [] { return std::make_unique(true); }}, + {DetID::TOF, [] { return std::make_unique(true); }}, + {DetID::FT0, [] { return std::make_unique(true); }}, + {DetID::FV0, [] { return std::make_unique(true); }}, + {DetID::FDD, [] { return std::make_unique(true); }}, + {DetID::MCH, [] { return std::make_unique(true); }}, + {DetID::MID, [] { return std::make_unique(true); }}, + {DetID::ZDC, [] { return std::make_unique(true); }}, + {DetID::FOC, [] { + TString sName = "$O2_ROOT/share/Detectors/Geometry/FOC/geometryFiles/geometry_Sheets.txt"; + gSystem->ExpandPathName(sName); + return std::make_unique(true, sName.Data()); + }}, +#ifdef ENABLE_UPGRADES + {DetID::IT3, [] { return std::make_unique(true, "IT3"); }}, + {DetID::TRK, [] { return std::make_unique(true); }}, + {DetID::FT3, [] { return std::make_unique(true); }}, + {DetID::FCT, [] { return std::make_unique(true); }}, + {DetID::TF3, [] { return std::make_unique(true); }}, + {DetID::RCH, [] { return std::make_unique(true); }}, + {DetID::MI3, [] { return std::make_unique(true); }}, + {DetID::ECL, [] { return std::make_unique(true); }}, + {DetID::FD3, [] { return std::make_unique(true); }}, +#endif + }; + + mDetectorInstances.resize(DetID::nDetectors); + for (int i = DetID::First; i <= DetID::Last; ++i) { + if (!isActivated(DetID::getName(i))) { + continue; + } + auto factory = factories.find(i); + if (factory == factories.end()) { + LOG(warning) << "O2HitMerger: no hit merging available for readout detector " << DetID::getName(i); + continue; + } + mDetectorInstances[i] = factory->second(); + } + + // also register external (CAD-derived) sensitive detectors so their hits are persisted + // in parallel (multi-worker) mode + initExternalDetInstances(); +} + +// init detector instances for external (CAD-derived) sensitive detectors. +// These are not part of the hard-coded DetID switch above: they are described in the +// external geometry JSON (the same file used by build_geometry.C on the worker side) and +// tied to an existing (free) DetID. The merger only needs an instance able to interpret the +// generic o2::ext::Hit wire format and write the "Hit" branch; no geometry is built here. +void O2HitMerger::initExternalDetInstances() +{ + using o2::detectors::DetID; + + auto& simConfig = o2::conf::SimConfig::Instance(); + const auto extGeomFile = simConfig.getExtGeomFilename(); + if (extGeomFile.empty()) { + return; + } + + // mirror the worker-side activation: an external detector participates when its module + // name is part of the active module list + auto const& activeModules = simConfig.getActiveModules(); + auto isActivated = [&activeModules](std::string const& s) -> bool { + return std::find(activeModules.begin(), activeModules.end(), s) != activeModules.end(); + }; + + for (auto* extdet : o2::ext::ExternalDetector::createFromJSON(extGeomFile)) { + const std::string name = extdet->GetName(); + if (!isActivated(name)) { + delete extdet; // not requested in the active module list + continue; + } + const int detID = extdet->GetDetId(); + if (detID < DetID::First || detID > DetID::Last) { + LOG(error) << "O2HitMerger: external detector " << name << " has invalid DetID " << detID << "; skipping"; + delete extdet; + continue; + } + if (mDetectorInstances[detID]) { + LOG(error) << "O2HitMerger: DetID " << DetID::getName(detID) << " requested by external detector " << name + << " is already occupied; its hits will not be persisted. Assign a free DetID."; + delete extdet; + continue; + } + mDetectorInstances[detID].reset(extdet); + mExternalDetIDs.emplace_back(detID); + LOG(info) << "O2HitMerger: registered external detector " << name << " on DetID " << DetID::getName(detID) + << " (branch " << name << "Hit)"; + } +} + +} // namespace devices +} // namespace o2 diff --git a/run/O2HitMerger.h b/run/O2HitMerger.h index 15f58c6dba351..9cc55a06fdf48 100644 --- a/run/O2HitMerger.h +++ b/run/O2HitMerger.h @@ -14,854 +14,104 @@ #ifndef ALICEO2_DEVICES_HITMERGER_H_ #define ALICEO2_DEVICES_HITMERGER_H_ +#include +#include +#include +#include #include #include -#include -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include -#include -#include "FairSystemInfo.h" - -#include "O2HitMerger.h" -#include "O2SimDevice.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "CommonUtils/ShmManager.h" -#include -#include -#include -#include -#include -#include -#include - -#include "SimPublishChannelHelper.h" - -#ifdef ENABLE_UPGRADES -#include -#include -#include -#include -#include -#include -#include -#include -#include -#endif - #include +#include +#include +#include +#include +#include +#include + +class TFile; +class TTree; namespace o2 { namespace devices { -// Function communicating to primary particle server that it is now safe to shutdown. -// From the perspective of o2-sim, this is the case when all configs have been propagated and the system -// is running ok: For instance after the HitMerger is initialized and got it's first data from Geant workers. -bool primaryServer_sendShutdownPermission(fair::mq::Channel& channel) -{ - std::unique_ptr request(channel.NewSimpleMessage((int)o2::O2PrimaryServerInfoRequest::AllowShutdown)); - std::unique_ptr reply(channel.NewMessage()); - - int timeoutinMS = 100; - if (channel.Send(request, timeoutinMS) > 0) { - LOG(info) << "Sending Shutdown permission to particle server"; - if (channel.Receive(reply, timeoutinMS) > 0) { - // the answer is a simple ack with a status code - LOG(info) << "Shutdown permission was acknowledged"; - } else { - LOG(error) << "No answer received within " << timeoutinMS << "ms\n"; - return false; - } - return true; - } - return false; -} - class O2HitMerger : public fair::mq::Device { - - class TMessageWrapper : public TMessage - { - public: - TMessageWrapper(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); } - ~TMessageWrapper() override = default; - }; - public: /// Default constructor - O2HitMerger() - { - mTimer.Start(); - mInitialOutputDir = std::filesystem::current_path().string(); - mCurrentOutputDir = mInitialOutputDir; - } + O2HitMerger(); /// Default destructor - ~O2HitMerger() override - { - FairSystemInfo sysinfo; - LOG(info) << "TIME-STAMP " << mTimer.RealTime() << "\t"; - mTimer.Continue(); - LOG(info) << "MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " - << sysinfo.GetMaxMemory() << " MB\n"; - } + ~O2HitMerger() override; private: /// Overloads the InitTask() method of fair::mq::Device - void InitTask() final - { - LOG(info) << "INIT HIT MERGER"; - ROOT::EnableThreadSafety(); + void InitTask() final; - std::string outfilename("o2sim_merged_hits.root"); // default name - // query the sim config ... which is used to extract the filenames - if (o2::devices::O2SimDevice::querySimConfig(GetChannels().at("o2sim-primserv-info").at(0))) { - outfilename = o2::base::NameConf::getMCKinematicsFileName(o2::conf::SimConfig::Instance().getOutPrefix().c_str()); - mNExpectedEvents = o2::conf::SimConfig::Instance().getNEvents(); - } else { - // we didn't manage to get a configuration --> better to fail - LOG(fatal) << "No configuration received. Aborting"; - } - mAsService = o2::conf::SimConfig::Instance().asService(); - mForwardKine = o2::conf::SimConfig::Instance().forwardKine(); - mWriteToDisc = o2::conf::SimConfig::Instance().writeToDisc(); - - mOutFileName = outfilename.c_str(); - if (mWriteToDisc) { - mOutFile = new TFile(outfilename.c_str(), "RECREATE"); - mOutTree = new TTree("o2sim", "o2sim"); - mOutTree->SetDirectory(mOutFile); - - mMCHeaderOnlyOutFile = new TFile(o2::base::NameConf::getMCHeadersFileName(o2::conf::SimConfig::Instance().getOutPrefix().c_str()).c_str(), "RECREATE"); - mMCHeaderTree = new TTree("o2sim", "o2sim"); - mMCHeaderTree->SetDirectory(mMCHeaderOnlyOutFile); - } - // detectors init only once - if (mDetectorInstances.size() == 0) { - initDetInstances(); - // has to be after init of Detectors - o2::utils::ShmManager::Instance().attachToGlobalSegment(); - initHitFiles(o2::conf::SimConfig::Instance().getOutPrefix()); - } - - // init pipe - auto pipeenv = getenv("ALICE_O2SIMMERGERTODRIVER_PIPE"); - if (pipeenv) { - mPipeToDriver = atoi(pipeenv); - LOG(info) << "ASSIGNED PIPE HANDLE " << mPipeToDriver; - } else { - LOG(warning) << "DID NOT FIND ENVIRONMENT VARIABLE TO INIT PIPE"; - } - - // if no data to expect we shut down the device NOW since it would otherwise hang - if (mNExpectedEvents == 0) { - if (mAsService) { - waitForControlInput(); - } else { - LOG(info) << "NOT EXPECTING ANY DATA; SHUTTING DOWN"; - raise(SIGINT); - } - } - } - - bool setWorkingDirectory(std::string const& dir) - { - namespace fs = std::filesystem; - - // sets the output directory where simulation files are produced - // and creates it when it doesn't exist already - - // 2 possibilities: - // a) dir is relative dir. Then we interpret it as relative to the initial - // base directory - // b) or dir is itself absolut. - try { - fs::current_path(fs::path(mInitialOutputDir)); // <--- to make sure relative start is always the same - if (!dir.empty()) { - auto absolutePath = fs::absolute(fs::path(dir)); - if (!fs::exists(absolutePath)) { - if (!fs::create_directory(absolutePath)) { - LOG(error) << "Could not create directory " << absolutePath.string(); - return false; - } - } - // set the current path - fs::current_path(absolutePath.string().c_str()); - mCurrentOutputDir = fs::current_path().string(); - } - LOG(info) << "FINAL PATH " << mCurrentOutputDir; - } catch (std::exception e) { - LOG(error) << " could not change path to " << dir; - } - return true; - } + bool setWorkingDirectory(std::string const& dir); // function for intermediate/on-the-fly reinitializations - bool ReInit(o2::conf::SimReconfigData const& reconfig) - { - if (reconfig.stop) { - return false; - } - if (!setWorkingDirectory(reconfig.outputDir)) { - return false; - } - - std::string outfilename("o2sim_merged_hits.root"); // default name - outfilename = o2::base::NameConf::getMCKinematicsFileName(reconfig.outputPrefix); - mNExpectedEvents = reconfig.nEvents; - mOutFileName = outfilename.c_str(); - if (mWriteToDisc) { - mOutFile = new TFile(outfilename.c_str(), "RECREATE"); - mOutTree = new TTree("o2sim", "o2sim"); - mOutTree->SetDirectory(mOutFile); - - mMCHeaderOnlyOutFile = new TFile(o2::base::NameConf::getMCHeadersFileName(reconfig.outputPrefix).c_str(), "RECREATE"); - mMCHeaderTree = new TTree("o2sim", "o2sim"); - mMCHeaderTree->SetDirectory(mMCHeaderOnlyOutFile); - } - // reinit detectorInstance files (also make sure they are closed before continuing) - initHitFiles(reconfig.outputPrefix); - - // clear "counter" datastructures - mPartsCheckSum.clear(); - mEventChecksum = 0; - - // clear collector datastructures - mMCTrackBuffer.clear(); - mTrackRefBuffer.clear(); - mSubEventInfoBuffer.clear(); - mFlushableEvents.clear(); - mNextFlushID = 1; - - return true; - } + bool ReInit(o2::conf::SimReconfigData const& reconfig); template - V insertAdd(std::map& m, T const& key, V value) - { - const auto iter = m.find(key); - V accum{0}; - if (iter != m.end()) { - iter->second += value; - accum = iter->second; - } else { - m.insert(std::make_pair(key, value)); - accum = value; - } - return accum; - } + V insertAdd(std::map& m, T const& key, V value); template - bool isDataComplete(T checksum, T nparts) - { - return checksum == nparts * (nparts + 1) / 2; - } + bool isDataComplete(T checksum, T nparts); - void consumeHits(int eventID, fair::mq::Parts& data, int& index) - { - auto detIDmessage = std::move(data.At(index++)); - // this should be a detector ID - if (detIDmessage->GetSize() == 4) { - auto ptr = (int*)detIDmessage->GetData(); - o2::detectors::DetID id(ptr[0]); - LOG(debug2) << "I1 " << ptr[0] << " NAME " << id.getName() << " MB " - << data.At(index)->GetSize() / 1024. / 1024.; - - // get the detector that can interpret it - auto detector = mDetectorInstances[id].get(); - if (detector) { - detector->collectHits(eventID, data, index); - } - } - } + void consumeHits(int eventID, fair::mq::Parts& data, int& index); template - void consumeData(int eventID, fair::mq::Parts& data, int& index, BT& buffer) - { - auto decodeddata = o2::base::decodeTMessage(data, index); - if (buffer.find(eventID) == buffer.end()) { - buffer[eventID] = typename BT::mapped_type(); - } - buffer[eventID].push_back(decodeddata); - // delete decodeddata; --> we store the pointers - index++; - } + void consumeData(int eventID, fair::mq::Parts& data, int& index, BT& buffer); // fills a special branch of SubEventInfos in order to keep // track of which entry corresponds to which event etc. // also creates the MCEventHeader branch expected for physics analysis - void fillSubEventInfoEntry(o2::data::SubEventInfo& info) - { - if (mSubEventInfoBuffer.find(info.eventID) == mSubEventInfoBuffer.end()) { - mSubEventInfoBuffer[info.eventID] = std::list(); - } - mSubEventInfoBuffer[info.eventID].push_back(&info); - } - - bool waitForControlInput() - { - o2::simpubsub::publishMessage(GetChannels()["merger-notifications"].at(0), o2::simpubsub::simStatusString("MERGER", "STATUS", "AWAITING INPUT")); - - auto factory = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); - auto channel = fair::mq::Channel{"o2sim-control", "sub", factory}; - auto controlsocketname = getenv("ALICE_O2SIMCONTROL"); - LOG(info) << "SOCKETNAME " << controlsocketname; - channel.Connect(std::string(controlsocketname)); - channel.Validate(); - std::unique_ptr reply(channel.NewMessage()); - - LOG(info) << "WAITING FOR INPUT"; - if (channel.Receive(reply) > 0) { - auto data = reply->GetData(); - auto size = reply->GetSize(); - - std::string command(reinterpret_cast(data), size); - LOG(info) << "message: " << command; - - o2::conf::SimReconfigData reconfig; - o2::conf::parseSimReconfigFromString(command, reconfig); - return ReInit(reconfig); - } else { - LOG(info) << "NOTHING RECEIVED"; - } - return true; - } - - bool ConditionalRun() override - { - auto& channel = GetChannels().at("simdata").at(0); - fair::mq::Parts request; - auto bytes = channel.Receive(request); - if (bytes < 0) { - LOG(error) << "Some error occurred on socket during receive on sim data"; - return true; // keep going - } - TStopwatch timer; - timer.Start(); - auto more = handleSimData(request, 0); - LOG(info) << "HitMerger processing took " << timer.RealTime(); - if (!more && mAsService) { - LOG(info) << " CONTROL "; - // if we are done treating data we may go back to init phase - // for the next batch - return waitForControlInput(); - } - - static bool initAcknowledged = false; - if (!initAcknowledged) { - primaryServer_sendShutdownPermission(GetChannels().at("o2sim-primserv-info").at(0)); - initAcknowledged = true; - } - - return more; - } - - bool handleSimData(fair::mq::Parts& data, int /*index*/) - { - bool expectmore = true; - int index = 0; - auto infoptr = o2::base::decodeTMessage(data, index++); - o2::data::SubEventInfo& info = *infoptr; - auto accum = insertAdd(mPartsCheckSum, info.eventID, (uint32_t)info.part); - - LOG(info) << "SIMDATA channel got " << data.Size() << " parts for event " << info.eventID << " part " << info.part << " out of " << info.nparts; - - fillSubEventInfoEntry(info); - consumeData>(info.eventID, data, index, mMCTrackBuffer); - consumeData>(info.eventID, data, index, mTrackRefBuffer); - while (index < data.Size()) { - consumeHits(info.eventID, data, index); - } + void fillSubEventInfoEntry(o2::data::SubEventInfo& info); - if (isDataComplete(accum, info.nparts)) { - LOG(info) << "Event " << info.eventID << " complete. Marking as flushable"; - mFlushableEvents[info.eventID] = true; + bool waitForControlInput(); - // check if previous flush finished - // start merging only when no merging currently happening - // Like this we don't have to join/wait on the thread here and do not block the outer ConditionalRun handling - // TODO: Let this run fully asynchronously (not even triggered by ConditionalRun) - if (!mergingInProgress) { - if (mMergerIOThread.joinable()) { - mMergerIOThread.join(); - } - // start hit merging and flushing in a separate thread in order not to block - mMergerIOThread = std::thread([info, this]() { mergingInProgress = true; mergeAndFlushData(); mergingInProgress = false; }); - } + bool ConditionalRun() override; - mEventChecksum += info.eventID; - // we also need to check if we have all events - if (isDataComplete(mEventChecksum, info.maxEvents)) { - LOG(info) << "ALL EVENTS HERE; CHECKSUM " << mEventChecksum; + bool handleSimData(fair::mq::Parts& data, int /*index*/); - // flush remaining data and close file - if (mMergerIOThread.joinable()) { - mMergerIOThread.join(); - } - mMergerIOThread = std::thread([info, this]() { mergingInProgress = true; mergeAndFlushData(); mergingInProgress = false; }); - if (mMergerIOThread.joinable()) { - mMergerIOThread.join(); - } - - expectmore = false; - } - - if (mPipeToDriver != -1) { - if (write(mPipeToDriver, &info.eventID, sizeof(info.eventID)) == -1) { - LOG(error) << "FAILED WRITING TO PIPE"; - }; - } - } - return expectmore; - } - - void cleanEvent(int eventID) - { - // cleanup intermediate per-Event buffers - } + // releases the buffered data of an event once it is flushed or discarded + void cleanEvent(int eventID); template - void backInsert(T const& from, T& to) - { - std::copy(from.begin(), from.end(), std::back_inserter(to)); - } - - void reorderAndMergeMCTracks(int eventID, TTree* target, const std::vector& nprimaries, const std::vector& nsubevents, std::function const&)> tracks_analysis_hook, o2::dataformats::MCEventHeader const* mceventheader) - { - // avoid doing this for trivial cases - std::vector* mcTracksPerSubEvent = nullptr; - auto targetdata = std::make_unique>(); - - auto& vectorOfSubEventMCTracks = mMCTrackBuffer[eventID]; - const auto entries = vectorOfSubEventMCTracks.size(); - - if (entries > 1) { - // - // loop over subevents to store the primary events - // - int nprimTot = 0; - for (int entry = entries - 1; entry >= 0; --entry) { - int index = nsubevents[entry]; - nprimTot += nprimaries[index]; - printf("merge %d %5d %5d %5d \n", entry, index, nsubevents[entry], nsubevents[index]); - for (int i = 0; i < nprimaries[index]; i++) { - auto& track = (*vectorOfSubEventMCTracks[index])[i]; - if (track.isTransported()) { // reset daughters only if track was transported, it will be fixed below - track.SetFirstDaughterTrackId(-1); - track.SetLastDaughterTrackId(-1); - } - targetdata->push_back(track); - } - } - // - // loop a second time to store the secondaries and fix the mother track IDs - // - Int_t idelta1 = nprimTot; - Int_t idelta0 = 0; - for (int entry = entries - 1; entry >= 0; --entry) { - int index = nsubevents[entry]; - - auto& subEventTracks = *(vectorOfSubEventMCTracks[index]); - // we need to fetch the right mctracks here!! - Int_t npart = (int)(subEventTracks.size()); - Int_t nprim = nprimaries[index]; - idelta1 -= nprim; + void backInsert(T const& from, T& to); - for (Int_t i = nprim; i < npart; i++) { - auto& track = subEventTracks[i]; - Int_t cId = track.getMotherTrackId(); - if (cId >= nprim) { - cId += idelta1; - } else { - cId += idelta0; - } - track.SetMotherTrackId(cId); - track.SetFirstDaughterTrackId(-1); - - Int_t hwm = (int)(targetdata->size()); - auto& mother = (*targetdata)[cId]; - if (mother.getFirstDaughterTrackId() == -1) { - mother.SetFirstDaughterTrackId(hwm); - } - mother.SetLastDaughterTrackId(hwm); - - targetdata->push_back(track); - } - idelta0 += nprim; - idelta1 += npart; - } - } - // - // write to output - auto filladdr = (entries > 1) ? targetdata.get() : vectorOfSubEventMCTracks[0]; - - // we give the possibility to produce some MC track statistics - // to be saved as part of the MCHeader structure - tracks_analysis_hook(*filladdr); - - if (mWriteToDisc && target) { - auto targetbr = o2::base::getOrMakeBranch(*target, "MCTrack", &filladdr); - targetbr->SetAddress(&filladdr); - targetbr->Fill(); - targetbr->ResetAddress(); - } - // forwarding the track data to other consumers (pub/sub) - if (mForwardKine) { - auto free_tmessage = [](void* data, void* hint) { delete static_cast(hint); }; - auto& channel = GetChannels().at("kineforward").at(0); - TMessage* tmsg = new TMessage(kMESS_OBJECT); - tmsg->WriteObjectAny((void*)filladdr, TClass::GetClass("std::vector")); - std::unique_ptr trackmessage(channel.NewMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); - tmsg = new TMessage(kMESS_OBJECT); - tmsg->WriteObjectAny((void*)mceventheader, TClass::GetClass("o2::dataformats::MCEventHeader")); - std::unique_ptr headermessage(channel.NewMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); - fair::mq::Parts reply; - reply.AddPart(std::move(headermessage)); - reply.AddPart(std::move(trackmessage)); - channel.Send(reply); - LOG(info) << "Forward publish MC tracks on channel"; - } - - // cleanup buffered data - for (auto ptr : vectorOfSubEventMCTracks) { - delete ptr; // avoid this by using unique ptr - } - } + void reorderAndMergeMCTracks(int eventID, TTree* target, const std::vector& nprimaries, const std::vector& nsubevents, std::function const&)> tracks_analysis_hook, o2::dataformats::MCEventHeader const* mceventheader); template void remapTrackIdsAndMerge(std::string brname, int eventID, TTree& target, - const std::vector& trackoffsets, const std::vector& nprimaries, const std::vector& subevOrdered, M& mapOfVectorOfTs) - { - // - // Remap the mother track IDs by adding an offset. - // The offset calculated as the sum of the number of entries in the particle list of the previous subevents. - // This method is called by O2HitMerger::mergeAndFlushData(int) - // - T* incomingdata = nullptr; - std::unique_ptr targetdata(nullptr); - auto& vectorOfT = mapOfVectorOfTs[eventID]; - const auto entries = vectorOfT.size(); - - if (entries == 1) { - // nothing to do in case there is only one entry - incomingdata = vectorOfT[0]; - } else { - targetdata = std::make_unique(); - // loop over subevents - Int_t nprimTot = 0; - for (int entry = 0; entry < entries; entry++) { - nprimTot += nprimaries[entry]; - } - Int_t idelta0 = 0; - Int_t idelta1 = nprimTot; - for (int entry = entries - 1; entry >= 0; --entry) { - Int_t index = subevOrdered[entry]; - Int_t nprim = nprimaries[index]; - incomingdata = vectorOfT[index]; - idelta1 -= nprim; - for (auto& data : *incomingdata) { - updateTrackIdWithOffset(data, nprim, idelta0, idelta1); - targetdata->push_back(data); - } - idelta0 += nprim; - idelta1 += trackoffsets[index]; - } - } - auto dataaddr = (entries == 1) ? incomingdata : targetdata.get(); - auto targetbr = o2::base::getOrMakeBranch(target, brname.c_str(), &dataaddr); - targetbr->SetAddress(&dataaddr); - targetbr->Fill(); - targetbr->ResetAddress(); + const std::vector& trackoffsets, const std::vector& nprimaries, const std::vector& subevOrdered, M& mapOfVectorOfTs); - // cleanup mem - for (auto ptr : vectorOfT) { - delete ptr; // avoid this by using unique ptr - } - } + void updateTrackIdWithOffset(MCTrack& track, Int_t nprim, Int_t idelta0, Int_t idelta1); - void updateTrackIdWithOffset(MCTrack& track, Int_t nprim, Int_t idelta0, Int_t idelta1) - { - Int_t cId = track.getMotherTrackId(); - Int_t ioffset = (cId < nprim) ? idelta0 : idelta1; - if (cId != -1) { - track.SetMotherTrackId(cId + ioffset); - } - } + void updateTrackIdWithOffset(TrackReference& ref, Int_t nprim, Int_t idelta0, Int_t idelta1); - void updateTrackIdWithOffset(TrackReference& ref, Int_t nprim, Int_t idelta0, Int_t idelta1) - { - ref.setTrackID(o2::base::Detector::offsetTrackIndex(ref.getTrackID(), nprim, idelta0, idelta1)); - } - - void initHitTreeAndOutFile(std::string prefix, int detID) - { - using o2::detectors::DetID; - if (mDetectorOutFiles.find(detID) != mDetectorOutFiles.end() && mDetectorOutFiles[detID]) { - LOG(warn) << "Hit outfile for detID " << DetID::getName(detID) << " already initialized --> Reopening"; - mDetectorOutFiles[detID]->Close(); - delete mDetectorOutFiles[detID]; - } - std::string name(o2::base::DetectorNameConf::getHitsFileName(detID, prefix)); - if (mWriteToDisc) { - mDetectorOutFiles[detID] = new TFile(name.c_str(), "RECREATE"); - mDetectorToTTreeMap[detID] = new TTree("o2sim", "o2sim"); - mDetectorToTTreeMap[detID]->SetDirectory(mDetectorOutFiles[detID]); - } else { - mDetectorOutFiles[detID] = nullptr; - mDetectorToTTreeMap[detID] = nullptr; - } - } + void initHitTreeAndOutFile(std::string prefix, int detID); // This method goes over the buffers containing data for a given event; potentially merges // them and flushes into the actual output file. // The method can be called asynchronously to data collection - bool mergeAndFlushData() - { - auto checkIfNextFlushable = [this]() -> bool { - mNextFlushID++; - return mFlushableEvents.find(mNextFlushID) != mFlushableEvents.end() && mFlushableEvents[mNextFlushID] == true; - }; - - LOG(info) << "Launching merge kernel "; - bool canflush = mFlushableEvents.find(mNextFlushID) != mFlushableEvents.end() && mFlushableEvents[mNextFlushID] == true; - if (!canflush) { - return false; - } - while (canflush == true) { - auto flusheventID = mNextFlushID; - LOG(info) << "Merge and flush event " << flusheventID; - auto iter = mSubEventInfoBuffer.find(flusheventID); - if (iter == mSubEventInfoBuffer.end()) { - LOG(error) << "No info/data found for event " << flusheventID; - if (!checkIfNextFlushable()) { - return false; - } - } - - auto& subEventInfoList = (*iter).second; - if (subEventInfoList.size() == 0 || mNExpectedEvents == 0) { - LOG(error) << "No data entries found for event " << flusheventID; - if (!checkIfNextFlushable()) { - return false; - } - } - - TStopwatch timer; - timer.Start(); - - // calculate trackoffsets - auto& confref = o2::conf::SimConfig::Instance(); - - // collecting trackoffsets (per data arrival id) to be used for global track-ID correction pass - std::vector trackoffsets; - // collecting primary particles in each subevent (data arrival id) - std::vector nprimaries; - // mapping of id to actual sub-event id (or part) - std::vector nsubevents; - - o2::dataformats::MCEventHeader* eventheader = nullptr; // The event header - - // the MC labels (trackID) for hits - for (auto info : subEventInfoList) { - assert(info->npersistenttracks >= 0); - trackoffsets.emplace_back(info->npersistenttracks); - nprimaries.emplace_back(info->nprimarytracks); - nsubevents.emplace_back(info->part); - if (eventheader == nullptr) { - eventheader = &info->mMCEventHeader; - } else { - eventheader->getMCEventStats().add(info->mMCEventHeader.getMCEventStats()); - } - } - - // now see which events can be discarded in any case due to no hits - if (confref.isFilterOutNoHitEvents()) { - if (eventheader && eventheader->getMCEventStats().getNHits() == 0) { - LOG(info) << " Taking out event " << flusheventID << " due to no hits "; - cleanEvent(flusheventID); - if (!checkIfNextFlushable()) { - return true; - } - } - } - - // attention: We need to make sure that we write everything in the same event order - // but iteration over keys of a standard map in C++ is ordered - - // b) merge the general data - // - // for MCTrack remap the motherIds and merge at the same go - const auto entries = subEventInfoList.size(); - std::vector subevOrdered((int)(nsubevents.size())); - for (int entry = entries - 1; entry >= 0; --entry) { - subevOrdered[nsubevents[entry] - 1] = entry; - printf("HitMerger entry: %d nprimry: %5d trackoffset: %5d \n", entry, nprimaries[entry], trackoffsets[entry]); - } - - // This is a hook that collects some useful statistics/properties on the event - // for use by other components; - // Properties are attached making use of the extensible "Info" feature which is already - // part of MCEventHeader. In such a way, one can also do this pass outside and attach arbitrary - // metadata to MCEventHeader without needing to change the data layout or API of the class itself. - // NOTE: This function might also be called directly in the primary server!? - auto mcheaderhook = [eventheader](std::vector const& tracks) { - int eta1Point2Counter = 0; - int eta1Point0Counter = 0; - int eta0Point8Counter = 0; - int eta1Point2CounterPi = 0; - int eta1Point0CounterPi = 0; - int eta0Point8CounterPi = 0; - int prims = 0; - for (auto& tr : tracks) { - if (tr.isPrimary()) { - prims++; - const auto eta = tr.GetEta(); - if (eta < 1.2) { - eta1Point2Counter++; - if (std::abs(tr.GetPdgCode()) == 211) { - eta1Point2CounterPi++; - } - } - if (eta < 1.0) { - eta1Point0Counter++; - if (std::abs(tr.GetPdgCode()) == 211) { - eta1Point0CounterPi++; - } - } - if (eta < 0.8) { - eta0Point8Counter++; - if (std::abs(tr.GetPdgCode()) == 211) { - eta0Point8CounterPi++; - } - } - } else { - break; // track layout is such that all prims are first anyway - } - } - // attach these properties to eventheader - // we only need to make the names standard - eventheader->putInfo("prims_eta_1.2", eta1Point2Counter); - eventheader->putInfo("prims_eta_1.0", eta1Point0Counter); - eventheader->putInfo("prims_eta_0.8", eta0Point8Counter); - eventheader->putInfo("prims_eta_1.2_pi", eta1Point2CounterPi); - eventheader->putInfo("prims_eta_1.0_pi", eta1Point0CounterPi); - eventheader->putInfo("prims_eta_0.8_pi", eta0Point8CounterPi); - eventheader->putInfo("prims_total", prims); - }; - reorderAndMergeMCTracks(flusheventID, mOutTree, nprimaries, subevOrdered, mcheaderhook, eventheader); - - if (mOutTree) { - // adjusting and merging track references - remapTrackIdsAndMerge>("TrackRefs", flusheventID, *mOutTree, trackoffsets, nprimaries, subevOrdered, mTrackRefBuffer); - - // write MC event headers - { - auto headerbr = o2::base::getOrMakeBranch(*mOutTree, "MCEventHeader.", &eventheader); - headerbr->SetAddress(&eventheader); - headerbr->Fill(); - headerbr->ResetAddress(); - } - - { - auto headerbr = o2::base::getOrMakeBranch(*mMCHeaderTree, "MCEventHeader.", &eventheader); - headerbr->SetAddress(&eventheader); - headerbr->Fill(); - headerbr->ResetAddress(); - } - } - - // c) do the merge procedure for all hits ... delegate this to detector specific functions - // since they know about types; number of branches; etc. - // this will also fix the trackIDs inside the hits - for (int id = 0; id < mDetectorInstances.size(); ++id) { - auto& det = mDetectorInstances[id]; - if (det) { - auto hittree = mDetectorToTTreeMap[id]; - if (hittree) { - det->mergeHitEntriesAndFlush(flusheventID, *hittree, trackoffsets, nprimaries, subevOrdered); - hittree->SetEntries(hittree->GetEntries() + 1); - LOG(info) << "flushing tree to file " << hittree->GetDirectory()->GetFile()->GetName(); - } - } - } - - // increase the entry count in the tree - if (mOutTree) { - mOutTree->SetEntries(mOutTree->GetEntries() + 1); - LOG(info) << "outtree has file " << mOutTree->GetDirectory()->GetFile()->GetName(); - } - if (mMCHeaderTree) { - mMCHeaderTree->SetEntries(mMCHeaderTree->GetEntries() + 1); - LOG(info) << "mc header outtree has file " << mMCHeaderTree->GetDirectory()->GetFile()->GetName(); - } - - cleanEvent(flusheventID); - LOG(info) << "Merge/flush for event " << flusheventID << " took " << timer.RealTime(); - if (!checkIfNextFlushable()) { - break; - } - } // end while - if (mWriteToDisc && mOutFile) { - LOG(info) << "Writing TTrees"; - mOutFile->Write("", TObject::kOverwrite); - for (int id = 0; id < mDetectorInstances.size(); ++id) { - auto& det = mDetectorInstances[id]; - if (det && mDetectorOutFiles[id]) { - mDetectorOutFiles[id]->Write("", TObject::kOverwrite); - } - } - if (mMCHeaderOnlyOutFile) { - mMCHeaderOnlyOutFile->Write("", TObject::kOverwrite); - } - } - return true; - } + bool mergeAndFlushData(); std::map mPartsCheckSum; //! mapping event id -> part checksum used to detect when all info std::string mOutFileName; //! // structures for the final flush - TFile* mOutFile; //! outfile for kinematics - TTree* mOutTree; //! tree (kinematics) associated to mOutFile - TFile* mMCHeaderOnlyOutFile; //! outfile for header only information - TTree* mMCHeaderTree; //! tree to hold MCHeader branch in mMCHeaderOnlyOutFile; + TFile* mOutFile = nullptr; //! outfile for kinematics + TTree* mOutTree = nullptr; //! tree (kinematics) associated to mOutFile + TFile* mMCHeaderOnlyOutFile = nullptr; //! outfile for header only information + TTree* mMCHeaderTree = nullptr; //! tree to hold MCHeader branch in mMCHeaderOnlyOutFile; template using Hashtable = tbb::concurrent_unordered_map; @@ -870,7 +120,7 @@ class O2HitMerger : public fair::mq::Device // intermediate structures to collect data per event std::thread mMergerIOThread; //! a thread used to do hit merging and IO flushing asynchronously - bool mergingInProgress = false; + std::atomic mergingInProgress{false}; Hashtable*>> mMCTrackBuffer; //! vector of sub-event track vectors; one per event Hashtable*>> mTrackRefBuffer; //! @@ -904,214 +154,6 @@ class O2HitMerger : public fair::mq::Device void initHitFiles(std::string prefix); }; -void O2HitMerger::initHitFiles(std::string prefix) -{ - using o2::detectors::DetID; - - // a little helper lambda - auto isActivated = [](std::string s) -> bool { - // access user configuration for list of wanted modules - auto& modulelist = o2::conf::SimConfig::Instance().getReadoutDetectors(); - auto active = std::find(modulelist.begin(), modulelist.end(), s) != modulelist.end(); - return active; }; - - for (int i = DetID::First; i <= DetID::Last; ++i) { - if (!isActivated(DetID::getName(i))) { - continue; - } - // init the detector specific output files - initHitTreeAndOutFile(prefix, i); - } - - // external (CAD) detectors are not part of the readout-detector list (their module names - // are not DetID names); their slots were determined in initDetInstances() - for (auto detID : mExternalDetIDs) { - initHitTreeAndOutFile(prefix, detID); - } -} - -// init detector instances used to write hit data to a TTree -void O2HitMerger::initDetInstances() -{ - using o2::detectors::DetID; - - // a little helper lambda - auto isActivated = [](std::string s) -> bool { - // access user configuration for list of wanted modules - auto& modulelist = o2::conf::SimConfig::Instance().getReadoutDetectors(); - auto active = std::find(modulelist.begin(), modulelist.end(), s) != modulelist.end(); - return active; }; - - mDetectorInstances.resize(DetID::nDetectors); - // like a factory of detector objects - - int counter = 0; - for (int i = DetID::First; i <= DetID::Last; ++i) { - if (!isActivated(DetID::getName(i))) { - continue; - } - - if (i == DetID::TPC) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::ITS) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::MFT) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::TRD) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::PHS) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::CPV) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::EMC) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::HMP) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::TOF) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FT0) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FV0) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FDD) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::MCH) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::MID) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::ZDC) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FOC) { - TString sName = "$O2_ROOT/share/Detectors/Geometry/FOC/geometryFiles/geometry_Sheets.txt"; - gSystem->ExpandPathName(sName); - mDetectorInstances[i] = std::move(std::make_unique(true, sName.Data())); - counter++; - } -#ifdef ENABLE_UPGRADES - if (i == DetID::IT3) { - mDetectorInstances[i] = std::move(std::make_unique(true, "IT3")); - counter++; - } - if (i == DetID::TRK) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FT3) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FCT) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::TF3) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::RCH) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::MI3) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::ECL) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } - if (i == DetID::FD3) { - mDetectorInstances[i] = std::move(std::make_unique(true)); - counter++; - } -#endif - } - if (counter != DetID::nDetectors) { - LOG(warning) << " O2HitMerger: Some Detectors are potentially missing in this initialization "; - } - - // also register external (CAD-derived) sensitive detectors so their hits are persisted - // in parallel (multi-worker) mode - initExternalDetInstances(); -} - -// init detector instances for external (CAD-derived) sensitive detectors. -// These are not part of the hard-coded DetID switch above: they are described in the -// external geometry JSON (the same file used by build_geometry.C on the worker side) and -// tied to an existing (free) DetID. The merger only needs an instance able to interpret the -// generic o2::ext::Hit wire format and write the "Hit" branch; no geometry is built here. -void O2HitMerger::initExternalDetInstances() -{ - using o2::detectors::DetID; - - auto& simConfig = o2::conf::SimConfig::Instance(); - const auto extGeomFile = simConfig.getExtGeomFilename(); - if (extGeomFile.empty()) { - return; - } - - // mirror the worker-side activation: an external detector participates when its module - // name is part of the active module list - auto const& activeModules = simConfig.getActiveModules(); - auto isActivated = [&activeModules](std::string const& s) -> bool { - return std::find(activeModules.begin(), activeModules.end(), s) != activeModules.end(); - }; - - for (auto* extdet : o2::ext::ExternalDetector::createFromJSON(extGeomFile)) { - const std::string name = extdet->GetName(); - if (!isActivated(name)) { - delete extdet; // not requested in the active module list - continue; - } - const int detID = extdet->GetDetId(); - if (detID < DetID::First || detID > DetID::Last) { - LOG(error) << "O2HitMerger: external detector " << name << " has invalid DetID " << detID << "; skipping"; - delete extdet; - continue; - } - if (mDetectorInstances[detID]) { - LOG(error) << "O2HitMerger: DetID " << DetID::getName(detID) << " requested by external detector " << name - << " is already occupied; its hits will not be persisted. Assign a free DetID."; - delete extdet; - continue; - } - mDetectorInstances[detID].reset(extdet); - mExternalDetIDs.emplace_back(detID); - LOG(info) << "O2HitMerger: registered external detector " << name << " on DetID " << DetID::getName(detID) - << " (branch " << name << "Hit)"; - } -} - } // namespace devices } // namespace o2 diff --git a/run/O2PrimaryServerDevice.cxx b/run/O2PrimaryServerDevice.cxx new file mode 100644 index 0000000000000..4ab6ad3c70017 --- /dev/null +++ b/run/O2PrimaryServerDevice.cxx @@ -0,0 +1,698 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @author Sandro Wenzel + +#include "O2PrimaryServerDevice.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "PrimaryServerState.h" +#include "SimPublishChannelHelper.h" +#include +#include +#include +#include + +namespace o2 +{ +namespace devices +{ + +O2PrimaryServerDevice::O2PrimaryServerDevice() +{ + mUseFixedChunkSeed = getenv("ALICEO2_O2SIM_SUBEVENTSEED") && atoi(getenv("ALICEO2_O2SIM_SUBEVENTSEED")); + if (mUseFixedChunkSeed) { + mFixedChunkSeed = atol(getenv("ALICEO2_O2SIM_SUBEVENTSEED")); + } +} + +O2PrimaryServerDevice::~O2PrimaryServerDevice() +{ + try { + if (mGeneratorThread.joinable()) { + mGeneratorThread.join(); + } + if (mControlThread.joinable()) { + mControlThread.join(); + } + } catch (...) { + } +} + +void O2PrimaryServerDevice::initGenerator() +{ + TStopwatch timer; + timer.Start(); + const auto& conf = mSimConfig; + auto& ccdbmgr = o2::ccdb::BasicCCDBManager::instance(); + ccdbmgr.setURL(conf.getConfigData().mCCDBUrl); + ccdbmgr.setTimestamp(conf.getTimestamp()); + + // set the global information about the number of events to be generated + unsigned int nTotalEvents = conf.getNEvents(); + o2::eventgen::Generator::setTotalNEvents(nTotalEvents); + + // init magnetic field as it might be needed by the generator + if (TGeoGlobalMagField::Instance()->GetField() == nullptr) { + TGeoGlobalMagField::Instance()->SetField(o2::base::SimFieldUtils::createMagField()); + TGeoGlobalMagField::Instance()->Lock(); + } + + // look if we find a cached instances of Pythia8 or external generators in order to avoid + // (long) initialization times. + // This is evidently a bit weak, as generators might need reconfiguration (to be treated later). + // For now, we'd like to allow for fast switches between say a pythia8 instance and reading from kinematics + // to continue an already started simulation. + // + // Not using cached instances for external kinematics since these might change input filenames etc. + // and are in any case quickly setup. + mPrimGen = nullptr; + if (conf.getGenerator().compare("extkin") != 0 && conf.getGenerator().compare("extkinO2") != 0) { + auto iter = mPrimGeneratorCache.find(conf.getGenerator()); + if (iter != mPrimGeneratorCache.end()) { + mPrimGen = iter->second.get(); + LOG(info) << "Found cached generator for " << conf.getGenerator(); + } + } + + if (mPrimGen == nullptr) { + mPrimGen = new o2::eventgen::PrimaryGenerator; + o2::eventgen::GeneratorFactory::setPrimaryGenerator(conf, mPrimGen); + + // setup vertexing + auto vtxMode = conf.getVertexMode(); + using o2::conf::VertexMode; + if (vtxMode == VertexMode::kNoVertex || vtxMode == VertexMode::kDiamondParam) { + mPrimGen->setVertexMode(vtxMode); + } else if (vtxMode == VertexMode::kCCDB) { + // we need to fetch the CCDB object + mPrimGen->setVertexMode(vtxMode, ccdbmgr.getForTimeStamp("GLO/Calib/MeanVertex", conf.getTimestamp())); + } else if (vtxMode == VertexMode::kCollCxt) { + // The vertex will be injected from the outside via setExternalVertex + } else { + LOG(fatal) << "Unsupported vertex mode"; + } + + auto embedinto_filename = conf.getEmbedIntoFileName(); + if (!embedinto_filename.empty()) { + // determine the sim prefix from the embedding filename + // the filename should be an MCHeader file ... so it should match SOME_PATH/prefix_MCHeader.root + std::regex re(R"((.*/)?([^/]+)_MCHeader\.root$)"); + std::smatch match; + + if (std::regex_search(embedinto_filename, match, re)) { + std::cout << "Extracted embedding prefix : " << match[2] << '\n'; + mEmbeddIntoPrefix = match[2]; + } else { + LOG(fatal) << "Embedding asked but no suitable embedding prefix extractable from " << embedinto_filename; + } + mPrimGen->embedInto(embedinto_filename); + } + + mPrimGen->Init(); + + std::unique_ptr ptr_wrapper; + ptr_wrapper.reset(mPrimGen); + mPrimGeneratorCache[conf.getGenerator()] = std::move(ptr_wrapper); + } + mPrimGen->SetEvent(&mEventHeader); + + // A good moment to couple to collision context + auto collContextFileName_PrefixPair = mSimConfig.getCollContextFilenameAndEventPrefix(); + auto collContextFileName = collContextFileName_PrefixPair.first; + if (collContextFileName.size() > 0) { + LOG(info) << "Simulation has collission context"; + mCollissionContext = o2::steer::DigitizationContext::loadFromFile(collContextFileName); + if (mCollissionContext) { + const auto& vertices = mCollissionContext->getInteractionVertices(); + LOG(info) << "We found " << vertices.size() << " vertices included "; + + // initialize the eventID to collID mapping + const auto source = mCollissionContext->findSimPrefix(collContextFileName_PrefixPair.second); + if (source == -1) { + LOG(fatal) << "Wrong simulation prefix"; + } + mEventID_to_CollID.clear(); + mEventID_to_CollID = mCollissionContext->getCollisionIndicesForSource(source); + } + } + + LOG(info) << "Generator initialization took " << timer.CpuTime() << "s"; + if (mMaxEvents > 0) { + generateEvent(); // generate a first event + } +} + +void O2PrimaryServerDevice::generateEvent() +{ + bool changeState = true; // false; + LOG(info) << "Event generation started "; + if (changeState) { + stateTransition(O2PrimaryServerState::WaitingEvent, "GENEVENT"); + } + TStopwatch timer; + timer.Start(); + try { + bool valid = false; + int retry_counter = 0; + const int MAX_RETRY = 100; + do { + mStack->Reset(); + const auto& conf = mSimConfig; + // see if we the vertex comes from the collision context + if (mCollissionContext && conf.getVertexMode() == o2::conf::VertexMode::kCollCxt) { + const auto& vertices = mCollissionContext->getInteractionVertices(); + if (vertices.size() > 0) { + auto collisionindex = mEventID_to_CollID.at(mEventCounter); + auto& vertex = vertices.at(collisionindex); + LOG(info) << "Setting vertex " << vertex << " for event " << mEventCounter << " for prefix " << mSimConfig.getOutPrefix() << " from CollContext"; + mPrimGen->setExternalVertexForNextEvent(vertex.X(), vertex.Y(), vertex.Z()); + + // set correct embedding index for PrimaryGenerator ... based on collision context for embedding + auto& collisionParts = mCollissionContext->getEventParts()[collisionindex]; + int background_index = -1; // -1 means no embedding taking place for this signal + + // find the part that corresponds to the event embeded into + for (auto& part : collisionParts) { + if (mCollissionContext->getSimPrefixes()[part.sourceID] == mEmbeddIntoPrefix) { + background_index = part.entryID; + LOG(info) << "Setting embedding index to " << background_index; + } + } + mPrimGen->setEmbedIndex(background_index); + } + } + mPrimGen->GenerateEvent(mStack); + if (mStack->getPrimaries().size() > 0) { + valid = true; + } else { + retry_counter++; + if (retry_counter > MAX_RETRY) { + LOG(warn) << "Not able to generate a non-empty event in " << MAX_RETRY << " trials"; + // empty event is sent out + valid = true; + } + } + } while (!valid); + } catch (std::exception const& e) { + LOG(error) << " Exception occurred during event gen " << e.what(); + } + timer.Stop(); + LOG(info) << "Event generation took " << timer.CpuTime() << "s" + << " and produced " << mStack->getPrimaries().size() << " primaries "; + if (changeState) { + stateTransition(O2PrimaryServerState::ReadyToServe, "GENEVENT"); + } +} + +void O2PrimaryServerDevice::launchInfoThread() +{ + static std::vector threads; + auto sendErrorReply = [](fair::mq::Channel& channel) { + LOG(error) << "UNKNOWN REQUEST"; + std::unique_ptr reply(channel.NewSimpleMessage((int)(404))); + channel.Send(reply); + }; + + LOG(info) << "LAUNCHING STATUS THREAD"; + auto lambda = [this, sendErrorReply]() { + bool canShutdown{false}; + // Exit only when both: serving stopped and allowed from outside. + while (!(mState == O2PrimaryServerState::Stopped && canShutdown)) { + auto& channel = GetChannels().at("o2sim-primserv-info").at(0); + if (!channel.IsValid()) { + LOG(error) << "channel primserv-info not valid"; + } + std::unique_ptr request(channel.NewSimpleMessage((int)(-1))); + int timeout = 100; // 100ms --> so as not to block and allow for proper termination of this thread + if (channel.Receive(request, timeout) > 0) { + int request_payload; // we expect an (int) ~ to type O2PrimaryServerInfoRequest + if (request->GetSize() != sizeof(request_payload)) { + LOG(error) << "Obtained request with unexpected payload size"; + sendErrorReply(channel); // ALWAYS reply + continue; + } + + memcpy(&request_payload, request->GetData(), sizeof(request_payload)); + + if (request_payload == (int)O2PrimaryServerInfoRequest::Status) { + LOG(info) << "Received status request"; + // request needs to be a simple enum of type O2PrimaryServerInfoRequest + std::unique_ptr reply(channel.NewSimpleMessage((int)mState.load())); + if (channel.Send(reply) > 0) { + LOG(info) << "Send status successful"; + } + } else if (request_payload == (int)O2PrimaryServerInfoRequest::Config) { + HandleConfigRequest(channel); + } else if (request_payload == (int)O2PrimaryServerInfoRequest::AllowShutdown) { + LOG(info) << "Got info that we may shutdown"; + std::unique_ptr ack(channel.NewSimpleMessage(200)); + channel.Send(ack); + canShutdown = true; + } else { + sendErrorReply(channel); + } + } + } + mInfoThreadStopped = true; + }; + threads.push_back(std::thread(lambda)); + threads.back().detach(); +} + +void O2PrimaryServerDevice::InitTask() +{ + // fatal without core dump + fair::Logger::OnFatal([] { throw fair::FatalException("Fatal error occured. Exiting without core dump..."); }); + + o2::simpubsub::publishMessage(GetChannels()["primary-notifications"].at(0), "SERVER : INITIALIZING"); + + stateTransition(O2PrimaryServerState::Initializing, "INITTASK"); + LOG(info) << "Init Server device "; + + // init sim config + auto& vm = GetConfig()->GetVarMap(); + auto& conf = o2::conf::SimConfig::Instance(); + if (vm.count("isRun5")) { + conf.setRun5(); + } + conf.resetFromParsedMap(vm); + + // update the parameters from an INI/JSON file, if given (overrides code-based version) + o2::conf::ConfigurableParam::updateFromFile(conf.getConfigFile()); + // update the parameters from stuff given at command line (overrides file-based version) + o2::conf::ConfigurableParam::updateFromString(conf.getKeyValueString()); + + // customize the level of log output + FairLogger::GetLogger()->SetLogScreenLevel(conf.getLogSeverity().c_str()); + FairLogger::GetLogger()->SetLogVerbosityLevel(conf.getLogVerbosity().c_str()); + + // from now on mSimConfig should be used within this process + mSimConfig = conf; + + mStack = new o2::data::Stack(); + mStack->setExternalMode(true); + + // MC ENGINE + LOG(info) << "ENGINE SET TO " << vm["mcEngine"].as(); + // CHUNK SIZE + mChunkGranularity = vm["chunkSize"].as(); + LOG(info) << "CHUNK SIZE SET TO " << mChunkGranularity; + + // initial initial seed --> we should store this somewhere + mInitialSeed = vm["seed"].as(); + mInitialSeed = o2::utils::RngHelper::setGRandomSeed(mInitialSeed); + mSeedGenerator.SetSeed(mInitialSeed); + LOG(info) << "RNG INITIAL SEED " << mInitialSeed; + + mMaxEvents = conf.getNEvents(); + + // need to make ROOT thread-safe since we use ROOT services in all places + ROOT::EnableThreadSafety(); + + launchInfoThread(); + + // launch initialization of particle generator asynchronously + // so that we reach the RUNNING state of the server quickly + // and do not block here + mGeneratorThread = std::thread(&O2PrimaryServerDevice::initGenerator, this); + if (mGeneratorThread.joinable()) { + try { + mGeneratorThread.join(); + } catch (std::exception const& e) { + LOG(warn) << "Exception during thread join ..ignoring"; + } + } + + // init pipe + auto pipeenv = getenv("ALICE_O2SIMSERVERTODRIVER_PIPE"); + if (pipeenv) { + mPipeToDriver = atoi(pipeenv); + LOG(info) << "ASSIGNED PIPE HANDLE " << mPipeToDriver; + } else { + LOG(info) << "DID NOT FIND ENVIRONMENT VARIABLE TO INIT PIPE"; + } + + mAsService = vm["asservice"].as(); + if (mAsService) { + mControlChannel = fair::mq::Channel{"o2sim-control", "sub", fTransportFactory}; + auto controlsocketname = getenv("ALICE_O2SIMCONTROL"); + if (!controlsocketname) { + LOG(fatal) << "Internal error: Socketname for control input missing"; + } + mControlChannel.Connect(std::string(controlsocketname)); + mControlChannel.Validate(); + } + + if (mMaxEvents <= 0) { + if (mAsService) { + stateTransition(O2PrimaryServerState::Idle, "INITTASK"); + } + } else { + stateTransition(O2PrimaryServerState::ReadyToServe, "INITTASK"); + } + + // feedback to driver that we are done initializing + if (mPipeToDriver != -1) { + int message = -111; // special code meaning end of initialization + if (write(mPipeToDriver, &message, sizeof(int))) { + } + } +} + +bool O2PrimaryServerDevice::ReInit(o2::conf::SimReconfigData const& reconfig) +{ + LOG(info) << "ReInit Server device "; + + if (reconfig.stop) { + return false; + } + + // mSimConfig.getConfigData().mKeyValueTokens=reconfig.keyValueTokens; + // Think about this: + // update the parameters from an INI/JSON file, if given (overrides code-based version) + o2::conf::ConfigurableParam::updateFromFile(reconfig.configFile); + // update the parameters from stuff given at command line (overrides file-based version) + o2::conf::ConfigurableParam::updateFromString(reconfig.keyValueTokens); + + // initial initial seed --> we should store this somewhere + mInitialSeed = reconfig.startSeed; + mInitialSeed = o2::utils::RngHelper::setGRandomSeed(mInitialSeed); + mSeedGenerator.SetSeed(mInitialSeed); + LOG(info) << "RNG INITIAL SEED " << mInitialSeed; + + mMaxEvents = reconfig.nEvents; + + // updating the simconfig member with new information especially concerning the generators + // TODO: put this into utility function? + mSimConfig.getConfigData().mGenerator = reconfig.generator; + mSimConfig.getConfigData().mTrigger = reconfig.trigger; + mSimConfig.getConfigData().mExtKinFileName = reconfig.extKinfileName; + + mEventCounter = 0; + mPartCounter = 0; + mNeedNewEvent = true; + // reinit generator and start generation of a new event + if (mGeneratorThread.joinable()) { + try { + mGeneratorThread.join(); + } catch (std::exception const& e) { + LOG(warn) << "Exception during thread join ..ignoring"; + } + } + // mGeneratorThread = std::thread(&O2PrimaryServerDevice::initGenerator, this); + initGenerator(); + + return true; +} + +bool O2PrimaryServerDevice::HandleConfigRequest(fair::mq::Channel& channel) +{ + LOG(info) << "Received config request"; + // just sending the simulation configuration to anyone that wants it + const auto& confdata = mSimConfig.getConfigData(); + + TMessage* tmsg = new TMessage(kMESS_OBJECT); + tmsg->WriteObjectAny((void*)&confdata, TClass::GetClass(typeid(confdata))); + + auto free_tmessage = [](void* data, void* hint) { delete static_cast(hint); }; + + std::unique_ptr message( + fTransportFactory->CreateMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); + + // send answer + if (channel.Send(message) > 0) { + LOG(info) << "config reply send "; + return true; + } else { + LOG(error) << "Failure sending config reply "; + } + return true; +} + +bool O2PrimaryServerDevice::ConditionalRun() +{ + // we might come here in IDLE mode + if (mState.load() == O2PrimaryServerState::Idle) { + if (mWaitingControlInput.load() == 0) { + if (mControlThread.joinable()) { + mControlThread.join(); + } + mControlThread = std::thread(&O2PrimaryServerDevice::waitForControlInput, this); + } + } + + auto& channel = GetChannels().at("primary-get").at(0); + PrimaryChunkRequest requestpayload; + std::unique_ptr request(channel.NewSimpleMessage(requestpayload)); + auto bytes = channel.Receive(request); + if (bytes < 0) { + LOG(error) << "Some error/interrupt occurred on socket during receive"; + if (NewStatePending()) { // new state is typically pending if (term) signal was received + WaitForNextState(); + // ask ourselves for termination of this loop + stateTransition(O2PrimaryServerState::Stopped, "CONDRUN"); + } + return false; + } + + TStopwatch timer; + timer.Start(); + auto& r = *((PrimaryChunkRequest*)(request->GetData())); + LOG(debug) << "PARTICLE REQUEST IN STATE " << PrimStateToString[(int)mState.load()] << " from " << r.workerid << ":" << r.requestid; + + auto prestate = mState.load(); + auto more = HandleRequest(request, 0, channel); + if (!more) { + if (mAsService) { + if (prestate == O2PrimaryServerState::ReadyToServe || prestate == O2PrimaryServerState::WaitingEvent) { + stateTransition(O2PrimaryServerState::Idle, "CONDRUN"); + } + } else { + stateTransition(O2PrimaryServerState::Stopped, "CONDRUN"); + } + } + timer.Stop(); + auto time = timer.CpuTime(); + LOG(debug) << "COND-RUN TOOK " << time << " s"; + return mState != O2PrimaryServerState::Stopped; +} + +void O2PrimaryServerDevice::PostRun() +{ + // We shouldn't shut down immediately when all events have been served + // Instead we also need to wait until the info thread running some communication server + // with other processes is finished. + while (!mInfoThreadStopped) { + LOG(info) << "Waiting info thread"; + using namespace std::chrono_literals; + std::this_thread::sleep_for(1000ms); + } +} + +bool O2PrimaryServerDevice::HandleRequest(fair::mq::MessagePtr& request, int /*index*/, fair::mq::Channel& channel) +{ + // LOG(debug) << "GOT A REQUEST WITH SIZE " << request->GetSize(); + // std::string requeststring(static_cast(request->GetData()), request->GetSize()); + // LOG(info) << "NORMAL REQUEST STRING " << requeststring; + bool workavailable = true; + if (mEventCounter >= mMaxEvents && mNeedNewEvent) { + workavailable = false; + } + if (!(mState.load() == O2PrimaryServerState::ReadyToServe || mState.load() == O2PrimaryServerState::WaitingEvent)) { + // send a zero answer + workavailable = false; + } + + PrimaryChunkAnswer header{mState, workavailable}; + fair::mq::Parts reply; + std::unique_ptr headermsg(channel.NewSimpleMessage(header)); + reply.AddPart(std::move(headermsg)); + + LOG(debug) << "Received request for work " << mEventCounter << " " << mMaxEvents << " " << mNeedNewEvent << " available " << workavailable; + if (workavailable) { + + if (mNeedNewEvent) { + // we need a newly generated event now + if (mGeneratorThread.joinable()) { + try { + mGeneratorThread.join(); + } catch (std::exception const& e) { + LOG(warn) << "Exception during thread join ..ignoring"; + } + } + // also if we are still in event waiting stage (doing some busy sleep) + while (mState.load() == O2PrimaryServerState::WaitingEvent) { + LOG(info) << "Waiting for event generation do become fully available"; + usleep(100); + } + mNeedNewEvent = false; + mPartCounter = 0; + mEventCounter++; + } + + auto& prims = mStack->getPrimaries(); + auto numberofparts = (int)std::ceil(prims.size() / (1. * mChunkGranularity)); + // number of parts should be at least 1 (even if empty) + numberofparts = std::max(1, numberofparts); + + LOG(debug) << "Have " << prims.size() << " " << numberofparts; + + o2::data::PrimaryChunk m; + o2::data::SubEventInfo i; + i.eventID = workavailable ? mEventCounter : -1; + i.maxEvents = mMaxEvents; + i.part = mPartCounter + 1; + i.nparts = numberofparts; + // assign a deterministic (yet collision free seed) to process this particle chunk in Geant + // limit range to uint32_t since internal limit of TRandom (despite API suggesting otherwise) + const uint64_t drawnSeed = (uint64_t)(static_cast(std::numeric_limits::max()) * mSeedGenerator.Rndm()); + i.seed = mUseFixedChunkSeed ? mFixedChunkSeed : drawnSeed; + i.index = m.mParticles.size(); + i.mMCEventHeader = mEventHeader; + m.mSubEventInfo = i; + + int endindex = prims.size() - mPartCounter * mChunkGranularity; + int startindex = prims.size() - (mPartCounter + 1) * mChunkGranularity; + LOG(debug) << "indices " << startindex << " " << endindex; + + if (startindex < 0) { + startindex = 0; + } + if (endindex < 0) { + endindex = 0; + } + + for (int index = startindex; index < endindex; ++index) { + m.mParticles.emplace_back(prims[index]); + } + + LOG(info) << "Sending " << m.mParticles.size() << " particles"; + LOG(info) << "treating ev " << mEventCounter << " part " << i.part << " out of " << i.nparts; + + // feedback to driver if new event started + if (mPipeToDriver != -1 && i.part == 1 && workavailable) { + if (write(mPipeToDriver, &mEventCounter, sizeof(mEventCounter))) { + } + } + + mPartCounter++; + if (mPartCounter == numberofparts) { + mNeedNewEvent = true; + // start generation of a new event + if (mEventCounter < mMaxEvents) { + mGeneratorThread = std::thread(&O2PrimaryServerDevice::generateEvent, this); + } + } + + TMessage* tmsg = new TMessage(kMESS_OBJECT); + tmsg->WriteObjectAny((void*)&m, TClass::GetClass("o2::data::PrimaryChunk")); + + auto free_tmessage = [](void* data, void* hint) { delete static_cast(hint); }; + + std::unique_ptr message(channel.NewMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); + + reply.AddPart(std::move(message)); + } + + // send answer + TStopwatch timer; + timer.Start(); + auto code = Send(reply, "primary-get", 0, 5000); // we introduce timeout in order not to block other requests + timer.Stop(); + auto time = timer.CpuTime(); + if (code > 0) { + LOG(debug) << "Reply send in " << time << "s"; + return workavailable; + } else { + LOG(warn) << "Sending process had problems. Return code : " << code << " time " << time << "s"; + } + return false; // -> error should not get here +} + +void O2PrimaryServerDevice::stateTransition(O2PrimaryServerState to, const char* message) +{ + LOG(info) << message << " CHANGING STATE TO " << PrimStateToString[(int)to]; + mState = to; +} + +void O2PrimaryServerDevice::waitForControlInput() +{ + mWaitingControlInput.store(1); + if (mState.load() != O2PrimaryServerState::Idle) { + mWaitingControlInput.store(0); + return; + } + + o2::simpubsub::publishMessage(GetChannels()["primary-notifications"].at(0), o2::simpubsub::simStatusString("PRIMSERVER", "STATUS", "AWAITING INPUT")); + // this means we are idling + + std::unique_ptr reply(mControlChannel.NewMessage()); + + bool ok = false; + + LOG(info) << "WAITING FOR CONTROL INPUT"; + if (mControlChannel.Receive(reply) > 0) { + stateTransition(O2PrimaryServerState::Initializing, "CONTROL"); + auto data = reply->GetData(); + auto size = reply->GetSize(); + + std::string command(reinterpret_cast(data), size); + LOG(info) << "message: " << command; + + o2::conf::SimReconfigData reconfig; + o2::conf::parseSimReconfigFromString(command, reconfig); + LOG(info) << "Processing " << reconfig.nEvents << " new events"; + try { + LOG(info) << "REINIT START"; + ok = ReInit(reconfig); + LOG(info) << "REINIT DONE"; + } catch (std::exception e) { + LOG(info) << "Exception during reinit"; + } + } else { + LOG(info) << "NOTHING RECEIVED"; + } + if (ok) { + // stateTransition(O2PrimaryServerState::ReadyToServe, "CONTROL"); --> SHOULD BE DONE FROM EVENT GENERATOR (which get's however called only when mEvents>0) + } else { + stateTransition(O2PrimaryServerState::Stopped, "CONTROL"); + } + mWaitingControlInput.store(0); +} + +} // namespace devices +} // namespace o2 diff --git a/run/O2PrimaryServerDevice.h b/run/O2PrimaryServerDevice.h index b8703ffcddb28..ad568469a110c 100644 --- a/run/O2PrimaryServerDevice.h +++ b/run/O2PrimaryServerDevice.h @@ -15,38 +15,19 @@ #define O2_DEVICES_PRIMSERVDEVICE_H_ #include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include #include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include +#include +#include +#include +#include +#include #include "PrimaryServerState.h" -#include "SimPublishChannelHelper.h" -#include -#include -#include -#include namespace o2 { @@ -57,653 +38,37 @@ class O2PrimaryServerDevice final : public fair::mq::Device { public: /// constructor - O2PrimaryServerDevice() - { - mUseFixedChunkSeed = getenv("ALICEO2_O2SIM_SUBEVENTSEED") && atoi(getenv("ALICEO2_O2SIM_SUBEVENTSEED")); - if (mUseFixedChunkSeed) { - mFixedChunkSeed = atol(getenv("ALICEO2_O2SIM_SUBEVENTSEED")); - } - } + O2PrimaryServerDevice(); /// Default destructor - ~O2PrimaryServerDevice() final - { - try { - if (mGeneratorThread.joinable()) { - mGeneratorThread.join(); - } - if (mControlThread.joinable()) { - mControlThread.join(); - } - } catch (...) { - } - } + ~O2PrimaryServerDevice() final; protected: - void initGenerator() - { - TStopwatch timer; - timer.Start(); - const auto& conf = mSimConfig; - auto& ccdbmgr = o2::ccdb::BasicCCDBManager::instance(); - ccdbmgr.setURL(conf.getConfigData().mCCDBUrl); - ccdbmgr.setTimestamp(conf.getTimestamp()); - - // set the global information about the number of events to be generated - unsigned int nTotalEvents = conf.getNEvents(); - o2::eventgen::Generator::setTotalNEvents(nTotalEvents); - - // init magnetic field as it might be needed by the generator - if (TGeoGlobalMagField::Instance()->GetField() == nullptr) { - TGeoGlobalMagField::Instance()->SetField(o2::base::SimFieldUtils::createMagField()); - TGeoGlobalMagField::Instance()->Lock(); - } - - // look if we find a cached instances of Pythia8 or external generators in order to avoid - // (long) initialization times. - // This is evidently a bit weak, as generators might need reconfiguration (to be treated later). - // For now, we'd like to allow for fast switches between say a pythia8 instance and reading from kinematics - // to continue an already started simulation. - // - // Not using cached instances for external kinematics since these might change input filenames etc. - // and are in any case quickly setup. - mPrimGen = nullptr; - if (conf.getGenerator().compare("extkin") != 0 || conf.getGenerator().compare("extkinO2") != 0) { - auto iter = mPrimGeneratorCache.find(conf.getGenerator()); - if (iter != mPrimGeneratorCache.end()) { - mPrimGen = iter->second.get(); - LOG(info) << "Found cached generator for " << conf.getGenerator(); - } - } - - if (mPrimGen == nullptr) { - mPrimGen = new o2::eventgen::PrimaryGenerator; - o2::eventgen::GeneratorFactory::setPrimaryGenerator(conf, mPrimGen); - - // setup vertexing - auto vtxMode = conf.getVertexMode(); - using o2::conf::VertexMode; - if (vtxMode == VertexMode::kNoVertex || vtxMode == VertexMode::kDiamondParam) { - mPrimGen->setVertexMode(vtxMode); - } else if (vtxMode == VertexMode::kCCDB) { - // we need to fetch the CCDB object - mPrimGen->setVertexMode(vtxMode, ccdbmgr.getForTimeStamp("GLO/Calib/MeanVertex", conf.getTimestamp())); - } else if (vtxMode == VertexMode::kCollCxt) { - // The vertex will be injected from the outside via setExternalVertex - } else { - LOG(fatal) << "Unsupported vertex mode"; - } - - auto embedinto_filename = conf.getEmbedIntoFileName(); - if (!embedinto_filename.empty()) { - // determine the sim prefix from the embedding filename - // the filename should be an MCHeader file ... so it should match SOME_PATH/prefix_MCHeader.root - std::regex re(R"((.*/)?([^/]+)_MCHeader\.root$)"); - std::smatch match; - - if (std::regex_search(embedinto_filename, match, re)) { - std::cout << "Extracted embedding prefix : " << match[2] << '\n'; - mEmbeddIntoPrefix = match[2]; - } else { - LOG(fatal) << "Embedding asked but no suitable embedding prefix extractable from " << embedinto_filename; - } - mPrimGen->embedInto(embedinto_filename); - } - - mPrimGen->Init(); - - std::unique_ptr ptr_wrapper; - ptr_wrapper.reset(mPrimGen); - mPrimGeneratorCache[conf.getGenerator()] = std::move(ptr_wrapper); - } - mPrimGen->SetEvent(&mEventHeader); - - // A good moment to couple to collision context - auto collContextFileName_PrefixPair = mSimConfig.getCollContextFilenameAndEventPrefix(); - auto collContextFileName = collContextFileName_PrefixPair.first; - if (collContextFileName.size() > 0) { - LOG(info) << "Simulation has collission context"; - mCollissionContext = o2::steer::DigitizationContext::loadFromFile(collContextFileName); - if (mCollissionContext) { - const auto& vertices = mCollissionContext->getInteractionVertices(); - LOG(info) << "We found " << vertices.size() << " vertices included "; - - // initialize the eventID to collID mapping - const auto source = mCollissionContext->findSimPrefix(collContextFileName_PrefixPair.second); - if (source == -1) { - LOG(fatal) << "Wrong simulation prefix"; - } - mEventID_to_CollID.clear(); - mEventID_to_CollID = mCollissionContext->getCollisionIndicesForSource(source); - } - } - - LOG(info) << "Generator initialization took " << timer.CpuTime() << "s"; - if (mMaxEvents > 0) { - generateEvent(); // generate a first event - } - } + void initGenerator(); // function generating one event - void generateEvent(/*bool changeState = false*/) - { - bool changeState = true; // false; - LOG(info) << "Event generation started "; - if (changeState) { - stateTransition(O2PrimaryServerState::WaitingEvent, "GENEVENT"); - } - TStopwatch timer; - timer.Start(); - try { - bool valid = false; - int retry_counter = 0; - const int MAX_RETRY = 100; - do { - mStack->Reset(); - const auto& conf = mSimConfig; - // see if we the vertex comes from the collision context - if (mCollissionContext && conf.getVertexMode() == o2::conf::VertexMode::kCollCxt) { - const auto& vertices = mCollissionContext->getInteractionVertices(); - if (vertices.size() > 0) { - auto collisionindex = mEventID_to_CollID.at(mEventCounter); - auto& vertex = vertices.at(collisionindex); - LOG(info) << "Setting vertex " << vertex << " for event " << mEventCounter << " for prefix " << mSimConfig.getOutPrefix() << " from CollContext"; - mPrimGen->setExternalVertexForNextEvent(vertex.X(), vertex.Y(), vertex.Z()); - - // set correct embedding index for PrimaryGenerator ... based on collision context for embedding - auto& collisionParts = mCollissionContext->getEventParts()[collisionindex]; - int background_index = -1; // -1 means no embedding taking place for this signal - - // find the part that corresponds to the event embeded into - for (auto& part : collisionParts) { - if (mCollissionContext->getSimPrefixes()[part.sourceID] == mEmbeddIntoPrefix) { - background_index = part.entryID; - LOG(info) << "Setting embedding index to " << background_index; - } - } - mPrimGen->setEmbedIndex(background_index); - } - } - mPrimGen->GenerateEvent(mStack); - if (mStack->getPrimaries().size() > 0) { - valid = true; - } else { - retry_counter++; - if (retry_counter > MAX_RETRY) { - LOG(warn) << "Not able to generate a non-empty event in " << MAX_RETRY << " trials"; - // empty event is sent out - valid = true; - } - } - } while (!valid); - } catch (std::exception const& e) { - LOG(error) << " Exception occurred during event gen " << e.what(); - } - timer.Stop(); - LOG(info) << "Event generation took " << timer.CpuTime() << "s" - << " and produced " << mStack->getPrimaries().size() << " primaries "; - if (changeState) { - stateTransition(O2PrimaryServerState::ReadyToServe, "GENEVENT"); - } - } + void generateEvent(/*bool changeState = false*/); // launches a thread that listens for status/config/shutdown requests from outside asynchronously - void launchInfoThread() - { - static std::vector threads; - auto sendErrorReply = [](fair::mq::Channel& channel) { - LOG(error) << "UNKNOWN REQUEST"; - std::unique_ptr reply(channel.NewSimpleMessage((int)(404))); - channel.Send(reply); - }; - - LOG(info) << "LAUNCHING STATUS THREAD"; - auto lambda = [this, sendErrorReply]() { - bool canShutdown{false}; - // Exit only when both: serving stopped and allowed from outside. - while (!(mState == O2PrimaryServerState::Stopped && canShutdown)) { - auto& channel = GetChannels().at("o2sim-primserv-info").at(0); - if (!channel.IsValid()) { - LOG(error) << "channel primserv-info not valid"; - } - std::unique_ptr request(channel.NewSimpleMessage((int)(-1))); - int timeout = 100; // 100ms --> so as not to block and allow for proper termination of this thread - if (channel.Receive(request, timeout) > 0) { - int request_payload; // we expect an (int) ~ to type O2PrimaryServerInfoRequest - if (request->GetSize() != sizeof(request_payload)) { - LOG(error) << "Obtained request with unexpected payload size"; - sendErrorReply(channel); // ALWAYS reply - } - - memcpy(&request_payload, request->GetData(), sizeof(request_payload)); - - if (request_payload == (int)O2PrimaryServerInfoRequest::Status) { - LOG(info) << "Received status request"; - // request needs to be a simple enum of type O2PrimaryServerInfoRequest - std::unique_ptr reply(channel.NewSimpleMessage((int)mState.load())); - if (channel.Send(reply) > 0) { - LOG(info) << "Send status successful"; - } - } else if (request_payload == (int)O2PrimaryServerInfoRequest::Config) { - HandleConfigRequest(channel); - } else if (request_payload == (int)O2PrimaryServerInfoRequest::AllowShutdown) { - LOG(info) << "Got info that we may shutdown"; - std::unique_ptr ack(channel.NewSimpleMessage(200)); - channel.Send(ack); - canShutdown = true; - } else { - sendErrorReply(channel); - } - } - } - mInfoThreadStopped = true; - }; - threads.push_back(std::thread(lambda)); - threads.back().detach(); - } - - void InitTask() final - { - // fatal without core dump - fair::Logger::OnFatal([] { throw fair::FatalException("Fatal error occured. Exiting without core dump..."); }); - - o2::simpubsub::publishMessage(GetChannels()["primary-notifications"].at(0), "SERVER : INITIALIZING"); - - stateTransition(O2PrimaryServerState::Initializing, "INITTASK"); - LOG(info) << "Init Server device "; - - // init sim config - auto& vm = GetConfig()->GetVarMap(); - auto& conf = o2::conf::SimConfig::Instance(); - if (vm.count("isRun5")) { - conf.setRun5(); - } - conf.resetFromParsedMap(vm); - - // update the parameters from an INI/JSON file, if given (overrides code-based version) - o2::conf::ConfigurableParam::updateFromFile(conf.getConfigFile()); - // update the parameters from stuff given at command line (overrides file-based version) - o2::conf::ConfigurableParam::updateFromString(conf.getKeyValueString()); - - // customize the level of log output - FairLogger::GetLogger()->SetLogScreenLevel(conf.getLogSeverity().c_str()); - FairLogger::GetLogger()->SetLogVerbosityLevel(conf.getLogVerbosity().c_str()); - - // from now on mSimConfig should be used within this process - mSimConfig = conf; - - mStack = new o2::data::Stack(); - mStack->setExternalMode(true); - - // MC ENGINE - LOG(info) << "ENGINE SET TO " << vm["mcEngine"].as(); - // CHUNK SIZE - mChunkGranularity = vm["chunkSize"].as(); - LOG(info) << "CHUNK SIZE SET TO " << mChunkGranularity; - - // initial initial seed --> we should store this somewhere - mInitialSeed = vm["seed"].as(); - mInitialSeed = o2::utils::RngHelper::setGRandomSeed(mInitialSeed); - mSeedGenerator.SetSeed(mInitialSeed); - LOG(info) << "RNG INITIAL SEED " << mInitialSeed; - - mMaxEvents = conf.getNEvents(); - - // need to make ROOT thread-safe since we use ROOT services in all places - ROOT::EnableThreadSafety(); - - launchInfoThread(); - - // launch initialization of particle generator asynchronously - // so that we reach the RUNNING state of the server quickly - // and do not block here - mGeneratorThread = std::thread(&O2PrimaryServerDevice::initGenerator, this); - if (mGeneratorThread.joinable()) { - try { - mGeneratorThread.join(); - } catch (std::exception const& e) { - LOG(warn) << "Exception during thread join ..ignoring"; - } - } - - // init pipe - auto pipeenv = getenv("ALICE_O2SIMSERVERTODRIVER_PIPE"); - if (pipeenv) { - mPipeToDriver = atoi(pipeenv); - LOG(info) << "ASSIGNED PIPE HANDLE " << mPipeToDriver; - } else { - LOG(info) << "DID NOT FIND ENVIRONMENT VARIABLE TO INIT PIPE"; - } + void launchInfoThread(); - mAsService = vm["asservice"].as(); - if (mAsService) { - mControlChannel = fair::mq::Channel{"o2sim-control", "sub", fTransportFactory}; - auto controlsocketname = getenv("ALICE_O2SIMCONTROL"); - if (!controlsocketname) { - LOG(fatal) << "Internal error: Socketname for control input missing"; - } - mControlChannel.Connect(std::string(controlsocketname)); - mControlChannel.Validate(); - } - - if (mMaxEvents <= 0) { - if (mAsService) { - stateTransition(O2PrimaryServerState::Idle, "INITTASK"); - } - } else { - stateTransition(O2PrimaryServerState::ReadyToServe, "INITTASK"); - } - - // feedback to driver that we are done initializing - if (mPipeToDriver != -1) { - int message = -111; // special code meaning end of initialization - if (write(mPipeToDriver, &message, sizeof(int))) { - } - } - } + void InitTask() final; // function for intermediate/on-the-fly reinitializations - bool ReInit(o2::conf::SimReconfigData const& reconfig) - { - LOG(info) << "ReInit Server device "; - - if (reconfig.stop) { - return false; - } - - // mSimConfig.getConfigData().mKeyValueTokens=reconfig.keyValueTokens; - // Think about this: - // update the parameters from an INI/JSON file, if given (overrides code-based version) - o2::conf::ConfigurableParam::updateFromFile(reconfig.configFile); - // update the parameters from stuff given at command line (overrides file-based version) - o2::conf::ConfigurableParam::updateFromString(reconfig.keyValueTokens); - - // initial initial seed --> we should store this somewhere - mInitialSeed = reconfig.startSeed; - mInitialSeed = o2::utils::RngHelper::setGRandomSeed(mInitialSeed); - mSeedGenerator.SetSeed(mInitialSeed); - LOG(info) << "RNG INITIAL SEED " << mInitialSeed; - - mMaxEvents = reconfig.nEvents; - - // updating the simconfig member with new information especially concerning the generators - // TODO: put this into utility function? - mSimConfig.getConfigData().mGenerator = reconfig.generator; - mSimConfig.getConfigData().mTrigger = reconfig.trigger; - mSimConfig.getConfigData().mExtKinFileName = reconfig.extKinfileName; - - mEventCounter = 0; - mPartCounter = 0; - mNeedNewEvent = true; - // reinit generator and start generation of a new event - if (mGeneratorThread.joinable()) { - try { - mGeneratorThread.join(); - } catch (std::exception const& e) { - LOG(warn) << "Exception during thread join ..ignoring"; - } - } - // mGeneratorThread = std::thread(&O2PrimaryServerDevice::initGenerator, this); - initGenerator(); - - return true; - } + bool ReInit(o2::conf::SimReconfigData const& reconfig); // method reacting to requests to get the simulation configuration - bool HandleConfigRequest(fair::mq::Channel& channel) - { - LOG(info) << "Received config request"; - // just sending the simulation configuration to anyone that wants it - const auto& confdata = mSimConfig.getConfigData(); - - TMessage* tmsg = new TMessage(kMESS_OBJECT); - tmsg->WriteObjectAny((void*)&confdata, TClass::GetClass(typeid(confdata))); - - auto free_tmessage = [](void* data, void* hint) { delete static_cast(hint); }; - - std::unique_ptr message( - fTransportFactory->CreateMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); - - // send answer - if (channel.Send(message) > 0) { - LOG(info) << "config reply send "; - return true; - } else { - LOG(error) << "Failure sending config reply "; - } - return true; - } - - bool ConditionalRun() override - { - // we might come here in IDLE mode - if (mState.load() == O2PrimaryServerState::Idle) { - if (mWaitingControlInput.load() == 0) { - if (mControlThread.joinable()) { - mControlThread.join(); - } - mControlThread = std::thread(&O2PrimaryServerDevice::waitForControlInput, this); - } - } - - auto& channel = GetChannels().at("primary-get").at(0); - PrimaryChunkRequest requestpayload; - std::unique_ptr request(channel.NewSimpleMessage(requestpayload)); - auto bytes = channel.Receive(request); - if (bytes < 0) { - LOG(error) << "Some error/interrupt occurred on socket during receive"; - if (NewStatePending()) { // new state is typically pending if (term) signal was received - WaitForNextState(); - // ask ourselves for termination of this loop - stateTransition(O2PrimaryServerState::Stopped, "CONDRUN"); - } - return false; - } - - TStopwatch timer; - timer.Start(); - auto& r = *((PrimaryChunkRequest*)(request->GetData())); - LOG(debug) << "PARTICLE REQUEST IN STATE " << PrimStateToString[(int)mState.load()] << " from " << r.workerid << ":" << r.requestid; - - auto prestate = mState.load(); - auto more = HandleRequest(request, 0, channel); - if (!more) { - if (mAsService) { - if (prestate == O2PrimaryServerState::ReadyToServe || prestate == O2PrimaryServerState::WaitingEvent) { - stateTransition(O2PrimaryServerState::Idle, "CONDRUN"); - } - } else { - stateTransition(O2PrimaryServerState::Stopped, "CONDRUN"); - } - } - timer.Stop(); - auto time = timer.CpuTime(); - LOG(debug) << "COND-RUN TOOK " << time << " s"; - return mState != O2PrimaryServerState::Stopped; - } - - void PostRun() override - { - // We shouldn't shut down immediately when all events have been served - // Instead we also need to wait until the info thread running some communication server - // with other processes is finished. - while (!mInfoThreadStopped) { - LOG(info) << "Waiting info thread"; - using namespace std::chrono_literals; - std::this_thread::sleep_for(1000ms); - } - } - - bool HandleRequest(fair::mq::MessagePtr& request, int /*index*/, fair::mq::Channel& channel) - { - // LOG(debug) << "GOT A REQUEST WITH SIZE " << request->GetSize(); - // std::string requeststring(static_cast(request->GetData()), request->GetSize()); - // LOG(info) << "NORMAL REQUEST STRING " << requeststring; - bool workavailable = true; - if (mEventCounter >= mMaxEvents && mNeedNewEvent) { - workavailable = false; - } - if (!(mState.load() == O2PrimaryServerState::ReadyToServe || mState.load() == O2PrimaryServerState::WaitingEvent)) { - // send a zero answer - workavailable = false; - } - - PrimaryChunkAnswer header{mState, workavailable}; - fair::mq::Parts reply; - std::unique_ptr headermsg(channel.NewSimpleMessage(header)); - reply.AddPart(std::move(headermsg)); - - LOG(debug) << "Received request for work " << mEventCounter << " " << mMaxEvents << " " << mNeedNewEvent << " available " << workavailable; - if (workavailable) { - - if (mNeedNewEvent) { - // we need a newly generated event now - if (mGeneratorThread.joinable()) { - try { - mGeneratorThread.join(); - } catch (std::exception const& e) { - LOG(warn) << "Exception during thread join ..ignoring"; - } - } - // also if we are still in event waiting stage (doing some busy sleep) - while (mState.load() == O2PrimaryServerState::WaitingEvent) { - LOG(info) << "Waiting for event generation do become fully available"; - usleep(100); - } - mNeedNewEvent = false; - mPartCounter = 0; - mEventCounter++; - } - - auto& prims = mStack->getPrimaries(); - auto numberofparts = (int)std::ceil(prims.size() / (1. * mChunkGranularity)); - // number of parts should be at least 1 (even if empty) - numberofparts = std::max(1, numberofparts); - - LOG(debug) << "Have " << prims.size() << " " << numberofparts; - - o2::data::PrimaryChunk m; - o2::data::SubEventInfo i; - i.eventID = workavailable ? mEventCounter : -1; - i.maxEvents = mMaxEvents; - i.part = mPartCounter + 1; - i.nparts = numberofparts; - // assign a deterministic (yet collision free seed) to process this particle chunk in Geant - // limit range to uint32_t since internal limit of TRandom (despite API suggesting otherwise) - const uint64_t drawnSeed = (uint64_t)(static_cast(std::numeric_limits::max()) * mSeedGenerator.Rndm()); - i.seed = mUseFixedChunkSeed ? mFixedChunkSeed : drawnSeed; - i.index = m.mParticles.size(); - i.mMCEventHeader = mEventHeader; - m.mSubEventInfo = i; - - int endindex = prims.size() - mPartCounter * mChunkGranularity; - int startindex = prims.size() - (mPartCounter + 1) * mChunkGranularity; - LOG(debug) << "indices " << startindex << " " << endindex; - - if (startindex < 0) { - startindex = 0; - } - if (endindex < 0) { - endindex = 0; - } - - for (int index = startindex; index < endindex; ++index) { - m.mParticles.emplace_back(prims[index]); - } - - LOG(info) << "Sending " << m.mParticles.size() << " particles"; - LOG(info) << "treating ev " << mEventCounter << " part " << i.part << " out of " << i.nparts; - - // feedback to driver if new event started - if (mPipeToDriver != -1 && i.part == 1 && workavailable) { - if (write(mPipeToDriver, &mEventCounter, sizeof(mEventCounter))) { - } - } - - mPartCounter++; - if (mPartCounter == numberofparts) { - mNeedNewEvent = true; - // start generation of a new event - if (mEventCounter < mMaxEvents) { - mGeneratorThread = std::thread(&O2PrimaryServerDevice::generateEvent, this); - } - } - - TMessage* tmsg = new TMessage(kMESS_OBJECT); - tmsg->WriteObjectAny((void*)&m, TClass::GetClass("o2::data::PrimaryChunk")); - - auto free_tmessage = [](void* data, void* hint) { delete static_cast(hint); }; - - std::unique_ptr message(channel.NewMessage(tmsg->Buffer(), tmsg->BufferSize(), free_tmessage, tmsg)); - - reply.AddPart(std::move(message)); - } - - // send answer - TStopwatch timer; - timer.Start(); - auto code = Send(reply, "primary-get", 0, 5000); // we introduce timeout in order not to block other requests - timer.Stop(); - auto time = timer.CpuTime(); - if (code > 0) { - LOG(debug) << "Reply send in " << time << "s"; - return workavailable; - } else { - LOG(warn) << "Sending process had problems. Return code : " << code << " time " << time << "s"; - } - return false; // -> error should not get here - } - - void stateTransition(O2PrimaryServerState to, const char* message) - { - LOG(info) << message << " CHANGING STATE TO " << PrimStateToString[(int)to]; - mState = to; - } - - void waitForControlInput() - { - mWaitingControlInput.store(1); - if (mState.load() != O2PrimaryServerState::Idle) { - mWaitingControlInput.store(0); - return; - } - - o2::simpubsub::publishMessage(GetChannels()["primary-notifications"].at(0), o2::simpubsub::simStatusString("PRIMSERVER", "STATUS", "AWAITING INPUT")); - // this means we are idling + bool HandleConfigRequest(fair::mq::Channel& channel); - std::unique_ptr reply(mControlChannel.NewMessage()); + bool ConditionalRun() override; - bool ok = false; + void PostRun() override; - LOG(info) << "WAITING FOR CONTROL INPUT"; - if (mControlChannel.Receive(reply) > 0) { - stateTransition(O2PrimaryServerState::Initializing, "CONTROL"); - auto data = reply->GetData(); - auto size = reply->GetSize(); + bool HandleRequest(fair::mq::MessagePtr& request, int /*index*/, fair::mq::Channel& channel); - std::string command(reinterpret_cast(data), size); - LOG(info) << "message: " << command; + void stateTransition(O2PrimaryServerState to, const char* message); - o2::conf::SimReconfigData reconfig; - o2::conf::parseSimReconfigFromString(command, reconfig); - LOG(info) << "Processing " << reconfig.nEvents << " new events"; - try { - LOG(info) << "REINIT START"; - ok = ReInit(reconfig); - LOG(info) << "REINIT DONE"; - } catch (std::exception e) { - LOG(info) << "Exception during reinit"; - } - } else { - LOG(info) << "NOTHING RECEIVED"; - } - if (ok) { - // stateTransition(O2PrimaryServerState::ReadyToServe, "CONTROL"); --> SHOULD BE DONE FROM EVENT GENERATOR (which get's however called only when mEvents>0) - } else { - stateTransition(O2PrimaryServerState::Stopped, "CONTROL"); - } - mWaitingControlInput.store(0); - } + void waitForControlInput(); private: o2::conf::SimConfig mSimConfig = o2::conf::SimConfig::Instance(); // local sim config object diff --git a/run/O2SimDevice.cxx b/run/O2SimDevice.cxx new file mode 100644 index 0000000000000..db12481e64622 --- /dev/null +++ b/run/O2SimDevice.cxx @@ -0,0 +1,269 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @author Sandro Wenzel + +#include "O2SimDevice.h" +#include "../macro/o2sim.C" +#include "TVirtualMC.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void doLogInfo(int workerID, std::string const& message) +{ + LOG(info) << "[W" << workerID << "] " << message; +} + +namespace o2 +{ +namespace devices +{ + +O2SimDevice::~O2SimDevice() +{ + FairSystemInfo sysinfo; + o2::utils::ShmManager::Instance().release(); + LOG(info) << "Shutting down O2SimDevice"; + LOG(info) << "TIME-STAMP " << mTimer.RealTime() << "\t"; + LOG(info) << "MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " << sysinfo.GetMaxMemory() << " MB\n"; +} + +void O2SimDevice::InitTask() +{ + // in the initialization phase we will init the simulation + // NOTE: In a fair::mq::Device this is better done here (instead of outside) since + // we have to setup simulation + worker in the same thread (due to many threadlocal variables + // in the simulation) ... at least as long fair::mq::Device is not spawning workers on the master thread + initSim(GetChannels().at("o2sim-primserv-info").at(0), mSimRun); + + // set the vmc and app pointers + mVMC = TVirtualMC::GetMC(); + mVMCApp = static_cast(TVirtualMCApplication::Instance()); + lateInit(); +} + +void O2SimDevice::lateInit() +{ + // late init + mVMCApp->initLate(); +} + +bool O2SimDevice::initSim(fair::mq::Channel& channel, std::unique_ptr& simptr) +{ + if (!o2::querySimConfig(channel)) { + return false; + } + + LOG(info) << "Setting up the simulation ..."; + simptr = std::move(std::unique_ptr(o2sim_init(true))); + FairSystemInfo sysinfo; + + // to finish initialization (trigger further cross section table building etc) -- which especially + // G4 is doing at the first ProcessRun + // The goal is to have everything setup before we fork + TVirtualMC::GetMC()->ProcessRun(0); + + LOG(info) << "MEM-STAMP END OF SIM INIT" << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " + << sysinfo.GetMaxMemory() << " MB\n"; + + return true; +} + +bool O2SimDevice::isWorkAvailable(fair::mq::Channel& statuschannel, int workerID) +{ + std::stringstream str; + str << "[W" << workerID << "]"; + auto workerStr = str.str(); + + int timeoutinMS = 2000; // wait for 2s max + bool reprobe = true; + while (reprobe) { + reprobe = false; + int i = -1; + fair::mq::MessagePtr request(statuschannel.NewSimpleMessage((int)O2PrimaryServerInfoRequest::Status)); + fair::mq::MessagePtr reply(statuschannel.NewSimpleMessage(i)); + auto sendcode = statuschannel.Send(request, timeoutinMS); + if (sendcode > 0) { + LOG(info) << workerStr << " Waiting for status answer "; + auto code = statuschannel.Receive(reply, timeoutinMS); + if (code > 0) { + int state(*((int*)(reply->GetData()))); + if (state == (int)o2::O2PrimaryServerState::ReadyToServe) { + LOG(info) << workerStr << " SERVER IS SERVING"; + return true; + } else if (state == (int)o2::O2PrimaryServerState::Initializing) { + LOG(info) << workerStr << " SERVER IS STILL INITIALIZING"; + reprobe = true; + sleep(1); + } else if (state == (int)o2::O2PrimaryServerState::WaitingEvent) { + LOG(info) << workerStr << " SERVER IS WAITING FOR EVENT"; + reprobe = true; + sleep(1); + } else if (state == (int)o2::O2PrimaryServerState::Idle) { + LOG(info) << workerStr << " SERVER IS IDLE"; + return false; + } else { + LOG(info) << workerStr << " SERVER STATE UNKNOWN OR STOPPED"; + } + } else { + LOG(error) << workerStr << " STATUS REQUEST UNSUCCESSFUL"; + } + } + } + return false; +} + +bool O2SimDevice::Kernel(int workerID, fair::mq::Channel& requestchannel, fair::mq::Channel& dataoutchannel, fair::mq::Channel* statuschannel) +{ + static int counter = 0; + bool reproducibleSim = true; + if (getenv("O2_DISABLE_REPRODUCIBLE_SIM")) { + reproducibleSim = false; + } + + // Mainly for debugging reasons, we allow to transport + // a specific event + eventpart. This allows to reproduce and debug bugs faster, once + // we know in which precise chunk they occur. The expected format for the environment variable + // is "eventnum:partid". + auto eventselection = getenv("O2SIM_RESTRICT_EVENTPART"); + int focus_on_event = -1; + int focus_on_part = -1; + if (eventselection) { + auto splitString = [](const std::string& str) { + std::pair parts; + size_t pos = str.find(':'); + if (pos != std::string::npos) { + parts.first = str.substr(0, pos); + parts.second = str.substr(pos + 1); + } + return parts; + }; + auto p = splitString(eventselection); + focus_on_event = std::atoi(p.first.c_str()); + focus_on_part = std::atoi(p.second.c_str()); + } + + fair::mq::MessagePtr request(requestchannel.NewSimpleMessage(PrimaryChunkRequest{workerID, -1, counter++})); // <-- don't need content; channel means -> give primaries + fair::mq::Parts reply; + + mVMCApp->setSimDataChannel(&dataoutchannel); + + // we log info with workerID prepended + auto workerStr = [workerID]() { + std::stringstream str; + str << "[W" << workerID << "]"; + return str.str(); + }; + + doLogInfo(workerID, "Requesting work chunk"); + int timeoutinMS = 2000; + auto sendcode = requestchannel.Send(request, timeoutinMS); + if (sendcode > 0) { + doLogInfo(workerID, "Waiting for answer"); + // asking for primary generation + + auto code = requestchannel.Receive(reply); + if (code > 0) { + doLogInfo(workerID, "Primary chunk received"); + auto rawmessage = std::move(reply.At(0)); + auto header = *(o2::PrimaryChunkAnswer*)(rawmessage->GetData()); + if (!header.payload_attached) { + doLogInfo(workerID, "No payload; Server in stage " + std::string(PrimStateToString[(int)header.serverstate])); + // if no payload attached we inspect the server state, to see what to do + if (header.serverstate == O2PrimaryServerState::Initializing || header.serverstate == O2PrimaryServerState::WaitingEvent) { + sleep(1); // back-off and retry + return true; + } + // we need to decide what to do when the server is idle ---> if this happens immediately after a new batch request it means that the server might just lag a bit behind + return false; + } else { + auto payload = std::move(reply.At(1)); + // wrap incoming bytes as a TMessageWrapper which offers "adoption" of a buffer + auto message = new TMessageWrapper(payload->GetData(), payload->GetSize()); + auto chunk = static_cast(message->ReadObjectAny(message->GetClass())); + + bool goon = true; + // no particles and eventID == -1 --> indication for no more work + if (chunk->mParticles.size() == 0 && chunk->mSubEventInfo.eventID == -1) { + doLogInfo(workerID, "No particles in reply : quitting kernel"); + goon = false; + } + + if (goon) { + + auto info = chunk->mSubEventInfo; + LOG(info) << workerStr() << " Processing " << chunk->mParticles.size() << " primary particles " + << "for event " << info.eventID << "/" << info.maxEvents << " " + << "part " << info.part << "/" << info.nparts; + + if (eventselection == nullptr || (focus_on_event == info.eventID && focus_on_part == info.part)) { + mVMCApp->setPrimaries(chunk->mParticles); + } else { + // nothing to transport here + mVMCApp->setPrimaries(std::vector{}); + LOG(info) << workerStr() << " This chunk will be skipped"; + } + + mVMCApp->setSubEventInfo(&info); + + if (reproducibleSim) { + LOG(info) << workerStr() << " Setting seed for this sub-event to " << chunk->mSubEventInfo.seed; + gRandom->SetSeed(chunk->mSubEventInfo.seed); + o2::base::VMCSeederService::instance().setSeed(); + } + + // Process one event + auto& conf = o2::conf::SimConfig::Instance(); + if (strcmp(conf.getMCEngine().c_str(), "TGeant4") == 0 || strcmp(conf.getMCEngine().c_str(), "O2TrivialMCEngine") == 0) { + // this is preferred and necessary for Geant4 + // since repeated "ProcessRun" might have significant overheads + mVMC->ProcessEvent(); + } else { + // for Geant3 calling ProcessEvent is not enough + // as some hooks are not called + mVMC->ProcessRun(1); + } + + FairSystemInfo sysinfo; + LOG(info) << workerStr() << " TIME-STAMP " << mTimer.RealTime() << "\t"; + mTimer.Continue(); + LOG(info) << workerStr() << " MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " + << sysinfo.GetMaxMemory() << " MB\n"; + } + delete message; + delete chunk; + } + } else { + LOG(info) << workerStr() << " No primary answer received from server (within timeout). Return code " << code; + } + } else { + LOG(info) << workerStr() << " Requesting work from server not possible. Return code " << sendcode; + return false; + } + return true; +} + +bool O2SimDevice::ConditionalRun() +{ + return Kernel(-1, GetChannels().at("primary-get").at(0), GetChannels().at("simdata").at(0)); +} + +void O2SimDevice::PostRun() { LOG(info) << "Shutting down "; } + +} // namespace devices +} // namespace o2 diff --git a/run/O2SimDevice.h b/run/O2SimDevice.h index 9256734cce487..730b0ed0c5d27 100644 --- a/run/O2SimDevice.h +++ b/run/O2SimDevice.h @@ -15,39 +15,27 @@ #define ALICEO2_DEVICES_SIMDEVICE_H_ #include -#include +#include #include -#include -#include -#include "../macro/o2sim.C" -#include "TVirtualMC.h" -#include "TMessage.h" -#include -#include -#include -#include -#include -#include +#include +#include #include "PrimaryServerState.h" -// a helper for logging with worker index prefixed -void doLogInfo(int workerID, std::string const& message) +class TVirtualMC; + +namespace o2::steer { - LOG(info) << "[W" << workerID << "] " << message; +class O2MCApplication; } +// a helper for logging with worker index prefixed +void doLogInfo(int workerID, std::string const& message); + namespace o2 { namespace devices { -class TMessageWrapper : public TMessage -{ - public: - TMessageWrapper(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); } - ~TMessageWrapper() override = default; -}; - // device representing a simulation worker class O2SimDevice final : public fair::mq::Device { @@ -56,282 +44,27 @@ class O2SimDevice final : public fair::mq::Device O2SimDevice(o2::steer::O2MCApplication* vmcapp, TVirtualMC* vmc) : mVMCApp{vmcapp}, mVMC{vmc} {} /// Default destructor - ~O2SimDevice() final - { - FairSystemInfo sysinfo; - o2::utils::ShmManager::Instance().release(); - LOG(info) << "Shutting down O2SimDevice"; - LOG(info) << "TIME-STAMP " << mTimer.RealTime() << "\t"; - LOG(info) << "MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " << sysinfo.GetMaxMemory() << " MB\n"; - } + ~O2SimDevice() final; protected: /// Overloads the InitTask() method of fair::mq::Device - void InitTask() final - { - // in the initialization phase we will init the simulation - // NOTE: In a fair::mq::Device this is better done here (instead of outside) since - // we have to setup simulation + worker in the same thread (due to many threadlocal variables - // in the simulation) ... at least as long fair::mq::Device is not spawning workers on the master thread - initSim(GetChannels().at("o2sim-primserv-info").at(0), mSimRun); - - // set the vmc and app pointers - mVMC = TVirtualMC::GetMC(); - mVMCApp = static_cast(TVirtualMCApplication::Instance()); - lateInit(); - } - - static void CustomCleanup(void* data, void* hint) { delete static_cast(hint); } + void InitTask() final; public: - void lateInit() - { - // late init - mVMCApp->initLate(); - } - - // should go into a helper - // this function queries the sim config data and initializes the SimConfig singleton - // returns true if successful / false if not - static bool querySimConfig(fair::mq::Channel& channel) - { - std::unique_ptr request(channel.NewSimpleMessage((int)O2PrimaryServerInfoRequest::Config)); - std::unique_ptr reply(channel.NewMessage()); - - int timeoutinMS = 60000; // wait for 60s max --> should be fast reply - if (channel.Send(request, timeoutinMS) > 0) { - LOG(info) << "Waiting for configuration answer "; - if (channel.Receive(reply, timeoutinMS) > 0) { - LOG(info) << "Configuration answer received, containing " << reply->GetSize() << " bytes "; - - // the answer is a TMessage containing the simulation Configuration - auto message = std::make_unique(reply->GetData(), reply->GetSize()); - auto config = static_cast(message.get()->ReadObjectAny(message.get()->GetClass())); - if (!config) { - return false; - } - - LOG(info) << "COMMUNICATED ENGINE " << config->mMCEngine; - - auto& conf = o2::conf::SimConfig::Instance(); - conf.resetFromConfigData(*config); - FairLogger::GetLogger()->SetLogVerbosityLevel(conf.getLogVerbosity().c_str()); - delete config; - } else { - LOG(error) << "No configuration received within " << timeoutinMS << "ms\n"; - return false; - } - } else { - LOG(error) << "Could not send configuration request within " << timeoutinMS << "ms\n"; - return false; - } - return true; - } + void lateInit(); // initializes the simulation classes; queries the configuration on a given channel - static bool initSim(fair::mq::Channel& channel, std::unique_ptr& simptr) - { - if (!querySimConfig(channel)) { - return false; - } - - LOG(info) << "Setting up the simulation ..."; - simptr = std::move(std::unique_ptr(o2sim_init(true))); - FairSystemInfo sysinfo; - - // to finish initialization (trigger further cross section table building etc) -- which especially - // G4 is doing at the first ProcessRun - // The goal is to have everything setup before we fork - TVirtualMC::GetMC()->ProcessRun(0); - - LOG(info) << "MEM-STAMP END OF SIM INIT" << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " - << sysinfo.GetMaxMemory() << " MB\n"; - - return true; - } - - bool isWorkAvailable(fair::mq::Channel& statuschannel, int workerID = -1) - { - std::stringstream str; - str << "[W" << workerID << "]"; - auto workerStr = str.str(); - - int timeoutinMS = 2000; // wait for 2s max - bool reprobe = true; - while (reprobe) { - reprobe = false; - int i = -1; - fair::mq::MessagePtr request(statuschannel.NewSimpleMessage((int)O2PrimaryServerInfoRequest::Status)); - fair::mq::MessagePtr reply(statuschannel.NewSimpleMessage(i)); - auto sendcode = statuschannel.Send(request, timeoutinMS); - if (sendcode > 0) { - LOG(info) << workerStr << " Waiting for status answer "; - auto code = statuschannel.Receive(reply, timeoutinMS); - if (code > 0) { - int state(*((int*)(reply->GetData()))); - if (state == (int)o2::O2PrimaryServerState::ReadyToServe) { - LOG(info) << workerStr << " SERVER IS SERVING"; - return true; - } else if (state == (int)o2::O2PrimaryServerState::Initializing) { - LOG(info) << workerStr << " SERVER IS STILL INITIALIZING"; - reprobe = true; - sleep(1); - } else if (state == (int)o2::O2PrimaryServerState::WaitingEvent) { - LOG(info) << workerStr << " SERVER IS WAITING FOR EVENT"; - reprobe = true; - sleep(1); - } else if (state == (int)o2::O2PrimaryServerState::Idle) { - LOG(info) << workerStr << " SERVER IS IDLE"; - return false; - } else { - LOG(info) << workerStr << " SERVER STATE UNKNOWN OR STOPPED"; - } - } else { - LOG(error) << workerStr << " STATUS REQUEST UNSUCCESSFUL"; - } - } - } - return false; - } - - bool Kernel(int workerID, fair::mq::Channel& requestchannel, fair::mq::Channel& dataoutchannel, fair::mq::Channel* statuschannel = nullptr) - { - static int counter = 0; - bool reproducibleSim = true; - if (getenv("O2_DISABLE_REPRODUCIBLE_SIM")) { - reproducibleSim = false; - } - - // Mainly for debugging reasons, we allow to transport - // a specific event + eventpart. This allows to reproduce and debug bugs faster, once - // we know in which precise chunk they occur. The expected format for the environment variable - // is "eventnum:partid". - auto eventselection = getenv("O2SIM_RESTRICT_EVENTPART"); - int focus_on_event = -1; - int focus_on_part = -1; - if (eventselection) { - auto splitString = [](const std::string& str) { - std::pair parts; - size_t pos = str.find(':'); - if (pos != std::string::npos) { - parts.first = str.substr(0, pos); - parts.second = str.substr(pos + 1); - } - return parts; - }; - auto p = splitString(eventselection); - focus_on_event = std::atoi(p.first.c_str()); - focus_on_part = std::atoi(p.second.c_str()); - } - - fair::mq::MessagePtr request(requestchannel.NewSimpleMessage(PrimaryChunkRequest{workerID, -1, counter++})); // <-- don't need content; channel means -> give primaries - fair::mq::Parts reply; - - mVMCApp->setSimDataChannel(&dataoutchannel); - - // we log info with workerID prepended - auto workerStr = [workerID]() { - std::stringstream str; - str << "[W" << workerID << "]"; - return str.str(); - }; - - doLogInfo(workerID, "Requesting work chunk"); - int timeoutinMS = 2000; - auto sendcode = requestchannel.Send(request, timeoutinMS); - if (sendcode > 0) { - doLogInfo(workerID, "Waiting for answer"); - // asking for primary generation - - auto code = requestchannel.Receive(reply); - if (code > 0) { - doLogInfo(workerID, "Primary chunk received"); - auto rawmessage = std::move(reply.At(0)); - auto header = *(o2::PrimaryChunkAnswer*)(rawmessage->GetData()); - if (!header.payload_attached) { - doLogInfo(workerID, "No payload; Server in stage " + std::string(PrimStateToString[(int)header.serverstate])); - // if no payload attached we inspect the server state, to see what to do - if (header.serverstate == O2PrimaryServerState::Initializing || header.serverstate == O2PrimaryServerState::WaitingEvent) { - sleep(1); // back-off and retry - return true; - } - // we need to decide what to do when the server is idle ---> if this happens immediately after a new batch request it means that the server might just lag a bit behind - return false; - } else { - auto payload = std::move(reply.At(1)); - // wrap incoming bytes as a TMessageWrapper which offers "adoption" of a buffer - auto message = new TMessageWrapper(payload->GetData(), payload->GetSize()); - auto chunk = static_cast(message->ReadObjectAny(message->GetClass())); - - bool goon = true; - // no particles and eventID == -1 --> indication for no more work - if (chunk->mParticles.size() == 0 && chunk->mSubEventInfo.eventID == -1) { - doLogInfo(workerID, "No particles in reply : quitting kernel"); - goon = false; - } - - if (goon) { - - auto info = chunk->mSubEventInfo; - LOG(info) << workerStr() << " Processing " << chunk->mParticles.size() << " primary particles " - << "for event " << info.eventID << "/" << info.maxEvents << " " - << "part " << info.part << "/" << info.nparts; - - if (eventselection == nullptr || (focus_on_event == info.eventID && focus_on_part == info.part)) { - mVMCApp->setPrimaries(chunk->mParticles); - } else { - // nothing to transport here - mVMCApp->setPrimaries(std::vector{}); - LOG(info) << workerStr() << " This chunk will be skipped"; - } - - mVMCApp->setSubEventInfo(&info); - - if (reproducibleSim) { - LOG(info) << workerStr() << " Setting seed for this sub-event to " << chunk->mSubEventInfo.seed; - gRandom->SetSeed(chunk->mSubEventInfo.seed); - o2::base::VMCSeederService::instance().setSeed(); - } + static bool initSim(fair::mq::Channel& channel, std::unique_ptr& simptr); - // Process one event - auto& conf = o2::conf::SimConfig::Instance(); - if (strcmp(conf.getMCEngine().c_str(), "TGeant4") == 0 || strcmp(conf.getMCEngine().c_str(), "O2TrivialMCEngine") == 0) { - // this is preferred and necessary for Geant4 - // since repeated "ProcessRun" might have significant overheads - mVMC->ProcessEvent(); - } else { - // for Geant3 calling ProcessEvent is not enough - // as some hooks are not called - mVMC->ProcessRun(1); - } + bool isWorkAvailable(fair::mq::Channel& statuschannel, int workerID = -1); - FairSystemInfo sysinfo; - LOG(info) << workerStr() << " TIME-STAMP " << mTimer.RealTime() << "\t"; - mTimer.Continue(); - LOG(info) << workerStr() << " MEM-STAMP " << sysinfo.GetCurrentMemory() / (1024. * 1024) << " " - << sysinfo.GetMaxMemory() << " MB\n"; - } - delete message; - delete chunk; - } - } else { - LOG(info) << workerStr() << " No primary answer received from server (within timeout). Return code " << code; - } - } else { - LOG(info) << workerStr() << " Requesting work from server not possible. Return code " << sendcode; - return false; - } - return true; - } + bool Kernel(int workerID, fair::mq::Channel& requestchannel, fair::mq::Channel& dataoutchannel, fair::mq::Channel* statuschannel = nullptr); protected: /// Overloads the ConditionalRun() method of fair::mq::Device - bool ConditionalRun() final - { - return Kernel(-1, GetChannels().at("primary-get").at(0), GetChannels().at("simdata").at(0)); - } + bool ConditionalRun() final; - void PostRun() final { LOG(info) << "Shutting down "; } + void PostRun() final; private: TStopwatch mTimer; //! diff --git a/run/O2SimDeviceRunner.cxx b/run/O2SimDeviceRunner.cxx index 609311809d5d9..87b6ae99df5f4 100644 --- a/run/O2SimDeviceRunner.cxx +++ b/run/O2SimDeviceRunner.cxx @@ -13,6 +13,10 @@ #include "O2SimDevice.h" #include "SimSetup/SimSetup.h" +#include +#include +#include +#include #include #include #include diff --git a/run/PrimaryServerState.cxx b/run/PrimaryServerState.cxx new file mode 100644 index 0000000000000..1d56983ba937d --- /dev/null +++ b/run/PrimaryServerState.cxx @@ -0,0 +1,58 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "PrimaryServerState.h" +#include +#include +#include +#include +#include +#include + +namespace o2 +{ + +bool querySimConfig(fair::mq::Channel& channel) +{ + std::unique_ptr request(channel.NewSimpleMessage((int)O2PrimaryServerInfoRequest::Config)); + std::unique_ptr reply(channel.NewMessage()); + + int timeoutinMS = 60000; // wait for 60s max --> should be fast reply + if (channel.Send(request, timeoutinMS) > 0) { + LOG(info) << "Waiting for configuration answer "; + if (channel.Receive(reply, timeoutinMS) > 0) { + LOG(info) << "Configuration answer received, containing " << reply->GetSize() << " bytes "; + + // the answer is a TMessage containing the simulation Configuration + auto message = std::make_unique(reply->GetData(), reply->GetSize()); + auto config = static_cast(message.get()->ReadObjectAny(message.get()->GetClass())); + if (!config) { + return false; + } + + LOG(info) << "COMMUNICATED ENGINE " << config->mMCEngine; + + auto& conf = o2::conf::SimConfig::Instance(); + conf.resetFromConfigData(*config); + FairLogger::GetLogger()->SetLogVerbosityLevel(conf.getLogVerbosity().c_str()); + delete config; + } else { + LOG(error) << "No configuration received within " << timeoutinMS << "ms\n"; + return false; + } + } else { + LOG(error) << "Could not send configuration request within " << timeoutinMS << "ms\n"; + return false; + } + return true; +} + +} // namespace o2 diff --git a/run/PrimaryServerState.h b/run/PrimaryServerState.h index 4bae1d566dc60..8a0b9435dd6df 100644 --- a/run/PrimaryServerState.h +++ b/run/PrimaryServerState.h @@ -12,6 +12,9 @@ #ifndef O2_PRIMARYSERVERSTATE_H #define O2_PRIMARYSERVERSTATE_H +#include +#include + namespace o2 { @@ -23,7 +26,7 @@ enum class O2PrimaryServerState { Idle = 3, Stopped = 4 }; -static const char* PrimStateToString[5] = {"INIT", "SERVING", "WAITEVENT", "IDLE", "STOPPED"}; +inline constexpr const char* PrimStateToString[5] = {"INIT", "SERVING", "WAITEVENT", "IDLE", "STOPPED"}; /// enum class for request to o2sim-primserv-info channel of the O2PrimaryServerDevice enum class O2PrimaryServerInfoRequest { @@ -47,6 +50,18 @@ struct PrimaryChunkAnswer { bool payload_attached; // whether real payload follows (or server has no work at this moment) }; +/// A TMessage reading from a buffer it does not own +class TMessageWrapper : public TMessage +{ + public: + TMessageWrapper(void* buf, Int_t len) : TMessage(buf, len) { ResetBit(kIsOwner); } + ~TMessageWrapper() override = default; +}; + +/// Queries the simulation configuration from the primary server and initializes the SimConfig singleton. +/// Returns true if successful. +bool querySimConfig(fair::mq::Channel& channel); + } // namespace o2 #endif //O2_PRIMARYSERVERSTATE_H diff --git a/run/SimPublishChannelHelper.h b/run/SimPublishChannelHelper.h index 57439a91e1514..899f07fc5efad 100644 --- a/run/SimPublishChannelHelper.h +++ b/run/SimPublishChannelHelper.h @@ -23,7 +23,7 @@ namespace o2::simpubsub // create an IPC socket name of the type // ipc:///tmp/base-PID // base should be for example "o2sim-worker" or "o2sim-merger" -std::string getPublishAddress(std::string const& base, int pid = getpid()) +inline std::string getPublishAddress(std::string const& base, int pid = getpid()) { std::stringstream publishsocketname; publishsocketname << "ipc:///tmp/" << base << "-" << pid; @@ -31,13 +31,13 @@ std::string getPublishAddress(std::string const& base, int pid = getpid()) } // some standard format for pub-sub subscribers -std::string simStatusString(std::string const& origin, std::string const& topic, std::string const& message) +inline std::string simStatusString(std::string const& origin, std::string const& topic, std::string const& message) { return origin + std::string("[") + topic + std::string("] : ") + message; } // helper function to publish a message to an outside subscriber -bool publishMessage(fair::mq::Channel& channel, std::string const& message) +inline bool publishMessage(fair::mq::Channel& channel, std::string const& message) { if (channel.IsValid()) { auto text = new std::string(message); @@ -54,8 +54,8 @@ bool publishMessage(fair::mq::Channel& channel, std::string const& message) } // make channel (transport factory needs to be injected) -fair::mq::Channel createPUBChannel(std::string const& address, - std::string const& type = "pub") +inline fair::mq::Channel createPUBChannel(std::string const& address, + std::string const& type = "pub") { auto factory = fair::mq::TransportFactory::CreateTransportFactory("zeromq"); static int i = 0; diff --git a/run/o2sim_parallel.cxx b/run/o2sim_parallel.cxx index 8a92a5f251cb0..0d5ff361141a8 100644 --- a/run/o2sim_parallel.cxx +++ b/run/o2sim_parallel.cxx @@ -763,7 +763,7 @@ int main(int argc, char* argv[]) // Handle mergerpid status separately if (cpid == mergerpid) { if (WIFEXITED(status)) { - if (WEXITSTATUS(status) != 0 || WEXITSTATUS(status) != 128) { + if (WEXITSTATUS(status) != 0 && WEXITSTATUS(status) != 128) { LOG(error) << "Merger process exited with abnormal exit status " << WEXITSTATUS(status); errored = true; }