Application/plugins/OscarMTProducer.cc

0001 #include <iostream>
0002 #include <memory>
0003
0004 #include "FWCore/Framework/interface/stream/EDProducer.h"
0005 #include "FWCore/Framework/interface/Event.h"
0006 #include "FWCore/Framework/interface/MakerMacros.h"
0007 #include "FWCore/Framework/interface/EventSetup.h"
0008 #include "FWCore/Framework/interface/Run.h"
0009
0010 #include "FWCore/PluginManager/interface/PluginManager.h"
0011 #include "FWCore/Framework/interface/MakerMacros.h"
0012 #include "FWCore/Framework/interface/ConsumesCollector.h"
0013 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0014
0015 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0016 #include "FWCore/ServiceRegistry/interface/Service.h"
0017 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0018 #include "FWCore/Utilities/interface/RandomNumberGenerator.h"
0019 #include "FWCore/Utilities/interface/Exception.h"
0020
0021 #include "SimG4Core/Application/interface/OscarMTMasterThread.h"
0022 #include "SimG4Core/Application/interface/RunManagerMT.h"
0023 #include "SimG4Core/Application/interface/RunManagerMTWorker.h"
0024 #include "SimG4Core/Notification/interface/TmpSimEvent.h"
0025 #include "SimG4Core/Notification/interface/TmpSimVertex.h"
0026
0027 #include "SimG4Core/SensitiveDetector/interface/SensitiveTkDetector.h"
0028 #include "SimG4Core/SensitiveDetector/interface/SensitiveCaloDetector.h"
0029
0030 #include "SimG4Core/Watcher/interface/SimProducer.h"
0031
0032 #include "SimDataFormats/Track/interface/SimTrackContainer.h"
0033 #include "SimDataFormats/Vertex/interface/SimVertexContainer.h"
0034 #include "SimDataFormats/TrackingHit/interface/PSimHitContainer.h"
0035 #include "SimDataFormats/CaloHit/interface/PCaloHitContainer.h"
0036
0037 #include "SimG4Core/Application/interface/ThreadHandoff.h"
0038
0039 #include "Randomize.hh"
0040
0041 // for some reason void doesn't compile
0042 class OscarMTProducer : public edm::stream::EDProducer<edm::GlobalCache<OscarMTMasterThread>, edm::RunCache<int>> {
0043 public:
0044   typedef std::vector<std::shared_ptr<SimProducer>> Producers;
0045
0046   explicit OscarMTProducer(edm::ParameterSet const& p, const OscarMTMasterThread*);
0047   ~OscarMTProducer() override;
0048
0049   static std::unique_ptr<OscarMTMasterThread> initializeGlobalCache(const edm::ParameterSet& iConfig);
0050   static std::shared_ptr<int> globalBeginRun(const edm::Run& iRun,
0051                                              const edm::EventSetup& iSetup,
0052                                              const OscarMTMasterThread* masterThread);
0053   static void globalEndRun(const edm::Run& iRun, const edm::EventSetup& iSetup, const RunContext* iContext);
0054   static void globalEndJob(OscarMTMasterThread* masterThread);
0055
0056   void beginRun(const edm::Run& r, const edm::EventSetup& c) override;
0057   void endRun(const edm::Run& r, const edm::EventSetup& c) override;
0058   void produce(edm::Event& e, const edm::EventSetup& c) override;
0059
0060 private:
0061   omt::ThreadHandoff m_handoff;
0062   std::unique_ptr<RunManagerMTWorker> m_runManagerWorker;
0063   const OscarMTMasterThread* m_masterThread;
0064   const edm::ParameterSetID m_psetID;
0065   int m_verbose;
0066   CMS_SA_ALLOW static const OscarMTMasterThread* s_masterThread;
0067   CMS_SA_ALLOW static edm::ParameterSetID s_psetID;
0068 };
0069
0070 const OscarMTMasterThread* OscarMTProducer::s_masterThread = nullptr;
0071 edm::ParameterSetID OscarMTProducer::s_psetID{};
0072
0073 namespace edm {
0074   class StreamID;
0075 }
0076
0077 namespace {
0078   //
0079   // this machinery allows to set CLHEP static engine
0080   // to the one defined by RandomNumberGenerator service
0081   // at the beginning of an event, and reset it back to
0082   // "default-default" at the end of the event;
0083   // Dave D. has decided to implement it this way because
0084   // we don't know if there're other modules using CLHEP
0085   // static engine, thus we want to ensure that the one
0086   // we use for OscarMTProducer is unique to OscarMTProducer
0087   //
0088   // !!! This not only sets the random engine used by GEANT.
0089   // There are a few SimWatchers/SimProducers that generate
0090   // random number and also use the global CLHEP random engine
0091   // set by this code. If we ever change this design be careful
0092   // not to forget about them!!!
0093
0094   class StaticRandomEngineSetUnset {
0095   public:
0096     StaticRandomEngineSetUnset(edm::StreamID const&);
0097     explicit StaticRandomEngineSetUnset(CLHEP::HepRandomEngine* engine);
0098     ~StaticRandomEngineSetUnset();
0099
0100     CLHEP::HepRandomEngine* currentEngine() { return m_currentEngine; }
0101
0102   private:
0103     CLHEP::HepRandomEngine* m_currentEngine;
0104     CLHEP::HepRandomEngine* m_previousEngine;
0105   };
0106 }  // namespace
0107
0108 OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMasterThread* ms)
0109     : m_handoff{p.getUntrackedParameter<int>("workerThreadStackSize", 10 * 1024 * 1024)}, m_psetID{p.id()} {
0110   m_verbose = p.getParameter<int>("EventVerbose");
0111   // Random number generation not allowed here
0112   StaticRandomEngineSetUnset random(nullptr);
0113
0114   auto token = edm::ServiceRegistry::instance().presentToken();
0115   m_handoff.runAndWait([this, &p, token]() {
0116     edm::ServiceRegistry::Operate guard{token};
0117     StaticRandomEngineSetUnset random(nullptr);
0118     m_runManagerWorker = std::make_unique<RunManagerMTWorker>(p, consumesCollector());
0119   });
0120   m_masterThread = (nullptr != ms) ? ms : s_masterThread;
0121   assert(m_masterThread);
0122   m_masterThread->callConsumes(consumesCollector());
0123
0124   // declair hit collections
0125   produces<edm::SimTrackContainer>().setBranchAlias("SimTracks");
0126   produces<edm::SimVertexContainer>().setBranchAlias("SimVertices");
0127
0128   auto trackHits = p.getParameter<std::vector<std::string>>("TrackHits");
0129   for (auto const& ss : trackHits) {
0130     produces<edm::PSimHitContainer>(ss);
0131   }
0132
0133   auto caloHits = p.getParameter<std::vector<std::string>>("CaloHits");
0134   for (auto const& ss : caloHits) {
0135     produces<edm::PCaloHitContainer>(ss);
0136   }
0137
0138   //register any products
0139   auto& producers = m_runManagerWorker->producers();
0140   for (auto& ptr : producers) {
0141     ptr->registerProducts(producesCollector());
0142   }
0143   edm::LogVerbatim("SimG4CoreApplication")
0144       << "OscarMTProducer is constructed with hit collections:" << trackHits.size() << " tracking type; "
0145       << caloHits.size() << " calo type; " << producers.size() << " watcher type.";
0146 }
0147
0148 OscarMTProducer::~OscarMTProducer() {
0149   auto token = edm::ServiceRegistry::instance().presentToken();
0150   m_handoff.runAndWait([this, token]() {
0151     edm::ServiceRegistry::Operate guard{token};
0152     m_runManagerWorker.reset();
0153   });
0154 }
0155
0156 std::unique_ptr<OscarMTMasterThread> OscarMTProducer::initializeGlobalCache(const edm::ParameterSet& iConfig) {
0157   // Random number generation not allowed here
0158   StaticRandomEngineSetUnset random(nullptr);
0159   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::initializeGlobalCache";
0160   if (nullptr == s_masterThread) {
0161     auto ret = std::make_unique<OscarMTMasterThread>(iConfig);
0162     s_masterThread = ret.get();
0163     s_psetID = iConfig.id();
0164     return ret;
0165   }
0166   return {};
0167 }
0168
0169 std::shared_ptr<int> OscarMTProducer::globalBeginRun(const edm::Run&,
0170                                                      const edm::EventSetup& iSetup,
0171                                                      const OscarMTMasterThread* masterThread) {
0172   // Random number generation not allowed here
0173   StaticRandomEngineSetUnset random(nullptr);
0174   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalBeginRun";
0175   if (masterThread) {
0176     masterThread->beginRun(iSetup);
0177   }
0178   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalBeginRun done";
0179   return std::shared_ptr<int>();
0180 }
0181
0182 void OscarMTProducer::globalEndRun(const edm::Run&, const edm::EventSetup&, const RunContext* iContext) {
0183   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalEndRun";
0184   if (nullptr != iContext->global()) {
0185     iContext->global()->endRun();
0186   }
0187 }
0188
0189 void OscarMTProducer::globalEndJob(OscarMTMasterThread* masterThread) {
0190   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalEndJob";
0191   if (masterThread) {
0192     masterThread->stopThread();
0193   }
0194 }
0195
0196 void OscarMTProducer::beginRun(const edm::Run&, const edm::EventSetup& es) {
0197   if (s_psetID != m_psetID) {
0198     throw cms::Exception("DiffOscarMTProducers")
0199         << "At least two different OscarMTProducer instances have been"
0200            "loaded into the job and they have different configurations.\n"
0201            " All OscarMTProducers in a job must have exactly the same configuration.";
0202   }
0203   int id = m_runManagerWorker->getThreadIndex();
0204   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::beginRun threadID=" << id;
0205   auto token = edm::ServiceRegistry::instance().presentToken();
0206   m_handoff.runAndWait([this, &es, token]() {
0207     edm::ServiceRegistry::Operate guard{token};
0208     m_runManagerWorker->beginRun(es);
0209     m_runManagerWorker->initializeG4(m_masterThread->runManagerMasterPtr(), es);
0210   });
0211   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::beginRun done threadID=" << id;
0212 }
0213
0214 void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) {
0215   StaticRandomEngineSetUnset random(nullptr);
0216   int id = m_runManagerWorker->getThreadIndex();
0217   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun threadID=" << id;
0218   auto token = edm::ServiceRegistry::instance().presentToken();
0219   m_handoff.runAndWait([this, token]() {
0220     edm::ServiceRegistry::Operate guard{token};
0221     m_runManagerWorker->endRun();
0222   });
0223   edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun done threadID=" << id;
0224 }
0225
0226 void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
0227   StaticRandomEngineSetUnset random(e.streamID());
0228   auto engine = random.currentEngine();
0229   int id = m_runManagerWorker->getThreadIndex();
0230   if (0 < m_verbose) {
0231     edm::LogVerbatim("SimG4CoreApplication")
0232         << "Produce event " << e.id() << " stream " << e.streamID() << " threadID=" << id;
0233     //edm::LogVerbatim("SimG4CoreApplication") << " rand= " << G4UniformRand();
0234   }
0235
0236   auto& sTk = m_runManagerWorker->sensTkDetectors();
0237   auto& sCalo = m_runManagerWorker->sensCaloDetectors();
0238
0239   TmpSimEvent* evt = nullptr;
0240   auto token = edm::ServiceRegistry::instance().presentToken();
0241   m_handoff.runAndWait([this, &e, &es, &evt, token, engine]() {
0242     edm::ServiceRegistry::Operate guard{token};
0243     StaticRandomEngineSetUnset random(engine);
0244     evt = m_runManagerWorker->produce(e, es, m_masterThread->runManagerMaster());
0245   });
0246
0247   std::unique_ptr<edm::SimTrackContainer> p1(new edm::SimTrackContainer);
0248   std::unique_ptr<edm::SimVertexContainer> p2(new edm::SimVertexContainer);
0249   evt->load(*p1);
0250   evt->load(*p2);
0251
0252   if (0 < m_verbose) {
0253     edm::LogVerbatim("SimG4CoreApplication")
0254         << "Produced " << p2->size() << " SimVertecies: position(cm), time(s), parentID, vertexID, processType";
0255     if (1 < m_verbose) {
0256       int nn = p2->size();
0257       for (int i = 0; i < nn; ++i) {
0258         edm::LogVerbatim("Vertex") << " " << i << ". " << (*p2)[i] << " " << (*p2)[i].processType();
0259       }
0260     }
0261     edm::LogVerbatim("SimG4CoreApplication")
0262         << "Produced " << p1->size()
0263         << " SimTracks: pdg, 4-momentum(GeV), vertexID, mcTruthID, flagBoundary, trackID at boundary";
0264     if (1 < m_verbose) {
0265       int nn = p1->size();
0266       for (int i = 0; i < nn; ++i) {
0267         edm::LogVerbatim("Track") << " " << i << ". " << (*p1)[i] << " " << (*p1)[i].crossedBoundary() << " "
0268                                   << (*p1)[i].getIDAtBoundary();
0269       }
0270     }
0271   }
0272   e.put(std::move(p1));
0273   e.put(std::move(p2));
0274
0275   for (auto const& tracker : sTk) {
0276     const std::vector<std::string>& v = tracker->getNames();
0277     for (auto const& name : v) {
0278       std::unique_ptr<edm::PSimHitContainer> product(new edm::PSimHitContainer);
0279       tracker->fillHits(*product, name);
0280       if (0 < m_verbose && product != nullptr && !product->empty())
0281         edm::LogVerbatim("SimG4CoreApplication") << "Produced " << product->size() << " tracker hits <" << name << ">";
0282       e.put(std::move(product), name);
0283     }
0284   }
0285   for (auto const& calo : sCalo) {
0286     const std::vector<std::string>& v = calo->getNames();
0287     for (auto const& name : v) {
0288       std::unique_ptr<edm::PCaloHitContainer> product(new edm::PCaloHitContainer);
0289       calo->fillHits(*product, name);
0290       if (0 < m_verbose && product != nullptr && !product->empty())
0291         edm::LogVerbatim("SimG4CoreApplication") << "Produced " << product->size() << " calo hits <" << name << ">";
0292       e.put(std::move(product), name);
0293     }
0294   }
0295
0296   auto& producers = m_runManagerWorker->producers();
0297   for (auto& prod : producers) {
0298     prod.get()->produce(e, es);
0299   }
0300   if (0 < m_verbose) {
0301     edm::LogVerbatim("SimG4CoreApplication")
0302         << "Event is produced event " << e.id() << " streamID=" << e.streamID() << " threadID=" << id;
0303     //edm::LogWarning("SimG4CoreApplication") << "EventID=" << e.id() << " rand=" << G4UniformRand();
0304   }
0305 }
0306
0307 StaticRandomEngineSetUnset::StaticRandomEngineSetUnset(edm::StreamID const& streamID) {
0308   edm::Service<edm::RandomNumberGenerator> rng;
0309   if (!rng.isAvailable()) {
0310     throw cms::Exception("Configuration")
0311         << "The OscarMTProducer module requires the RandomNumberGeneratorService\n"
0312            "which is not present in the configuration file.  You must add the service\n"
0313            "in the configuration file if you want to run OscarMTProducer";
0314   }
0315   m_currentEngine = &(rng->getEngine(streamID));
0316
0317   m_previousEngine = G4Random::getTheEngine();
0318   G4Random::setTheEngine(m_currentEngine);
0319 }
0320
0321 StaticRandomEngineSetUnset::StaticRandomEngineSetUnset(CLHEP::HepRandomEngine* engine) {
0322   m_currentEngine = engine;
0323   m_previousEngine = G4Random::getTheEngine();
0324   G4Random::setTheEngine(m_currentEngine);
0325 }
0326
0327 StaticRandomEngineSetUnset::~StaticRandomEngineSetUnset() { G4Random::setTheEngine(m_previousEngine); }
0328
0329 DEFINE_FWK_MODULE(OscarMTProducer);