File indexing completed on 2022-05-06 00:35:30
0001 #include "FWCore/Framework/interface/global/EDFilter.h"
0002 #include "FWCore/Framework/interface/Event.h"
0003 #include "FWCore/Utilities/interface/EDPutToken.h"
0004 #include "FWCore/Framework/interface/MakerMacros.h"
0005 #include "FWCore/ServiceRegistry/interface/Service.h"
0006 #include "FWCore/Utilities/interface/RandomNumberGenerator.h"
0007
0008 #include "SimDataFormats/GeneratorProducts/interface/HepMCProduct.h"
0009 #include "SimDataFormats/GeneratorProducts/interface/GenRunInfoProduct.h"
0010 #include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoHeader.h"
0011 #include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h"
0012 #include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h"
0013 #include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h"
0014 #include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h"
0015
0016 #include "FWCore/SharedMemory/interface/ReadBuffer.h"
0017 #include "FWCore/SharedMemory/interface/WriteBuffer.h"
0018 #include "FWCore/SharedMemory/interface/ControllerChannel.h"
0019 #include "FWCore/SharedMemory/interface/ROOTDeserializer.h"
0020 #include "FWCore/SharedMemory/interface/ROOTSerializer.h"
0021
0022 #include "CLHEP/Random/RandomEngine.h"
0023 #include "CLHEP/Random/engineIDulong.h"
0024 #include "CLHEP/Random/RanecuEngine.h"
0025
0026 #include <cstdio>
0027 #include <iostream>
0028
0029 using namespace edm::shared_memory;
0030 namespace externalgen {
0031
0032 struct StreamCache {
0033 StreamCache(const std::string& iConfig, int id, bool verbose, unsigned int waitTime)
0034 : id_{id},
0035 channel_("extGen", id_, waitTime),
0036 readBuffer_{channel_.sharedMemoryName(), channel_.fromWorkerBufferInfo()},
0037 writeBuffer_{std::string("Rand") + channel_.sharedMemoryName(), channel_.toWorkerBufferInfo()},
0038 deserializer_{readBuffer_},
0039 er_deserializer_{readBuffer_},
0040 bl_deserializer_{readBuffer_},
0041 el_deserializer_(readBuffer_),
0042 randSerializer_(writeBuffer_) {
0043
0044 fflush(stdout);
0045 fflush(stderr);
0046
0047 channel_.setupWorker([&]() {
0048 using namespace std::string_literals;
0049 edm::LogSystem("ExternalProcess") << id_ << " starting external process \n";
0050 std::string verboseCommand;
0051 if (verbose) {
0052 verboseCommand = "--verbose ";
0053 }
0054 pipe_ =
0055 popen(("cmsExternalGenerator "s + verboseCommand + channel_.sharedMemoryName() + " " + channel_.uniqueID())
0056 .c_str(),
0057 "w");
0058 if (nullptr == pipe_) {
0059 abort();
0060 }
0061
0062 {
0063 auto nlines = std::to_string(std::count(iConfig.begin(), iConfig.end(), '\n'));
0064 auto result = fwrite(nlines.data(), sizeof(char), nlines.size(), pipe_);
0065 assert(result == nlines.size());
0066 result = fwrite(iConfig.data(), sizeof(char), iConfig.size(), pipe_);
0067 assert(result == iConfig.size());
0068 fflush(pipe_);
0069 }
0070 });
0071 }
0072
0073 template <typename SERIAL>
0074 auto doTransition(SERIAL& iDeserializer, edm::Transition iTrans, unsigned long long iTransitionID)
0075 -> decltype(iDeserializer.deserialize()) {
0076 decltype(iDeserializer.deserialize()) value;
0077 if (not channel_.doTransition(
0078 [&value, &iDeserializer]() { value = iDeserializer.deserialize(); }, iTrans, iTransitionID)) {
0079 externalFailed_ = true;
0080 throw cms::Exception("ExternalFailed") << "failed waiting for external process " << channel_.uniqueID()
0081 << ". Timed out after " << channel_.maxWaitInSeconds() << " seconds.";
0082 }
0083 return value;
0084 }
0085 ExternalGeneratorEventInfo produce(edm::StreamID iStream, unsigned long long iTransitionID) {
0086 edm::Service<edm::RandomNumberGenerator> gen;
0087 auto& engine = gen->getEngine(iStream);
0088 edm::RandomNumberGeneratorState state{engine.put(), engine.getSeed()};
0089 randSerializer_.serialize(state);
0090
0091 return doTransition(deserializer_, edm::Transition::Event, iTransitionID);
0092 }
0093
0094 std::optional<GenRunInfoProduct> endRunProduce(unsigned long long iTransitionID) {
0095 if (not externalFailed_) {
0096 return doTransition(er_deserializer_, edm::Transition::EndRun, iTransitionID);
0097 }
0098 return {};
0099 }
0100
0101 ExternalGeneratorLumiInfo beginLumiProduce(unsigned long long iTransitionID,
0102 edm::RandomNumberGeneratorState const& iState) {
0103
0104 randSerializer_.serialize(const_cast<edm::RandomNumberGeneratorState&>(iState));
0105 return doTransition(bl_deserializer_, edm::Transition::BeginLuminosityBlock, iTransitionID);
0106 }
0107
0108 std::optional<GenLumiInfoProduct> endLumiProduce(unsigned long long iTransitionID) {
0109 if (not externalFailed_) {
0110 return doTransition(el_deserializer_, edm::Transition::EndLuminosityBlock, iTransitionID);
0111 }
0112 return {};
0113 }
0114
0115 ~StreamCache() {
0116 channel_.stopWorker();
0117 pclose(pipe_);
0118 }
0119
0120 private:
0121 std::string unique_name(std::string iBase) {
0122 auto pid = getpid();
0123 iBase += std::to_string(pid);
0124 iBase += "_";
0125 iBase += std::to_string(id_);
0126
0127 return iBase;
0128 }
0129
0130 int id_;
0131 FILE* pipe_;
0132 ControllerChannel channel_;
0133 ReadBuffer readBuffer_;
0134 WriteBuffer writeBuffer_;
0135
0136 template <typename T>
0137 using Deserializer = ROOTDeserializer<T, ReadBuffer>;
0138 Deserializer<ExternalGeneratorEventInfo> deserializer_;
0139 Deserializer<GenRunInfoProduct> er_deserializer_;
0140 Deserializer<ExternalGeneratorLumiInfo> bl_deserializer_;
0141 Deserializer<GenLumiInfoProduct> el_deserializer_;
0142 ROOTSerializer<edm::RandomNumberGeneratorState, WriteBuffer> randSerializer_;
0143
0144 bool externalFailed_ = false;
0145 };
0146
0147 struct RunCache {
0148
0149
0150 CMS_THREAD_SAFE mutable GenRunInfoProduct runInfo_;
0151 };
0152 struct LumiCache {
0153 LumiCache(std::vector<unsigned long> iState, long iSeed) : randomState_(std::move(iState), iSeed) {}
0154
0155
0156 CMS_THREAD_SAFE mutable edm::RandomNumberGeneratorState randomState_;
0157 };
0158 }
0159
0160 class ExternalGeneratorFilter : public edm::global::EDFilter<edm::StreamCache<externalgen::StreamCache>,
0161 edm::RunCache<externalgen::RunCache>,
0162 edm::EndRunProducer,
0163 edm::LuminosityBlockCache<externalgen::LumiCache>,
0164 edm::LuminosityBlockSummaryCache<GenLumiInfoProduct>,
0165 edm::BeginLuminosityBlockProducer,
0166 edm::EndLuminosityBlockProducer> {
0167 public:
0168 ExternalGeneratorFilter(edm::ParameterSet const&);
0169
0170 std::unique_ptr<externalgen::StreamCache> beginStream(edm::StreamID) const final;
0171 bool filter(edm::StreamID, edm::Event&, edm::EventSetup const&) const final;
0172
0173 std::shared_ptr<externalgen::RunCache> globalBeginRun(edm::Run const&, edm::EventSetup const&) const final;
0174 void streamBeginRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const final;
0175 void streamEndRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const final;
0176 void globalEndRun(edm::Run const&, edm::EventSetup const&) const final {}
0177 void globalEndRunProduce(edm::Run&, edm::EventSetup const&) const final;
0178
0179 void globalBeginLuminosityBlockProduce(edm::LuminosityBlock&, edm::EventSetup const&) const final;
0180 std::shared_ptr<externalgen::LumiCache> globalBeginLuminosityBlock(edm::LuminosityBlock const&,
0181 edm::EventSetup const&) const final;
0182 std::shared_ptr<GenLumiInfoProduct> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const&,
0183 edm::EventSetup const&) const final;
0184 void streamBeginLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const final;
0185 void streamEndLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const final;
0186 void streamEndLuminosityBlockSummary(edm::StreamID,
0187 edm::LuminosityBlock const&,
0188 edm::EventSetup const&,
0189 GenLumiInfoProduct*) const final;
0190 void globalEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) const final {}
0191 void globalEndLuminosityBlockSummary(edm::LuminosityBlock const&,
0192 edm::EventSetup const&,
0193 GenLumiInfoProduct*) const final {}
0194 void globalEndLuminosityBlockProduce(edm::LuminosityBlock&,
0195 edm::EventSetup const&,
0196 GenLumiInfoProduct const*) const final;
0197
0198 private:
0199 edm::EDPutTokenT<edm::HepMCProduct> const hepMCToken_;
0200 edm::EDPutTokenT<GenEventInfoProduct> const genEventToken_;
0201 edm::EDPutTokenT<GenRunInfoProduct> const runInfoToken_;
0202 edm::EDPutTokenT<GenLumiInfoHeader> const lumiHeaderToken_;
0203 edm::EDPutTokenT<GenLumiInfoProduct> const lumiInfoToken_;
0204
0205 std::string const config_;
0206 bool const verbose_;
0207 unsigned int waitTime_;
0208 std::string const extraConfig_;
0209
0210
0211
0212 CMS_THREAD_SAFE mutable externalgen::StreamCache* stream0Cache_ = nullptr;
0213
0214
0215 mutable std::atomic<externalgen::StreamCache*> availableForBeginLumi_;
0216
0217
0218 mutable std::atomic<unsigned int> lastLumiIndex_ = 0;
0219 };
0220
0221 ExternalGeneratorFilter::ExternalGeneratorFilter(edm::ParameterSet const& iPSet)
0222 : hepMCToken_{produces<edm::HepMCProduct>("unsmeared")},
0223 genEventToken_{produces<GenEventInfoProduct>()},
0224 runInfoToken_{produces<GenRunInfoProduct, edm::Transition::EndRun>()},
0225 lumiHeaderToken_{produces<GenLumiInfoHeader, edm::Transition::BeginLuminosityBlock>()},
0226 lumiInfoToken_{produces<GenLumiInfoProduct, edm::Transition::EndLuminosityBlock>()},
0227 config_{iPSet.getUntrackedParameter<std::string>("@python_config")},
0228 verbose_{iPSet.getUntrackedParameter<bool>("_external_process_verbose_")},
0229 waitTime_{iPSet.getUntrackedParameter<unsigned int>("_external_process_waitTime_")},
0230 extraConfig_{iPSet.getUntrackedParameter<std::string>("_external_process_extraConfig_")} {}
0231
0232 std::unique_ptr<externalgen::StreamCache> ExternalGeneratorFilter::beginStream(edm::StreamID iID) const {
0233 auto const label = moduleDescription().moduleLabel();
0234
0235 using namespace std::string_literals;
0236
0237 std::string config = R"_(from FWCore.TestProcessor.TestProcess import *
0238 process = TestProcess()
0239 )_";
0240 config += "process."s + label + "=" + config_ + "\n";
0241 config += "process.moduleToTest(process."s + label + ")\n";
0242 config += R"_(
0243 process.add_(cms.Service("InitRootHandlers", AbortOnSignal=cms.untracked.bool(False)))
0244 )_";
0245 if (not extraConfig_.empty()) {
0246 config += "\n";
0247 config += extraConfig_;
0248 }
0249
0250 auto cache = std::make_unique<externalgen::StreamCache>(config, iID.value(), verbose_, waitTime_);
0251 if (iID.value() == 0) {
0252 stream0Cache_ = cache.get();
0253
0254 availableForBeginLumi_ = stream0Cache_;
0255 }
0256
0257 return cache;
0258 }
0259
0260 bool ExternalGeneratorFilter::filter(edm::StreamID iID, edm::Event& iEvent, edm::EventSetup const&) const {
0261 auto value = streamCache(iID)->produce(iID, iEvent.id().event());
0262
0263 edm::Service<edm::RandomNumberGenerator> gen;
0264 auto& engine = gen->getEngine(iID);
0265
0266
0267
0268 engine.get(value.randomState_.state_);
0269
0270 iEvent.emplace(hepMCToken_, std::move(value.hepmc_));
0271 iEvent.emplace(genEventToken_, std::move(value.eventInfo_));
0272 return value.keepEvent_;
0273 }
0274
0275 std::shared_ptr<externalgen::RunCache> ExternalGeneratorFilter::globalBeginRun(edm::Run const&,
0276 edm::EventSetup const&) const {
0277 return std::make_shared<externalgen::RunCache>();
0278 }
0279
0280 void ExternalGeneratorFilter::streamBeginRun(edm::StreamID iID, edm::Run const& iRun, edm::EventSetup const&) const {}
0281 void ExternalGeneratorFilter::streamEndRun(edm::StreamID iID, edm::Run const& iRun, edm::EventSetup const&) const {
0282 if (iID.value() == 0) {
0283 runCache(iRun.index())->runInfo_ = *streamCache(iID)->endRunProduce(iRun.run());
0284 } else {
0285 (void)streamCache(iID)->endRunProduce(iRun.run());
0286 }
0287 }
0288 void ExternalGeneratorFilter::globalEndRunProduce(edm::Run& iRun, edm::EventSetup const&) const {
0289 iRun.emplace(runInfoToken_, std::move(runCache(iRun.index())->runInfo_));
0290 }
0291
0292 void ExternalGeneratorFilter::globalBeginLuminosityBlockProduce(edm::LuminosityBlock& iLuminosityBlock,
0293 edm::EventSetup const&) const {
0294 while (not availableForBeginLumi_.load()) {
0295 }
0296
0297 auto v = availableForBeginLumi_.load()->beginLumiProduce(
0298 iLuminosityBlock.luminosityBlock(), luminosityBlockCache(iLuminosityBlock.index())->randomState_);
0299
0300 edm::Service<edm::RandomNumberGenerator> gen;
0301 auto& engine = gen->getEngine(iLuminosityBlock.index());
0302 engine.get(v.randomState_.state_);
0303
0304 iLuminosityBlock.emplace(lumiHeaderToken_, std::move(v.header_));
0305
0306 lastLumiIndex_.store(iLuminosityBlock.index());
0307 }
0308
0309 std::shared_ptr<externalgen::LumiCache> ExternalGeneratorFilter::globalBeginLuminosityBlock(
0310 edm::LuminosityBlock const& iLumi, edm::EventSetup const&) const {
0311 edm::Service<edm::RandomNumberGenerator> gen;
0312 auto& engine = gen->getEngine(iLumi.index());
0313 auto s = engine.put();
0314 return std::make_shared<externalgen::LumiCache>(s, engine.getSeed());
0315 }
0316
0317 std::shared_ptr<GenLumiInfoProduct> ExternalGeneratorFilter::globalBeginLuminosityBlockSummary(
0318 edm::LuminosityBlock const&, edm::EventSetup const&) const {
0319 return std::make_shared<GenLumiInfoProduct>();
0320 }
0321
0322 void ExternalGeneratorFilter::streamBeginLuminosityBlock(edm::StreamID iID,
0323 edm::LuminosityBlock const& iLuminosityBlock,
0324 edm::EventSetup const&) const {
0325 auto cache = streamCache(iID);
0326 if (cache != availableForBeginLumi_.load()) {
0327 (void)cache->beginLumiProduce(iLuminosityBlock.run(), luminosityBlockCache(iLuminosityBlock.index())->randomState_);
0328 } else {
0329 availableForBeginLumi_ = nullptr;
0330 }
0331 }
0332
0333 void ExternalGeneratorFilter::streamEndLuminosityBlock(edm::StreamID iID,
0334 edm::LuminosityBlock const& iLuminosityBlock,
0335 edm::EventSetup const&) const {}
0336
0337 void ExternalGeneratorFilter::streamEndLuminosityBlockSummary(edm::StreamID iID,
0338 edm::LuminosityBlock const& iLuminosityBlock,
0339 edm::EventSetup const&,
0340 GenLumiInfoProduct* iProduct) const {
0341 iProduct->mergeProduct(*streamCache(iID)->endLumiProduce(iLuminosityBlock.run()));
0342
0343 if (lastLumiIndex_ == iLuminosityBlock.index()) {
0344 externalgen::StreamCache* expected = nullptr;
0345
0346 availableForBeginLumi_.compare_exchange_strong(expected, streamCache(iID));
0347 }
0348 }
0349
0350 void ExternalGeneratorFilter::globalEndLuminosityBlockProduce(edm::LuminosityBlock& iLuminosityBlock,
0351 edm::EventSetup const&,
0352 GenLumiInfoProduct const* iProduct) const {
0353 iLuminosityBlock.emplace(lumiInfoToken_, *iProduct);
0354 }
0355
0356 DEFINE_FWK_MODULE(ExternalGeneratorFilter);