File indexing completed on 2022-01-13 01:43:41
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020 #include "oneapi/tbb/task_arena.h"
0021 #include "oneapi/tbb/task_group.h"
0022 #include <cstdio>
0023 #include <cstdlib>
0024 #include <dirent.h>
0025 #include <fcntl.h>
0026 #include <filesystem>
0027 #include <fstream>
0028 #include <memory>
0029 #include <string>
0030 #include <sys/resource.h>
0031 #include <sys/time.h>
0032 #include <sys/wait.h>
0033 #include <system_error>
0034 #include <unistd.h>
0035 #include <vector>
0036
0037 #include "boost/ptr_container/ptr_deque.hpp"
0038
0039
0040 #include "FWCore/Framework/interface/Frameworkfwd.h"
0041 #include "FWCore/Framework/interface/one/EDProducer.h"
0042 #include "FWCore/Framework/interface/LuminosityBlock.h"
0043
0044 #include "FWCore/Framework/interface/Run.h"
0045 #include "FWCore/Framework/interface/Event.h"
0046 #include "FWCore/Framework/interface/MakerMacros.h"
0047
0048 #include "FWCore/ParameterSet/interface/FileInPath.h"
0049 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0050
0051 #include "SimDataFormats/GeneratorProducts/interface/LesHouches.h"
0052 #include "SimDataFormats/GeneratorProducts/interface/LHERunInfoProduct.h"
0053 #include "SimDataFormats/GeneratorProducts/interface/LHEEventProduct.h"
0054 #include "SimDataFormats/GeneratorProducts/interface/LHEXMLStringProduct.h"
0055
0056 #include "GeneratorInterface/LHEInterface/interface/LHERunInfo.h"
0057 #include "GeneratorInterface/LHEInterface/interface/LHEEvent.h"
0058 #include "GeneratorInterface/LHEInterface/interface/LHEReader.h"
0059
0060 #include "FWCore/ServiceRegistry/interface/Service.h"
0061 #include "FWCore/Utilities/interface/RandomNumberGenerator.h"
0062
0063 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0064
0065
0066
0067
0068
0069 class ExternalLHEProducer : public edm::one::EDProducer<edm::BeginRunProducer, edm::one::WatchRuns> {
0070 public:
0071 explicit ExternalLHEProducer(const edm::ParameterSet& iConfig);
0072
0073 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0074
0075 private:
0076 void produce(edm::Event&, const edm::EventSetup&) override;
0077 void beginRunProduce(edm::Run& run, edm::EventSetup const& es) override;
0078 void beginRun(edm::Run const&, edm::EventSetup const&) override;
0079 void endRun(edm::Run const&, edm::EventSetup const&) override;
0080 void preallocThreads(unsigned int) override;
0081
0082 std::vector<std::string> makeArgs(uint32_t nEvents, unsigned int nThreads, std::uint32_t seed) const;
0083 int closeDescriptors(int preserve) const;
0084 void executeScript(std::vector<std::string> const& args, int id, bool isPost) const;
0085
0086 void nextEvent();
0087 std::unique_ptr<LHERunInfoProduct> generateRunInfo(std::vector<std::string> const& files) const;
0088
0089
0090 std::string scriptName_;
0091 std::string outputFile_;
0092 const std::vector<std::string> args_;
0093 uint32_t npars_;
0094 uint32_t nEvents_;
0095 bool storeXML_;
0096 unsigned int nThreads_{1};
0097 std::string outputContents_;
0098 bool generateConcurrently_{false};
0099 const std::vector<std::string> postGenerationCommand_;
0100
0101
0102 std::map<unsigned, std::pair<unsigned, unsigned>> nPartonMapping_{};
0103
0104 std::unique_ptr<lhef::LHEReader> reader_;
0105 std::shared_ptr<lhef::LHEEvent> partonLevel_;
0106 bool wasMerged_;
0107
0108 edm::EDPutTokenT<LHEXMLStringProduct> xmlPutToken_;
0109 edm::EDPutTokenT<LHEEventProduct> eventPutToken_;
0110 edm::EDPutTokenT<LHERunInfoProduct> beginRunPutToken_;
0111 class FileCloseSentry {
0112 public:
0113 explicit FileCloseSentry(int fd) : fd_(fd){};
0114
0115 ~FileCloseSentry() { close(fd_); }
0116
0117
0118 FileCloseSentry(const FileCloseSentry&) = delete;
0119 FileCloseSentry& operator=(const FileCloseSentry&) = delete;
0120
0121 private:
0122 int fd_;
0123 };
0124 };
0125
0126
0127
0128
0129 ExternalLHEProducer::ExternalLHEProducer(const edm::ParameterSet& iConfig)
0130 : scriptName_((iConfig.getParameter<edm::FileInPath>("scriptName")).fullPath()),
0131 outputFile_(iConfig.getParameter<std::string>("outputFile")),
0132 args_(iConfig.getParameter<std::vector<std::string>>("args")),
0133 npars_(iConfig.getParameter<uint32_t>("numberOfParameters")),
0134 nEvents_(iConfig.getUntrackedParameter<uint32_t>("nEvents")),
0135 storeXML_(iConfig.getUntrackedParameter<bool>("storeXML")),
0136 generateConcurrently_(iConfig.getUntrackedParameter<bool>("generateConcurrently")),
0137 postGenerationCommand_(iConfig.getUntrackedParameter<std::vector<std::string>>("postGenerationCommand")) {
0138 if (npars_ != args_.size())
0139 throw cms::Exception("ExternalLHEProducer")
0140 << "Problem with configuration: " << args_.size() << " script arguments given, expected " << npars_;
0141
0142 if (iConfig.exists("nPartonMapping")) {
0143 auto& processMap(iConfig.getParameterSetVector("nPartonMapping"));
0144 for (auto& cfg : processMap) {
0145 unsigned processId(cfg.getParameter<unsigned>("idprup"));
0146
0147 auto orderStr(cfg.getParameter<std::string>("order"));
0148 unsigned order(0);
0149 if (orderStr == "LO")
0150 order = 0;
0151 else if (orderStr == "NLO")
0152 order = 1;
0153 else
0154 throw cms::Exception("ExternalLHEProducer")
0155 << "Invalid order specification for process " << processId << ": " << orderStr;
0156
0157 unsigned np(cfg.getParameter<unsigned>("np"));
0158
0159 nPartonMapping_.emplace(processId, std::make_pair(order, np));
0160 }
0161 }
0162
0163 xmlPutToken_ = produces<LHEXMLStringProduct, edm::Transition::BeginRun>("LHEScriptOutput");
0164
0165 eventPutToken_ = produces<LHEEventProduct>();
0166 beginRunPutToken_ = produces<LHERunInfoProduct, edm::Transition::BeginRun>();
0167 }
0168
0169
0170
0171
0172
0173
0174 void ExternalLHEProducer::preallocThreads(unsigned int iThreads) { nThreads_ = iThreads; }
0175
0176
0177 void ExternalLHEProducer::produce(edm::Event& iEvent, const edm::EventSetup& iSetup) {
0178 nextEvent();
0179 if (!partonLevel_) {
0180 throw edm::Exception(edm::errors::EventGenerationFailure)
0181 << "No lhe event found in ExternalLHEProducer::produce(). "
0182 << "The likely cause is that the lhe file contains fewer events than were requested, which is possible "
0183 << "in case of phase space integration or uneweighting efficiency problems.";
0184 }
0185
0186 std::unique_ptr<LHEEventProduct> product(
0187 new LHEEventProduct(*partonLevel_->getHEPEUP(), partonLevel_->originalXWGTUP()));
0188 if (partonLevel_->getPDF()) {
0189 product->setPDF(*partonLevel_->getPDF());
0190 }
0191 std::for_each(partonLevel_->weights().begin(),
0192 partonLevel_->weights().end(),
0193 std::bind(&LHEEventProduct::addWeight, product.get(), std::placeholders::_1));
0194 product->setScales(partonLevel_->scales());
0195 if (nPartonMapping_.empty()) {
0196 product->setNpLO(partonLevel_->npLO());
0197 product->setNpNLO(partonLevel_->npNLO());
0198 } else {
0199
0200 unsigned processId(partonLevel_->getHEPEUP()->IDPRUP);
0201 unsigned order(0);
0202 unsigned np(0);
0203 try {
0204 auto procDef(nPartonMapping_.at(processId));
0205 order = procDef.first;
0206 np = procDef.second;
0207 } catch (std::out_of_range&) {
0208 throw cms::Exception("ExternalLHEProducer")
0209 << "Unexpected IDPRUP encountered: " << partonLevel_->getHEPEUP()->IDPRUP;
0210 }
0211
0212 switch (order) {
0213 case 0:
0214 product->setNpLO(np);
0215 product->setNpNLO(-1);
0216 break;
0217 case 1:
0218 product->setNpLO(-1);
0219 product->setNpNLO(np);
0220 break;
0221 default:
0222 break;
0223 }
0224 }
0225
0226 std::for_each(partonLevel_->getComments().begin(),
0227 partonLevel_->getComments().end(),
0228 std::bind(&LHEEventProduct::addComment, product.get(), std::placeholders::_1));
0229
0230 iEvent.put(eventPutToken_, std::move(product));
0231
0232 partonLevel_.reset();
0233 return;
0234 }
0235
0236
0237 void ExternalLHEProducer::beginRunProduce(edm::Run& run, edm::EventSetup const& es) {
0238
0239
0240
0241
0242 edm::Service<edm::RandomNumberGenerator> rng;
0243
0244 if (!rng.isAvailable()) {
0245 throw cms::Exception("Configuration")
0246 << "The ExternalLHEProducer module requires the RandomNumberGeneratorService\n"
0247 "which is not present in the configuration file. You must add the service\n"
0248 "in the configuration file if you want to run ExternalLHEProducer";
0249 }
0250
0251 std::vector<std::string> infiles;
0252 auto const seed = rng->mySeed();
0253 if (generateConcurrently_) {
0254 infiles.resize(nThreads_);
0255 auto const nEventsAve = nEvents_ / nThreads_;
0256 unsigned int const overflow = nThreads_ - (nEvents_ % nThreads_);
0257 std::exception_ptr except;
0258 std::atomic<char> exceptSet{0};
0259
0260 tbb::this_task_arena::isolate([this, &except, &infiles, &exceptSet, nEventsAve, overflow, seed]() {
0261 tbb::task_group group;
0262 for (unsigned int t = 0; t < nThreads_; ++t) {
0263 uint32_t nEvents = nEventsAve;
0264 if (nEvents_ % nThreads_ != 0 and t >= overflow) {
0265 nEvents += 1;
0266 }
0267 group.run([t, this, &infiles, seed, nEvents, &except, &exceptSet]() {
0268 CMS_SA_ALLOW try {
0269 using namespace std::filesystem;
0270 using namespace std::string_literals;
0271 auto out = path("thread"s + std::to_string(t)) / path(outputFile_);
0272 infiles[t] = out.native();
0273 executeScript(makeArgs(nEvents, 1, seed + t), t, false);
0274 } catch (...) {
0275 char expected = 0;
0276 if (exceptSet.compare_exchange_strong(expected, 1)) {
0277 except = std::current_exception();
0278 exceptSet.store(2);
0279 }
0280 }
0281 });
0282 }
0283 group.wait();
0284 });
0285 if (exceptSet != 0) {
0286 std::rethrow_exception(except);
0287 }
0288 } else {
0289 infiles = std::vector<std::string>(1, outputFile_);
0290 executeScript(makeArgs(nEvents_, nThreads_, seed), 0, false);
0291 }
0292
0293
0294 if (!postGenerationCommand_.empty()) {
0295 std::vector<std::string> postcmd = postGenerationCommand_;
0296 try {
0297 postcmd[0] = edm::FileInPath(postcmd[0]).fullPath();
0298 } catch (const edm::Exception& e) {
0299 edm::LogWarning("ExternalLHEProducer") << postcmd[0] << " is not a relative path. Run it as a shell command.";
0300 }
0301 executeScript(postcmd, 0, true);
0302 }
0303
0304
0305 std::unique_ptr<LHEXMLStringProduct> p(new LHEXMLStringProduct);
0306
0307
0308 if (storeXML_) {
0309 std::string file;
0310 if (generateConcurrently_) {
0311 using namespace std::filesystem;
0312 file = (path("thread0") / path(outputFile_)).native();
0313 } else {
0314 file = outputFile_;
0315 }
0316 std::ifstream instream(file);
0317 if (!instream) {
0318 throw cms::Exception("OutputOpenError") << "Unable to open script output file " << outputFile_ << ".";
0319 }
0320 instream.seekg(0, instream.end);
0321 int insize = instream.tellg();
0322 instream.seekg(0, instream.beg);
0323 p->fillCompressedContent(instream, 0.25 * insize);
0324 instream.close();
0325 }
0326 run.put(xmlPutToken_, std::move(p));
0327
0328
0329 auto runInfo = generateRunInfo(infiles);
0330 if (runInfo) {
0331 run.put(beginRunPutToken_, std::move(runInfo));
0332 }
0333
0334
0335
0336 unsigned int skip = 0;
0337 reader_ = std::make_unique<lhef::LHEReader>(infiles, skip);
0338
0339 nextEvent();
0340 }
0341
0342 void ExternalLHEProducer::beginRun(edm::Run const& run, edm::EventSetup const& es) {}
0343
0344 void ExternalLHEProducer::endRun(edm::Run const& run, edm::EventSetup const& es) {
0345 nextEvent();
0346 if (partonLevel_) {
0347
0348 if (std::getenv("VALIDATION_RUN") != nullptr) {
0349 edm::LogWarning("ExternalLHEProducer")
0350 << "Event loop is over, but there are still lhe events to process, ignoring...";
0351 } else {
0352 throw edm::Exception(edm::errors::EventGenerationFailure)
0353 << "Error in ExternalLHEProducer::endRunProduce(). "
0354 << "Event loop is over, but there are still lhe events to process."
0355 << "This could happen if lhe file contains more events than requested. This is never expected to happen.";
0356 }
0357 }
0358
0359 reader_.reset();
0360 if (generateConcurrently_) {
0361 for (unsigned int t = 0; t < nThreads_; ++t) {
0362 using namespace std::filesystem;
0363 using namespace std::string_literals;
0364 auto out = path("thread"s + std::to_string(t)) / path(outputFile_);
0365 if (unlink(out.c_str())) {
0366 throw cms::Exception("OutputDeleteError") << "Unable to delete original script output file " << out
0367 << " (errno=" << errno << ", " << strerror(errno) << ").";
0368 }
0369 }
0370 } else {
0371 if (unlink(outputFile_.c_str())) {
0372 throw cms::Exception("OutputDeleteError") << "Unable to delete original script output file " << outputFile_
0373 << " (errno=" << errno << ", " << strerror(errno) << ").";
0374 }
0375 }
0376 }
0377
0378 std::vector<std::string> ExternalLHEProducer::makeArgs(uint32_t nEvents,
0379 unsigned int nThreads,
0380 std::uint32_t seed) const {
0381 std::vector<std::string> args;
0382 args.reserve(3 + args_.size());
0383
0384 args.push_back(args_.front());
0385 args.push_back(std::to_string(nEvents));
0386
0387 args.push_back(std::to_string(seed));
0388
0389 args.push_back(std::to_string(nThreads));
0390 std::copy(args_.begin() + 1, args_.end(), std::back_inserter(args));
0391
0392 for (unsigned int iArg = 0; iArg < args.size(); iArg++) {
0393 LogDebug("LHEInputArgs") << "arg [" << iArg << "] = " << args[iArg];
0394 }
0395
0396 return args;
0397 }
0398
0399
0400 int ExternalLHEProducer::closeDescriptors(int preserve) const {
0401 int maxfd = 1024;
0402 int fd;
0403 #ifdef __linux__
0404 DIR* dir;
0405 struct dirent* dp;
0406 maxfd = preserve;
0407 if ((dir = opendir("/proc/self/fd"))) {
0408 errno = 0;
0409 while ((dp = readdir(dir)) != nullptr) {
0410 if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) {
0411 continue;
0412 }
0413 if (sscanf(dp->d_name, "%d", &fd) != 1) {
0414
0415 return -1;
0416 }
0417 if (fd > maxfd) {
0418 maxfd = fd;
0419 }
0420 }
0421 if (errno) {
0422
0423 return errno;
0424 }
0425 closedir(dir);
0426 }
0427 #endif
0428
0429 for (fd = 3; fd < maxfd + 1; fd++) {
0430 if (fd != preserve)
0431 close(fd);
0432 }
0433 return 0;
0434 }
0435
0436
0437 void ExternalLHEProducer::executeScript(std::vector<std::string> const& args, int id, bool isPost) const {
0438
0439
0440 int rc = 0, rc2 = 0;
0441 int filedes[2], fd_flags;
0442
0443 if (pipe(filedes)) {
0444 throw cms::Exception("Unable to create a new pipe");
0445 }
0446 FileCloseSentry sentry1(filedes[0]), sentry2(filedes[1]);
0447
0448 if ((fd_flags = fcntl(filedes[1], F_GETFD, NULL)) == -1) {
0449 throw cms::Exception("ExternalLHEProducer")
0450 << "Failed to get pipe file descriptor flags (errno=" << rc << ", " << strerror(rc) << ")";
0451 }
0452 if (fcntl(filedes[1], F_SETFD, fd_flags | FD_CLOEXEC) == -1) {
0453 throw cms::Exception("ExternalLHEProducer")
0454 << "Failed to set pipe file descriptor flags (errno=" << rc << ", " << strerror(rc) << ")";
0455 }
0456
0457 unsigned int argc_pre = 0;
0458
0459 if (!isPost) {
0460 argc_pre = 1;
0461 }
0462 unsigned int argc = argc_pre + args.size();
0463
0464 char** argv = new char*[argc + 1];
0465 if (!isPost) {
0466 argv[0] = strdup(scriptName_.c_str());
0467 }
0468 for (unsigned int i = 0; i < args.size(); i++) {
0469 argv[argc_pre + i] = strdup(args[i].c_str());
0470 }
0471 argv[argc] = nullptr;
0472
0473 pid_t pid = fork();
0474 if (pid == 0) {
0475
0476 if (!(rc = closeDescriptors(filedes[1]))) {
0477 if (!isPost && generateConcurrently_) {
0478 using namespace std::filesystem;
0479 using namespace std::string_literals;
0480 std::error_code ec;
0481 auto newDir = path("thread"s + std::to_string(id));
0482 create_directory(newDir, ec);
0483 current_path(newDir, ec);
0484 }
0485 execvp(argv[0], argv);
0486 rc = errno;
0487 }
0488 while ((write(filedes[1], &rc, sizeof(int)) == -1) && (errno == EINTR)) {
0489 }
0490 _exit(1);
0491 }
0492
0493
0494 for (unsigned int i = 0; i < args.size() + 1; i++) {
0495 free(argv[i]);
0496 }
0497 delete[] argv;
0498
0499 if (pid == -1) {
0500 throw cms::Exception("ForkException")
0501 << "Unable to fork a child (errno=" << errno << ", " << strerror(errno) << ")";
0502 }
0503
0504 close(filedes[1]);
0505
0506 while (((rc2 = read(filedes[0], &rc, sizeof(int))) == -1) && (errno == EINTR)) {
0507 rc2 = 0;
0508 }
0509 if ((rc2 == sizeof(int)) && rc) {
0510 throw cms::Exception("ExternalLHEProducer")
0511 << "Failed to execute script (errno=" << rc << ", " << strerror(rc) << ")";
0512 }
0513 close(filedes[0]);
0514
0515 int status = 0;
0516 errno = 0;
0517 do {
0518 if (waitpid(pid, &status, 0) < 0) {
0519 if (errno == EINTR) {
0520 continue;
0521 } else {
0522 throw cms::Exception("ExternalLHEProducer")
0523 << "Failed to read child status (errno=" << errno << ", " << strerror(errno) << ")";
0524 }
0525 }
0526 if (WIFSIGNALED(status)) {
0527 throw cms::Exception("ExternalLHEProducer") << "Child exited due to signal " << WTERMSIG(status) << ".";
0528 }
0529 if (WIFEXITED(status)) {
0530 rc = WEXITSTATUS(status);
0531 break;
0532 }
0533 } while (true);
0534 if (rc) {
0535 throw cms::Exception("ExternalLHEProducer") << "Child failed with exit code " << rc << ".";
0536 }
0537 }
0538
0539
0540 void ExternalLHEProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0541
0542
0543 edm::ParameterSetDescription desc;
0544 desc.setComment("Executes an external script and places its output file into an EDM collection");
0545
0546 edm::FileInPath thePath;
0547 desc.add<edm::FileInPath>("scriptName", thePath);
0548 desc.add<std::string>("outputFile", "myoutput");
0549 desc.add<std::vector<std::string>>("args");
0550 desc.add<uint32_t>("numberOfParameters");
0551 desc.addUntracked<uint32_t>("nEvents");
0552 desc.addUntracked<bool>("storeXML", false);
0553 desc.addUntracked<bool>("generateConcurrently", false)
0554 ->setComment("If true, run the script concurrently in separate processes.");
0555 desc.addUntracked<std::vector<std::string>>("postGenerationCommand", std::vector<std::string>())
0556 ->setComment(
0557 "Command to run after the generation script has completed. The first argument can be a relative path.");
0558
0559 edm::ParameterSetDescription nPartonMappingDesc;
0560 nPartonMappingDesc.add<unsigned>("idprup");
0561 nPartonMappingDesc.add<std::string>("order");
0562 nPartonMappingDesc.add<unsigned>("np");
0563 desc.addVPSetOptional("nPartonMapping", nPartonMappingDesc);
0564
0565 descriptions.addDefault(desc);
0566 }
0567
0568 std::unique_ptr<LHERunInfoProduct> ExternalLHEProducer::generateRunInfo(std::vector<std::string> const& iFiles) const {
0569 std::unique_ptr<LHERunInfoProduct> retValue;
0570
0571 for (auto const& file : iFiles) {
0572 unsigned int skip = 0;
0573 std::vector<std::string> infiles(1, file);
0574 auto reader = std::make_unique<lhef::LHEReader>(infiles, skip);
0575 auto parton = reader->next();
0576 if (!parton) {
0577 break;
0578 }
0579 auto runInfo = parton->getRunInfo();
0580 LHERunInfoProduct product(*runInfo->getHEPRUP());
0581
0582 std::for_each(runInfo->getHeaders().begin(),
0583 runInfo->getHeaders().end(),
0584 std::bind(&LHERunInfoProduct::addHeader, &product, std::placeholders::_1));
0585 std::for_each(runInfo->getComments().begin(),
0586 runInfo->getComments().end(),
0587 std::bind(&LHERunInfoProduct::addComment, &product, std::placeholders::_1));
0588 if (not retValue) {
0589 retValue = std::make_unique<LHERunInfoProduct>(std::move(product));
0590 } else {
0591 retValue->mergeProduct(product);
0592 }
0593 }
0594
0595 return retValue;
0596 }
0597
0598 void ExternalLHEProducer::nextEvent() {
0599 if (partonLevel_)
0600 return;
0601
0602 if (not reader_) {
0603 return;
0604 }
0605
0606 partonLevel_ = reader_->next();
0607 if (!partonLevel_) {
0608
0609 bool newFileOpened;
0610 do {
0611 newFileOpened = false;
0612 partonLevel_ = reader_->next(&newFileOpened);
0613 } while (newFileOpened && !partonLevel_);
0614 }
0615 }
0616
0617
0618 DEFINE_FWK_MODULE(ExternalLHEProducer);