File indexing completed on 2024-04-06 12:13:49
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018
0019
0020 #include "oneapi/tbb/task_arena.h"
0021 #include "oneapi/tbb/task_group.h"
0022 #include <cstdio>
0023 #include <cstdlib>
0024 #include <dirent.h>
0025 #include <fcntl.h>
0026 #include <filesystem>
0027 #include <fstream>
0028 #include <memory>
0029 #include <string>
0030 #include <sys/resource.h>
0031 #include <sys/time.h>
0032 #include <sys/wait.h>
0033 #include <system_error>
0034 #include <unistd.h>
0035 #include <vector>
0036
0037 #include "boost/ptr_container/ptr_deque.hpp"
0038
0039
0040 #include "FWCore/Framework/interface/Frameworkfwd.h"
0041 #include "FWCore/Framework/interface/one/EDProducer.h"
0042 #include "FWCore/Framework/interface/LuminosityBlock.h"
0043
0044 #include "FWCore/Framework/interface/Run.h"
0045 #include "FWCore/Framework/interface/Event.h"
0046 #include "FWCore/Framework/interface/MakerMacros.h"
0047
0048 #include "FWCore/ParameterSet/interface/FileInPath.h"
0049 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0050
0051 #include "SimDataFormats/GeneratorProducts/interface/LesHouches.h"
0052 #include "SimDataFormats/GeneratorProducts/interface/LHERunInfoProduct.h"
0053 #include "SimDataFormats/GeneratorProducts/interface/LHEEventProduct.h"
0054 #include "SimDataFormats/GeneratorProducts/interface/LHEXMLStringProduct.h"
0055
0056 #include "GeneratorInterface/LHEInterface/interface/LHERunInfo.h"
0057 #include "GeneratorInterface/LHEInterface/interface/LHEEvent.h"
0058 #include "GeneratorInterface/LHEInterface/interface/LHEReader.h"
0059
0060 #include "FWCore/ServiceRegistry/interface/Service.h"
0061 #include "FWCore/Utilities/interface/RandomNumberGenerator.h"
0062
0063 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0064
0065
0066
0067
0068
0069 class ExternalLHEProducer : public edm::one::EDProducer<edm::BeginRunProducer, edm::one::WatchRuns> {
0070 public:
0071 explicit ExternalLHEProducer(const edm::ParameterSet& iConfig);
0072
0073 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0074
0075 private:
0076 void produce(edm::Event&, const edm::EventSetup&) override;
0077 void beginRunProduce(edm::Run& run, edm::EventSetup const& es) override;
0078 void beginRun(edm::Run const&, edm::EventSetup const&) override;
0079 void endRun(edm::Run const&, edm::EventSetup const&) override;
0080 void preallocThreads(unsigned int) override;
0081
0082 std::vector<std::string> makeArgs(uint32_t nEvents, unsigned int nThreads, std::uint32_t seed) const;
0083 int closeDescriptors(int preserve) const;
0084 void executeScript(std::vector<std::string> const& args, int id, bool isPost) const;
0085
0086 void nextEvent();
0087 std::unique_ptr<LHERunInfoProduct> generateRunInfo(std::vector<std::string> const& files) const;
0088
0089
0090 std::string scriptName_;
0091 std::string outputFile_;
0092 const std::vector<std::string> args_;
0093 uint32_t npars_;
0094 uint32_t nEvents_;
0095 bool storeXML_;
0096 unsigned int nThreads_{1};
0097 std::string outputContents_;
0098 bool generateConcurrently_{false};
0099 const std::vector<std::string> postGenerationCommand_;
0100
0101
0102 std::map<unsigned, std::pair<unsigned, unsigned>> nPartonMapping_{};
0103
0104 std::unique_ptr<lhef::LHEReader> reader_;
0105 std::shared_ptr<lhef::LHEEvent> partonLevel_;
0106 bool wasMerged_;
0107
0108 edm::EDPutTokenT<LHEXMLStringProduct> xmlPutToken_;
0109 edm::EDPutTokenT<LHEEventProduct> eventPutToken_;
0110 edm::EDPutTokenT<LHERunInfoProduct> beginRunPutToken_;
0111 class FileCloseSentry {
0112 public:
0113 explicit FileCloseSentry(int fd) : fd_(fd){};
0114
0115 ~FileCloseSentry() { close(fd_); }
0116
0117
0118 FileCloseSentry(const FileCloseSentry&) = delete;
0119 FileCloseSentry& operator=(const FileCloseSentry&) = delete;
0120
0121 private:
0122 int fd_;
0123 };
0124 };
0125
0126
0127
0128
0129 ExternalLHEProducer::ExternalLHEProducer(const edm::ParameterSet& iConfig)
0130 : scriptName_((iConfig.getParameter<edm::FileInPath>("scriptName")).fullPath()),
0131 outputFile_(iConfig.getParameter<std::string>("outputFile")),
0132 args_(iConfig.getParameter<std::vector<std::string>>("args")),
0133 npars_(iConfig.getParameter<uint32_t>("numberOfParameters")),
0134 nEvents_(iConfig.getUntrackedParameter<uint32_t>("nEvents")),
0135 storeXML_(iConfig.getUntrackedParameter<bool>("storeXML")),
0136 generateConcurrently_(iConfig.getUntrackedParameter<bool>("generateConcurrently")),
0137 postGenerationCommand_(iConfig.getUntrackedParameter<std::vector<std::string>>("postGenerationCommand")) {
0138 if (npars_ != args_.size())
0139 throw cms::Exception("ExternalLHEProducer")
0140 << "Problem with configuration: " << args_.size() << " script arguments given, expected " << npars_;
0141
0142 if (iConfig.exists("nPartonMapping")) {
0143 auto& processMap(iConfig.getParameterSetVector("nPartonMapping"));
0144 for (auto& cfg : processMap) {
0145 unsigned processId(cfg.getParameter<unsigned>("idprup"));
0146
0147 auto orderStr(cfg.getParameter<std::string>("order"));
0148 unsigned order(0);
0149 if (orderStr == "LO")
0150 order = 0;
0151 else if (orderStr == "NLO")
0152 order = 1;
0153 else
0154 throw cms::Exception("ExternalLHEProducer")
0155 << "Invalid order specification for process " << processId << ": " << orderStr;
0156
0157 unsigned np(cfg.getParameter<unsigned>("np"));
0158
0159 nPartonMapping_.emplace(processId, std::make_pair(order, np));
0160 }
0161 }
0162
0163 xmlPutToken_ = produces<LHEXMLStringProduct, edm::Transition::BeginRun>("LHEScriptOutput");
0164
0165 eventPutToken_ = produces<LHEEventProduct>();
0166 beginRunPutToken_ = produces<LHERunInfoProduct, edm::Transition::BeginRun>();
0167 }
0168
0169
0170
0171
0172
0173
0174 void ExternalLHEProducer::preallocThreads(unsigned int iThreads) { nThreads_ = iThreads; }
0175
0176
0177 void ExternalLHEProducer::produce(edm::Event& iEvent, const edm::EventSetup& iSetup) {
0178 nextEvent();
0179 if (!partonLevel_) {
0180 throw edm::Exception(edm::errors::EventGenerationFailure)
0181 << "No lhe event found in ExternalLHEProducer::produce(). "
0182 << "The likely cause is that the lhe file contains fewer events than were requested, which is possible "
0183 << "in case of phase space integration or uneweighting efficiency problems.";
0184 }
0185
0186 std::unique_ptr<LHEEventProduct> product(
0187 new LHEEventProduct(*partonLevel_->getHEPEUP(), partonLevel_->originalXWGTUP()));
0188 if (partonLevel_->getPDF()) {
0189 product->setPDF(*partonLevel_->getPDF());
0190 }
0191 std::for_each(partonLevel_->weights().begin(),
0192 partonLevel_->weights().end(),
0193 std::bind(&LHEEventProduct::addWeight, product.get(), std::placeholders::_1));
0194 product->setScales(partonLevel_->scales());
0195 product->setEvtNum(partonLevel_->evtnum());
0196 if (nPartonMapping_.empty()) {
0197 product->setNpLO(partonLevel_->npLO());
0198 product->setNpNLO(partonLevel_->npNLO());
0199 } else {
0200
0201 unsigned processId(partonLevel_->getHEPEUP()->IDPRUP);
0202 unsigned order(0);
0203 unsigned np(0);
0204 try {
0205 auto procDef(nPartonMapping_.at(processId));
0206 order = procDef.first;
0207 np = procDef.second;
0208 } catch (std::out_of_range&) {
0209 throw cms::Exception("ExternalLHEProducer")
0210 << "Unexpected IDPRUP encountered: " << partonLevel_->getHEPEUP()->IDPRUP;
0211 }
0212
0213 switch (order) {
0214 case 0:
0215 product->setNpLO(np);
0216 product->setNpNLO(-1);
0217 break;
0218 case 1:
0219 product->setNpLO(-1);
0220 product->setNpNLO(np);
0221 break;
0222 default:
0223 break;
0224 }
0225 }
0226
0227 std::for_each(partonLevel_->getComments().begin(),
0228 partonLevel_->getComments().end(),
0229 std::bind(&LHEEventProduct::addComment, product.get(), std::placeholders::_1));
0230
0231 iEvent.put(eventPutToken_, std::move(product));
0232
0233 partonLevel_.reset();
0234 return;
0235 }
0236
0237
0238 void ExternalLHEProducer::beginRunProduce(edm::Run& run, edm::EventSetup const& es) {
0239
0240
0241
0242
0243 edm::Service<edm::RandomNumberGenerator> rng;
0244
0245 if (!rng.isAvailable()) {
0246 throw cms::Exception("Configuration")
0247 << "The ExternalLHEProducer module requires the RandomNumberGeneratorService\n"
0248 "which is not present in the configuration file. You must add the service\n"
0249 "in the configuration file if you want to run ExternalLHEProducer";
0250 }
0251
0252 std::vector<std::string> infiles;
0253 auto const seed = rng->mySeed();
0254 if (generateConcurrently_) {
0255 infiles.resize(nThreads_);
0256 auto const nEventsAve = nEvents_ / nThreads_;
0257 unsigned int const overflow = nThreads_ - (nEvents_ % nThreads_);
0258 std::exception_ptr except;
0259 std::atomic<char> exceptSet{0};
0260
0261 tbb::this_task_arena::isolate([this, &except, &infiles, &exceptSet, nEventsAve, overflow, seed]() {
0262 tbb::task_group group;
0263 for (unsigned int t = 0; t < nThreads_; ++t) {
0264 uint32_t nEvents = nEventsAve;
0265 if (nEvents_ % nThreads_ != 0 and t >= overflow) {
0266 nEvents += 1;
0267 }
0268 group.run([t, this, &infiles, seed, nEvents, &except, &exceptSet]() {
0269 CMS_SA_ALLOW try {
0270 using namespace std::filesystem;
0271 using namespace std::string_literals;
0272 auto out = path("thread"s + std::to_string(t)) / path(outputFile_);
0273 infiles[t] = out.native();
0274 executeScript(makeArgs(nEvents, 1, seed + t), t, false);
0275 } catch (...) {
0276 char expected = 0;
0277 if (exceptSet.compare_exchange_strong(expected, 1)) {
0278 except = std::current_exception();
0279 exceptSet.store(2);
0280 }
0281 }
0282 });
0283 }
0284 group.wait();
0285 });
0286 if (exceptSet != 0) {
0287 std::rethrow_exception(except);
0288 }
0289 } else {
0290 infiles = std::vector<std::string>(1, outputFile_);
0291 executeScript(makeArgs(nEvents_, nThreads_, seed), 0, false);
0292 }
0293
0294
0295 if (!postGenerationCommand_.empty()) {
0296 std::vector<std::string> postcmd = postGenerationCommand_;
0297 try {
0298 postcmd[0] = edm::FileInPath(postcmd[0]).fullPath();
0299 } catch (const edm::Exception& e) {
0300 edm::LogWarning("ExternalLHEProducer") << postcmd[0] << " is not a relative path. Run it as a shell command.";
0301 }
0302 executeScript(postcmd, 0, true);
0303 }
0304
0305
0306 std::unique_ptr<LHEXMLStringProduct> p(new LHEXMLStringProduct);
0307
0308
0309 if (storeXML_) {
0310 std::string file;
0311 if (generateConcurrently_) {
0312 using namespace std::filesystem;
0313 file = (path("thread0") / path(outputFile_)).native();
0314 } else {
0315 file = outputFile_;
0316 }
0317 std::ifstream instream(file);
0318 if (!instream) {
0319 throw cms::Exception("OutputOpenError") << "Unable to open script output file " << outputFile_ << ".";
0320 }
0321 instream.seekg(0, instream.end);
0322 int insize = instream.tellg();
0323 instream.seekg(0, instream.beg);
0324 p->fillCompressedContent(instream, 0.25 * insize);
0325 instream.close();
0326 }
0327 run.put(xmlPutToken_, std::move(p));
0328
0329
0330 auto runInfo = generateRunInfo(infiles);
0331 if (runInfo) {
0332 run.put(beginRunPutToken_, std::move(runInfo));
0333 }
0334
0335
0336
0337 unsigned int skip = 0;
0338 reader_ = std::make_unique<lhef::LHEReader>(infiles, skip);
0339
0340 nextEvent();
0341 }
0342
0343 void ExternalLHEProducer::beginRun(edm::Run const& run, edm::EventSetup const& es) {}
0344
0345 void ExternalLHEProducer::endRun(edm::Run const& run, edm::EventSetup const& es) {
0346 nextEvent();
0347 if (partonLevel_) {
0348
0349 if (std::getenv("VALIDATION_RUN") != nullptr) {
0350 edm::LogWarning("ExternalLHEProducer")
0351 << "Event loop is over, but there are still lhe events to process, ignoring...";
0352 } else {
0353 throw edm::Exception(edm::errors::EventGenerationFailure)
0354 << "Error in ExternalLHEProducer::endRunProduce(). "
0355 << "Event loop is over, but there are still lhe events to process."
0356 << "This could happen if lhe file contains more events than requested. This is never expected to happen.";
0357 }
0358 }
0359
0360 reader_.reset();
0361 if (generateConcurrently_) {
0362 for (unsigned int t = 0; t < nThreads_; ++t) {
0363 using namespace std::filesystem;
0364 using namespace std::string_literals;
0365 auto out = path("thread"s + std::to_string(t)) / path(outputFile_);
0366 if (unlink(out.c_str())) {
0367 throw cms::Exception("OutputDeleteError") << "Unable to delete original script output file " << out
0368 << " (errno=" << errno << ", " << strerror(errno) << ").";
0369 }
0370 }
0371 } else {
0372 if (unlink(outputFile_.c_str())) {
0373 throw cms::Exception("OutputDeleteError") << "Unable to delete original script output file " << outputFile_
0374 << " (errno=" << errno << ", " << strerror(errno) << ").";
0375 }
0376 }
0377 }
0378
0379 std::vector<std::string> ExternalLHEProducer::makeArgs(uint32_t nEvents,
0380 unsigned int nThreads,
0381 std::uint32_t seed) const {
0382 std::vector<std::string> args;
0383 args.reserve(3 + args_.size());
0384
0385 args.push_back(args_.front());
0386 args.push_back(std::to_string(nEvents));
0387
0388 args.push_back(std::to_string(seed));
0389
0390 args.push_back(std::to_string(nThreads));
0391 std::copy(args_.begin() + 1, args_.end(), std::back_inserter(args));
0392
0393 for (unsigned int iArg = 0; iArg < args.size(); iArg++) {
0394 LogDebug("LHEInputArgs") << "arg [" << iArg << "] = " << args[iArg];
0395 }
0396
0397 return args;
0398 }
0399
0400
0401 int ExternalLHEProducer::closeDescriptors(int preserve) const {
0402 int maxfd = 1024;
0403 int fd;
0404 #ifdef __linux__
0405 DIR* dir;
0406 struct dirent* dp;
0407 maxfd = preserve;
0408 if ((dir = opendir("/proc/self/fd"))) {
0409 errno = 0;
0410 while ((dp = readdir(dir)) != nullptr) {
0411 if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) {
0412 continue;
0413 }
0414 if (sscanf(dp->d_name, "%d", &fd) != 1) {
0415
0416 return -1;
0417 }
0418 if (fd > maxfd) {
0419 maxfd = fd;
0420 }
0421 }
0422 if (errno) {
0423
0424 return errno;
0425 }
0426 closedir(dir);
0427 }
0428 #endif
0429
0430 for (fd = 3; fd < maxfd + 1; fd++) {
0431 if (fd != preserve)
0432 close(fd);
0433 }
0434 return 0;
0435 }
0436
0437
0438 void ExternalLHEProducer::executeScript(std::vector<std::string> const& args, int id, bool isPost) const {
0439
0440
0441 int rc = 0, rc2 = 0;
0442 int filedes[2], fd_flags;
0443
0444 if (pipe(filedes)) {
0445 throw cms::Exception("Unable to create a new pipe");
0446 }
0447 FileCloseSentry sentry1(filedes[0]), sentry2(filedes[1]);
0448
0449 if ((fd_flags = fcntl(filedes[1], F_GETFD, NULL)) == -1) {
0450 throw cms::Exception("ExternalLHEProducer")
0451 << "Failed to get pipe file descriptor flags (errno=" << rc << ", " << strerror(rc) << ")";
0452 }
0453 if (fcntl(filedes[1], F_SETFD, fd_flags | FD_CLOEXEC) == -1) {
0454 throw cms::Exception("ExternalLHEProducer")
0455 << "Failed to set pipe file descriptor flags (errno=" << rc << ", " << strerror(rc) << ")";
0456 }
0457
0458 unsigned int argc_pre = 0;
0459
0460 if (!isPost) {
0461 argc_pre = 1;
0462 }
0463 unsigned int argc = argc_pre + args.size();
0464
0465 char** argv = new char*[argc + 1];
0466 if (!isPost) {
0467 argv[0] = strdup(scriptName_.c_str());
0468 }
0469 for (unsigned int i = 0; i < args.size(); i++) {
0470 argv[argc_pre + i] = strdup(args[i].c_str());
0471 }
0472 argv[argc] = nullptr;
0473
0474 pid_t pid = fork();
0475 if (pid == 0) {
0476
0477 if (!(rc = closeDescriptors(filedes[1]))) {
0478 if (!isPost && generateConcurrently_) {
0479 using namespace std::filesystem;
0480 using namespace std::string_literals;
0481 std::error_code ec;
0482 auto newDir = path("thread"s + std::to_string(id));
0483 create_directory(newDir, ec);
0484 current_path(newDir, ec);
0485 }
0486 execvp(argv[0], argv);
0487 rc = errno;
0488 }
0489 while ((write(filedes[1], &rc, sizeof(int)) == -1) && (errno == EINTR)) {
0490 }
0491 _exit(1);
0492 }
0493
0494
0495 for (unsigned int i = 0; i < args.size() + 1; i++) {
0496 free(argv[i]);
0497 }
0498 delete[] argv;
0499
0500 if (pid == -1) {
0501 throw cms::Exception("ForkException")
0502 << "Unable to fork a child (errno=" << errno << ", " << strerror(errno) << ")";
0503 }
0504
0505 close(filedes[1]);
0506
0507 while (((rc2 = read(filedes[0], &rc, sizeof(int))) == -1) && (errno == EINTR)) {
0508 rc2 = 0;
0509 }
0510 if ((rc2 == sizeof(int)) && rc) {
0511 throw cms::Exception("ExternalLHEProducer")
0512 << "Failed to execute script (errno=" << rc << ", " << strerror(rc) << ")";
0513 }
0514 close(filedes[0]);
0515
0516 int status = 0;
0517 errno = 0;
0518 do {
0519 if (waitpid(pid, &status, 0) < 0) {
0520 if (errno == EINTR) {
0521 continue;
0522 } else {
0523 throw cms::Exception("ExternalLHEProducer")
0524 << "Failed to read child status (errno=" << errno << ", " << strerror(errno) << ")";
0525 }
0526 }
0527 if (WIFSIGNALED(status)) {
0528 throw cms::Exception("ExternalLHEProducer") << "Child exited due to signal " << WTERMSIG(status) << ".";
0529 }
0530 if (WIFEXITED(status)) {
0531 rc = WEXITSTATUS(status);
0532 break;
0533 }
0534 } while (true);
0535 if (rc) {
0536 throw cms::Exception("ExternalLHEProducer") << "Child failed with exit code " << rc << ".";
0537 }
0538 }
0539
0540
0541 void ExternalLHEProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0542
0543
0544 edm::ParameterSetDescription desc;
0545 desc.setComment("Executes an external script and places its output file into an EDM collection");
0546
0547 edm::FileInPath thePath;
0548 desc.add<edm::FileInPath>("scriptName", thePath);
0549 desc.add<std::string>("outputFile", "myoutput");
0550 desc.add<std::vector<std::string>>("args");
0551 desc.add<uint32_t>("numberOfParameters");
0552 desc.addUntracked<uint32_t>("nEvents");
0553 desc.addUntracked<bool>("storeXML", false);
0554 desc.addUntracked<bool>("generateConcurrently", false)
0555 ->setComment("If true, run the script concurrently in separate processes.");
0556 desc.addUntracked<std::vector<std::string>>("postGenerationCommand", std::vector<std::string>())
0557 ->setComment(
0558 "Command to run after the generation script has completed. The first argument can be a relative path.");
0559
0560 edm::ParameterSetDescription nPartonMappingDesc;
0561 nPartonMappingDesc.add<unsigned>("idprup");
0562 nPartonMappingDesc.add<std::string>("order");
0563 nPartonMappingDesc.add<unsigned>("np");
0564 desc.addVPSetOptional("nPartonMapping", nPartonMappingDesc);
0565
0566 descriptions.addDefault(desc);
0567 }
0568
0569 std::unique_ptr<LHERunInfoProduct> ExternalLHEProducer::generateRunInfo(std::vector<std::string> const& iFiles) const {
0570 std::unique_ptr<LHERunInfoProduct> retValue;
0571
0572 for (auto const& file : iFiles) {
0573 unsigned int skip = 0;
0574 std::vector<std::string> infiles(1, file);
0575 auto reader = std::make_unique<lhef::LHEReader>(infiles, skip);
0576 auto parton = reader->next();
0577 if (!parton) {
0578 break;
0579 }
0580 auto runInfo = parton->getRunInfo();
0581 LHERunInfoProduct product(*runInfo->getHEPRUP());
0582
0583 std::for_each(runInfo->getHeaders().begin(),
0584 runInfo->getHeaders().end(),
0585 std::bind(&LHERunInfoProduct::addHeader, &product, std::placeholders::_1));
0586 std::for_each(runInfo->getComments().begin(),
0587 runInfo->getComments().end(),
0588 std::bind(&LHERunInfoProduct::addComment, &product, std::placeholders::_1));
0589 if (not retValue) {
0590 retValue = std::make_unique<LHERunInfoProduct>(std::move(product));
0591 } else {
0592 retValue->mergeProduct(product);
0593 }
0594 }
0595
0596 return retValue;
0597 }
0598
0599 void ExternalLHEProducer::nextEvent() {
0600 if (partonLevel_)
0601 return;
0602
0603 if (not reader_) {
0604 return;
0605 }
0606
0607 partonLevel_ = reader_->next();
0608 if (!partonLevel_) {
0609
0610 bool newFileOpened;
0611 do {
0612 newFileOpened = false;
0613 partonLevel_ = reader_->next(&newFileOpened);
0614 } while (newFileOpened && !partonLevel_);
0615 }
0616 }
0617
0618
0619 DEFINE_FWK_MODULE(ExternalLHEProducer);