JeProf/plugins/JeProfService.cc

0001
0002 //
0003 //  Description: FWK service to implement hook for jemalloc heap profile
0004 //               dump functionality
0005 //
0006
0007 #include "FWCore/Framework/interface/MakerMacros.h"
0008 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
0009 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0010 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0011 #include "FWCore/Framework/interface/LuminosityBlock.h"
0012 #include "FWCore/Framework/interface/FileBlock.h"
0013 #include "FWCore/Framework/interface/Event.h"
0014 #include "FWCore/Framework/interface/Run.h"
0015 #include <string>
0016 #include <dlfcn.h>
0017 #include <cstdio>
0018 #include <cstring>
0019
0020 extern "C" {
0021 typedef int (*mallctl_t)(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
0022 }
0023
0024 namespace {
0025   bool initialize_prof();
0026
0027   mallctl_t mallctl = nullptr;
0028   const bool have_jemalloc_and_prof = initialize_prof();
0029
0030   bool initialize_prof() {
0031     // check if mallctl and friends are available, if we are using jemalloc
0032     mallctl = (mallctl_t)::dlsym(RTLD_DEFAULT, "mallctl");
0033     if (mallctl == nullptr)
0034       return false;
0035     // check if heap profiling available, if --enable-prof was specified at build time
0036     bool enable_prof = false;
0037     size_t bool_s = sizeof(bool);
0038     mallctl("prof.active", &enable_prof, &bool_s, nullptr, 0);
0039     return enable_prof;
0040   }
0041
0042 }  // namespace
0043
0044 namespace edm {
0045   class GlobalContext;
0046   class StreamContext;
0047
0048   namespace service {
0049     class JeProfService {
0050     public:
0051       JeProfService(const ParameterSet &, ActivityRegistry &);
0052
0053       void postBeginJob();
0054
0055       void postBeginRun(GlobalContext const &gc);
0056
0057       void postBeginLumi(GlobalContext const &gc);
0058
0059       void preEvent(StreamContext const &sc);
0060       void postEvent(StreamContext const &sc);
0061
0062       void preModuleEvent(StreamContext const &sc, ModuleCallingContext const &mcc);
0063       void postModuleEvent(StreamContext const &sc, ModuleCallingContext const &mcc);
0064
0065       void postEndLumi(GlobalContext const &gc);
0066
0067       void preEndRun(GlobalContext const &gc);
0068       void postEndRun(GlobalContext const &gc);
0069
0070       void preEndProcessBlock(GlobalContext const &gc);
0071       void postEndProcessBlock(GlobalContext const &gc);
0072
0073       void preEndJob();
0074       void postEndJob();
0075
0076       void postOpenFile(std::string const &);
0077
0078       void postCloseFile(std::string const &);
0079
0080     private:
0081       void makeDump(const std::string &format, std::string_view moduleLabel = "");
0082       static std::string replace(const std::string &s, const char *pat, int val);
0083       static std::string replaceU64(const std::string &s, const char *pat, unsigned long long val);
0084       static std::string replace(const std::string &s, const char *pat, std::string_view val);
0085
0086       std::string atPostBeginJob_;
0087       std::string atPostBeginRun_;
0088       std::string atPostBeginLumi_;
0089
0090       std::string atPreEvent_;
0091       std::string atPostEvent_;
0092
0093       std::vector<std::string> modules_;
0094       std::vector<std::string> moduleTypes_;
0095       std::string atPreModuleEvent_;
0096       std::string atPostModuleEvent_;
0097
0098       std::string atPostEndLumi_;
0099       std::string atPreEndRun_;
0100       std::string atPostEndRun_;
0101       std::string atPreEndProcessBlock_;
0102       std::string atPostEndProcessBlock_;
0103       std::string atPreEndJob_;
0104       std::string atPostEndJob_;
0105
0106       std::string atPostOpenFile_;
0107       std::string atPostCloseFile_;
0108
0109       int mineventrecord_;
0110       int prescale_;
0111       int nrecord_;  // counter
0112       edm::EventNumber_t nevent_;
0113       edm::RunNumber_t nrun_;
0114       edm::LuminosityBlockNumber_t nlumi_;
0115       int nfileopened_;  // counter of files opened thus far
0116       int nfileclosed_;  // counter of files closed thus far
0117     };
0118     inline bool isProcessWideService(JeProfService const *) { return true; }
0119   }  // namespace service
0120 }  // namespace edm
0121
0122 using namespace edm::service;
0123
0124 JeProfService::JeProfService(ParameterSet const &ps, ActivityRegistry &iRegistry)
0125     : mineventrecord_(1), prescale_(1), nrecord_(0), nevent_(0), nrun_(0), nlumi_(0), nfileopened_(0), nfileclosed_(0) {
0126   if (!have_jemalloc_and_prof) {
0127     edm::LogWarning("JeProfModule") << "JeProfModule requested but application is not"
0128                                     << " currently being profiled with jemalloc profiling\n";
0129   }
0130   // Get the configuration
0131   prescale_ = ps.getUntrackedParameter<int>("reportEventInterval", prescale_);
0132   mineventrecord_ = ps.getUntrackedParameter<int>("reportFirstEvent", mineventrecord_);
0133
0134   atPostBeginJob_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostBeginJob", atPostBeginJob_);
0135   atPostBeginRun_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostBeginRun", atPostBeginRun_);
0136   atPostBeginLumi_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostBeginLumi", atPostBeginLumi_);
0137
0138   atPreEvent_ = ps.getUntrackedParameter<std::string>("reportToFileAtPreEvent", atPreEvent_);
0139   atPostEvent_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostEvent", atPostEvent_);
0140
0141   modules_ = ps.getUntrackedParameter<std::vector<std::string>>("reportModules", modules_);
0142   moduleTypes_ = ps.getUntrackedParameter<std::vector<std::string>>("reportModuleTypes", moduleTypes_);
0143   std::sort(modules_.begin(), modules_.end());
0144   std::sort(moduleTypes_.begin(), moduleTypes_.end());
0145   atPreModuleEvent_ = ps.getUntrackedParameter<std::string>("reportToFileAtPreModuleEvent", atPreModuleEvent_);
0146   atPostModuleEvent_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostModuleEvent", atPostModuleEvent_);
0147
0148   atPostEndLumi_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostEndLumi", atPostEndLumi_);
0149   atPreEndRun_ = ps.getUntrackedParameter<std::string>("reportToFileAtPreEndRun", atPreEndRun_);
0150   atPostEndRun_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostEndRun", atPostEndRun_);
0151   atPreEndProcessBlock_ =
0152       ps.getUntrackedParameter<std::string>("reportToFileAtPreEndProcessBlock", atPreEndProcessBlock_);
0153   atPostEndProcessBlock_ =
0154       ps.getUntrackedParameter<std::string>("reportToFileAtPostEndProcessBlock", atPostEndProcessBlock_);
0155   atPreEndJob_ = ps.getUntrackedParameter<std::string>("reportToFileAtPreEndJob", atPreEndJob_);
0156   atPostEndJob_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostEndJob", atPostEndJob_);
0157
0158   atPostOpenFile_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostOpenFile", atPostOpenFile_);
0159   atPostCloseFile_ = ps.getUntrackedParameter<std::string>("reportToFileAtPostCloseFile", atPostCloseFile_);
0160
0161   // Register for the framework signals
0162   iRegistry.watchPostBeginJob(this, &JeProfService::postBeginJob);
0163   iRegistry.watchPostGlobalBeginRun(this, &JeProfService::postBeginRun);
0164   iRegistry.watchPostGlobalBeginLumi(this, &JeProfService::postBeginLumi);
0165
0166   iRegistry.watchPreEvent(this, &JeProfService::preEvent);
0167   iRegistry.watchPostEvent(this, &JeProfService::postEvent);
0168
0169   if (not modules_.empty() or not moduleTypes_.empty()) {
0170     iRegistry.watchPreModuleEvent(this, &JeProfService::preModuleEvent);
0171     iRegistry.watchPostModuleEvent(this, &JeProfService::postModuleEvent);
0172   }
0173
0174   iRegistry.watchPostGlobalEndLumi(this, &JeProfService::postEndLumi);
0175   iRegistry.watchPreGlobalEndRun(this, &JeProfService::preEndRun);
0176   iRegistry.watchPostGlobalEndRun(this, &JeProfService::postEndRun);
0177   iRegistry.watchPreEndProcessBlock(this, &JeProfService::preEndProcessBlock);
0178   iRegistry.watchPostEndProcessBlock(this, &JeProfService::postEndProcessBlock);
0179   iRegistry.watchPreEndJob(this, &JeProfService::preEndJob);
0180   iRegistry.watchPostEndJob(this, &JeProfService::postEndJob);
0181
0182   iRegistry.watchPostOpenFile(this, &JeProfService::postOpenFile);
0183   iRegistry.watchPostCloseFile(this, &JeProfService::postCloseFile);
0184 }
0185
0186 void JeProfService::postBeginJob() { makeDump(atPostBeginJob_); }
0187
0188 void JeProfService::postBeginRun(GlobalContext const &gc) {
0189   nrun_ = gc.luminosityBlockID().run();
0190   makeDump(atPostBeginRun_);
0191 }
0192
0193 void JeProfService::postBeginLumi(GlobalContext const &gc) {
0194   nlumi_ = gc.luminosityBlockID().luminosityBlock();
0195   makeDump(atPostBeginLumi_);
0196 }
0197
0198 void JeProfService::preEvent(StreamContext const &iStream) {
0199   ++nrecord_;  // count before events
0200   nevent_ = iStream.eventID().event();
0201   if ((prescale_ > 0) && (nrecord_ >= mineventrecord_) && (((nrecord_ - mineventrecord_) % prescale_) == 0))
0202     makeDump(atPreEvent_);
0203 }
0204
0205 void JeProfService::postEvent(StreamContext const &iStream) {
0206   nevent_ = iStream.eventID().event();
0207   if ((prescale_ > 0) && (nrecord_ >= mineventrecord_) && (((nrecord_ - mineventrecord_) % prescale_) == 0))
0208     makeDump(atPostEvent_);
0209 }
0210
0211 void JeProfService::preModuleEvent(StreamContext const &iStream, ModuleCallingContext const &mcc) {
0212   nevent_ = iStream.eventID().event();
0213   if ((prescale_ > 0) && (nrecord_ >= mineventrecord_) && (((nrecord_ - mineventrecord_) % prescale_) == 0)) {
0214     auto const &moduleLabel = mcc.moduleDescription()->moduleLabel();
0215     auto const &moduleType = mcc.moduleDescription()->moduleName();
0216     if (std::binary_search(modules_.begin(), modules_.end(), moduleLabel) or
0217         std::binary_search(moduleTypes_.begin(), moduleTypes_.end(), moduleType)) {
0218       makeDump(atPreModuleEvent_, moduleLabel);
0219     }
0220   }
0221 }
0222
0223 void JeProfService::postModuleEvent(StreamContext const &iStream, ModuleCallingContext const &mcc) {
0224   nevent_ = iStream.eventID().event();
0225   if ((prescale_ > 0) && (nrecord_ >= mineventrecord_) && (((nrecord_ - mineventrecord_) % prescale_) == 0)) {
0226     auto const &moduleLabel = mcc.moduleDescription()->moduleLabel();
0227     auto const &moduleType = mcc.moduleDescription()->moduleName();
0228     if (std::binary_search(modules_.begin(), modules_.end(), moduleLabel) or
0229         std::binary_search(moduleTypes_.begin(), moduleTypes_.end(), moduleType)) {
0230       makeDump(atPostModuleEvent_, moduleLabel);
0231     }
0232   }
0233 }
0234
0235 void JeProfService::postEndLumi(GlobalContext const &gc) {
0236   nlumi_ = gc.luminosityBlockID().luminosityBlock();
0237   makeDump(atPostEndLumi_);
0238 }
0239
0240 void JeProfService::preEndRun(GlobalContext const &gc) {
0241   nrun_ = gc.luminosityBlockID().run();
0242   makeDump(atPreEndRun_);
0243 }
0244
0245 void JeProfService::postEndRun(GlobalContext const &gc) {
0246   nrun_ = gc.luminosityBlockID().run();
0247   makeDump(atPostEndRun_);
0248 }
0249
0250 void JeProfService::preEndProcessBlock(GlobalContext const &gc) { makeDump(atPreEndProcessBlock_); }
0251
0252 void JeProfService::postEndProcessBlock(GlobalContext const &gc) { makeDump(atPostEndProcessBlock_); }
0253
0254 void JeProfService::preEndJob() { makeDump(atPreEndJob_); }
0255
0256 void JeProfService::postEndJob() { makeDump(atPostEndJob_); }
0257
0258 void JeProfService::postOpenFile(std::string const &) {
0259   ++nfileopened_;
0260   makeDump(atPostOpenFile_);
0261 }
0262
0263 void JeProfService::postCloseFile(std::string const &) {
0264   ++nfileclosed_;
0265   makeDump(atPostCloseFile_);
0266 }
0267
0268 void JeProfService::makeDump(const std::string &format, std::string_view moduleLabel) {
0269   if (!have_jemalloc_and_prof || format.empty())
0270     return;
0271
0272   std::string final(format);
0273   final = replace(final, "%I", nrecord_);
0274   final = replaceU64(final, "%E", nevent_);
0275   final = replaceU64(final, "%R", nrun_);
0276   final = replaceU64(final, "%L", nlumi_);
0277   final = replace(final, "%F", nfileopened_);
0278   final = replace(final, "%C", nfileclosed_);
0279   final = replace(final, "%M", moduleLabel);
0280   const char *fileName = final.c_str();
0281   mallctl("prof.dump", nullptr, nullptr, &fileName, sizeof(const char *));
0282 }
0283
0284 std::string JeProfService::replace(const std::string &s, const char *pat, int val) {
0285   size_t pos = 0;
0286   size_t patlen = strlen(pat);
0287   std::string result = s;
0288   while ((pos = result.find(pat, pos)) != std::string::npos) {
0289     char buf[64];
0290     int n = sprintf(buf, "%d", val);
0291     result.replace(pos, patlen, buf);
0292     pos = pos - patlen + n;
0293   }
0294
0295   return result;
0296 }
0297
0298 std::string JeProfService::replaceU64(const std::string &s, const char *pat, unsigned long long val) {
0299   size_t pos = 0;
0300   size_t patlen = strlen(pat);
0301   std::string result = s;
0302   while ((pos = result.find(pat, pos)) != std::string::npos) {
0303     char buf[64];
0304     int n = sprintf(buf, "%llu", val);
0305     result.replace(pos, patlen, buf);
0306     pos = pos - patlen + n;
0307   }
0308
0309   return result;
0310 }
0311
0312 std::string JeProfService::replace(const std::string &s, const char *pat, std::string_view val) {
0313   size_t pos = 0;
0314   size_t patlen = strlen(pat);
0315   std::string result = s;
0316   while ((pos = result.find(pat, pos)) != std::string::npos) {
0317     result.replace(pos, patlen, val.data());
0318     pos = pos - patlen + val.size();
0319   }
0320
0321   return result;
0322 }
0323
0324 DEFINE_FWK_SERVICE(JeProfService);