CUDAServices/plugins/NVProfilerService.cc

0001 #include <algorithm>
0002 #include <iostream>
0003 #include <sstream>
0004 #include <string>
0005 #include <vector>
0006
0007 #include <oneapi/tbb/concurrent_vector.h>
0008
0009 #include <fmt/printf.h>
0010
0011 #include <cuda_profiler_api.h>
0012 #include <nvtx3/nvToolsExt.h>
0013
0014 #include "DataFormats/Common/interface/HLTPathStatus.h"
0015 #include "DataFormats/Provenance/interface/EventID.h"
0016 #include "DataFormats/Provenance/interface/LuminosityBlockID.h"
0017 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0018 #include "DataFormats/Provenance/interface/RunID.h"
0019 #include "DataFormats/Provenance/interface/Timestamp.h"
0020 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0021 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0022 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0023 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0024 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0025 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0026 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0027 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0028 #include "FWCore/ServiceRegistry/interface/PathsAndConsumesOfModulesBase.h"
0029 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0030 #include "FWCore/ServiceRegistry/interface/Service.h"
0031 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0032 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0033 #include "FWCore/Utilities/interface/BranchType.h"
0034 #include "FWCore/Utilities/interface/Exception.h"
0035 #include "FWCore/Utilities/interface/ProductKindOfType.h"
0036 #include "FWCore/Utilities/interface/TimeOfDay.h"
0037 #include "HeterogeneousCore/CUDAServices/interface/CUDAInterface.h"
0038
0039 using namespace std::string_literals;
0040
0041 namespace {
0042   int nvtxDomainRangePush(nvtxDomainHandle_t domain, const char* message) {
0043     nvtxEventAttributes_t eventAttrib = {};
0044     eventAttrib.version = NVTX_VERSION;
0045     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0046     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0047     eventAttrib.message.ascii = message;
0048     return nvtxDomainRangePushEx(domain, &eventAttrib);
0049   }
0050
0051   __attribute__((unused)) int nvtxDomainRangePushColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0052     nvtxEventAttributes_t eventAttrib = {};
0053     eventAttrib.version = NVTX_VERSION;
0054     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0055     eventAttrib.colorType = NVTX_COLOR_ARGB;
0056     eventAttrib.color = color;
0057     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0058     eventAttrib.message.ascii = message;
0059     return nvtxDomainRangePushEx(domain, &eventAttrib);
0060   }
0061
0062   __attribute__((unused)) nvtxRangeId_t nvtxDomainRangeStart(nvtxDomainHandle_t domain, const char* message) {
0063     nvtxEventAttributes_t eventAttrib = {};
0064     eventAttrib.version = NVTX_VERSION;
0065     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0066     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0067     eventAttrib.message.ascii = message;
0068     return nvtxDomainRangeStartEx(domain, &eventAttrib);
0069   }
0070
0071   nvtxRangeId_t nvtxDomainRangeStartColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0072     nvtxEventAttributes_t eventAttrib = {};
0073     eventAttrib.version = NVTX_VERSION;
0074     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0075     eventAttrib.colorType = NVTX_COLOR_ARGB;
0076     eventAttrib.color = color;
0077     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0078     eventAttrib.message.ascii = message;
0079     return nvtxDomainRangeStartEx(domain, &eventAttrib);
0080   }
0081
0082   void nvtxDomainMark(nvtxDomainHandle_t domain, const char* message) {
0083     nvtxEventAttributes_t eventAttrib = {};
0084     eventAttrib.version = NVTX_VERSION;
0085     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0086     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0087     eventAttrib.message.ascii = message;
0088     nvtxDomainMarkEx(domain, &eventAttrib);
0089   }
0090
0091   __attribute__((unused)) void nvtxDomainMarkColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0092     nvtxEventAttributes_t eventAttrib = {};
0093     eventAttrib.version = NVTX_VERSION;
0094     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0095     eventAttrib.colorType = NVTX_COLOR_ARGB;
0096     eventAttrib.color = color;
0097     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0098     eventAttrib.message.ascii = message;
0099     nvtxDomainMarkEx(domain, &eventAttrib);
0100   }
0101
0102   enum {
0103     nvtxBlack = 0x00000000,
0104     nvtxRed = 0x00ff0000,
0105     nvtxDarkGreen = 0x00009900,
0106     nvtxGreen = 0x0000ff00,
0107     nvtxLightGreen = 0x00ccffcc,
0108     nvtxBlue = 0x000000ff,
0109     nvtxAmber = 0x00ffbf00,
0110     nvtxLightAmber = 0x00fff2cc,
0111     nvtxWhite = 0x00ffffff
0112   };
0113
0114   constexpr nvtxRangeId_t nvtxInvalidRangeId = 0xfffffffffffffffful;
0115 }  // namespace
0116
0117 class NVProfilerService {
0118 public:
0119   NVProfilerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0120   ~NVProfilerService();
0121
0122   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0123
0124   void preallocate(edm::service::SystemBounds const&);
0125
0126   // these signal pair are NOT guaranteed to be called by the same thread
0127   void preBeginJob(edm::ProcessContext const&);
0128   void postBeginJob();
0129
0130   void lookupInitializationComplete(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0131
0132   // there is no preEndJob() signal
0133   void postEndJob();
0134
0135   // these signal pair are NOT guaranteed to be called by the same thread
0136   void preGlobalBeginRun(edm::GlobalContext const&);
0137   void postGlobalBeginRun(edm::GlobalContext const&);
0138
0139   // these signal pair are NOT guaranteed to be called by the same thread
0140   void preGlobalEndRun(edm::GlobalContext const&);
0141   void postGlobalEndRun(edm::GlobalContext const&);
0142
0143   // these signal pair are NOT guaranteed to be called by the same thread
0144   void preStreamBeginRun(edm::StreamContext const&);
0145   void postStreamBeginRun(edm::StreamContext const&);
0146
0147   // these signal pair are NOT guaranteed to be called by the same thread
0148   void preStreamEndRun(edm::StreamContext const&);
0149   void postStreamEndRun(edm::StreamContext const&);
0150
0151   // these signal pair are NOT guaranteed to be called by the same thread
0152   void preGlobalBeginLumi(edm::GlobalContext const&);
0153   void postGlobalBeginLumi(edm::GlobalContext const&);
0154
0155   // these signal pair are NOT guaranteed to be called by the same thread
0156   void preGlobalEndLumi(edm::GlobalContext const&);
0157   void postGlobalEndLumi(edm::GlobalContext const&);
0158
0159   // these signal pair are NOT guaranteed to be called by the same thread
0160   void preStreamBeginLumi(edm::StreamContext const&);
0161   void postStreamBeginLumi(edm::StreamContext const&);
0162
0163   // these signal pair are NOT guaranteed to be called by the same thread
0164   void preStreamEndLumi(edm::StreamContext const&);
0165   void postStreamEndLumi(edm::StreamContext const&);
0166
0167   // these signal pair are NOT guaranteed to be called by the same thread
0168   void preEvent(edm::StreamContext const&);
0169   void postEvent(edm::StreamContext const&);
0170
0171   // these signal pair are NOT guaranteed to be called by the same thread
0172   void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0173   void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0174
0175   // these signal pair are NOT guaranteed to be called by the same thread
0176   void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0177   void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0178
0179   // these signal pair are guaranteed to be called by the same thread
0180   void preOpenFile(std::string const&);
0181   void postOpenFile(std::string const&);
0182
0183   // these signal pair are guaranteed to be called by the same thread
0184   void preCloseFile(std::string const&);
0185   void postCloseFile(std::string const&);
0186
0187   // these signal pair are guaranteed to be called by the same thread
0188   void preSourceConstruction(edm::ModuleDescription const&);
0189   void postSourceConstruction(edm::ModuleDescription const&);
0190
0191   // these signal pair are guaranteed to be called by the same thread
0192   void preSourceRun(edm::RunIndex);
0193   void postSourceRun(edm::RunIndex);
0194
0195   // these signal pair are guaranteed to be called by the same thread
0196   void preSourceLumi(edm::LuminosityBlockIndex);
0197   void postSourceLumi(edm::LuminosityBlockIndex);
0198
0199   // these signal pair are guaranteed to be called by the same thread
0200   void preSourceEvent(edm::StreamID);
0201   void postSourceEvent(edm::StreamID);
0202
0203   // these signal pair are guaranteed to be called by the same thread
0204   void preModuleConstruction(edm::ModuleDescription const&);
0205   void postModuleConstruction(edm::ModuleDescription const&);
0206
0207   // these signal pair are guaranteed to be called by the same thread
0208   void preModuleDestruction(edm::ModuleDescription const&);
0209   void postModuleDestruction(edm::ModuleDescription const&);
0210
0211   // these signal pair are guaranteed to be called by the same thread
0212   void preModuleBeginJob(edm::ModuleDescription const&);
0213   void postModuleBeginJob(edm::ModuleDescription const&);
0214
0215   // these signal pair are guaranteed to be called by the same thread
0216   void preModuleEndJob(edm::ModuleDescription const&);
0217   void postModuleEndJob(edm::ModuleDescription const&);
0218
0219   // these signal pair are guaranteed to be called by the same thread
0220   void preModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0221   void postModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0222
0223   // these signal pair are guaranteed to be called by the same thread
0224   void preModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0225   void postModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0226
0227   // these signal pair are guaranteed to be called by the same thread
0228   void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0229   void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0230
0231   // these signal pair are guaranteed to be called by the same thread
0232   void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0233   void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0234
0235   // these signal pair are guaranteed to be called by the same thread
0236   void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0237   void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0238
0239   // these signal pair are guaranteed to be called by the same thread
0240   void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0241   void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0242
0243   // these signal pair are guaranteed to be called by the same thread
0244   void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0245   void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0246
0247   // these signal pair are guaranteed to be called by the same thread
0248   void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0249   void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0250
0251   // these signal pair are guaranteed to be called by the same thread
0252   void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0253   void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0254
0255   // these signal pair are guaranteed to be called by the same thread
0256   void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0257   void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0258
0259   // these signal pair are guaranteed to be called by the same thread
0260   void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0261   void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0262
0263   // these signal pair are guaranteed to be called by the same thread
0264   void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0265   void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0266
0267   // these signal pair are guaranteed to be called by the same thread
0268   void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0269   void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0270
0271   // these signal pair are guaranteed to be called by the same thread
0272   void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0273   void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0274
0275   // these signal pair are NOT guaranteed to be called by the same thread
0276   void preModuleTransformPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0277   void postModuleTransformPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0278
0279   // these signal pair are guaranteed to be called by the same thread
0280   void preModuleTransformAcquiring(edm::StreamContext const&, edm::ModuleCallingContext const&);
0281   void postModuleTransformAcquiring(edm::StreamContext const&, edm::ModuleCallingContext const&);
0282
0283   // these signal pair are guaranteed to be called by the same thread
0284   void preModuleTransform(edm::StreamContext const&, edm::ModuleCallingContext const&);
0285   void postModuleTransform(edm::StreamContext const&, edm::ModuleCallingContext const&);
0286
0287 private:
0288   bool highlight(std::string const& label) const {
0289     return (std::binary_search(highlightModules_.begin(), highlightModules_.end(), label));
0290   }
0291
0292   uint32_t labelColor(std::string const& label) const { return highlight(label) ? nvtxAmber : nvtxGreen; }
0293
0294   uint32_t labelColorLight(std::string const& label) const {
0295     return highlight(label) ? nvtxLightAmber : nvtxLightGreen;
0296   }
0297
0298   std::vector<std::string> highlightModules_;
0299   const bool showModulePrefetching_;
0300   const bool skipFirstEvent_;
0301
0302   std::atomic<bool> globalFirstEventDone_ = false;
0303   std::vector<std::atomic<bool>> streamFirstEventDone_;
0304   std::vector<nvtxRangeId_t> event_;                        // per-stream event ranges
0305   std::vector<std::vector<nvtxRangeId_t>> stream_modules_;  // per-stream, per-module ranges
0306   // use a tbb::concurrent_vector rather than an std::vector because its final size is not known
0307   tbb::concurrent_vector<nvtxRangeId_t> global_modules_;  // global per-module events
0308
0309   nvtxDomainHandle_t global_domain_;               // NVTX domain for global EDM transitions
0310   std::vector<nvtxDomainHandle_t> stream_domain_;  // NVTX domains for per-EDM-stream transitions
0311 };
0312
0313 NVProfilerService::NVProfilerService(edm::ParameterSet const& config, edm::ActivityRegistry& registry)
0314     : highlightModules_(config.getUntrackedParameter<std::vector<std::string>>("highlightModules")),
0315       showModulePrefetching_(config.getUntrackedParameter<bool>("showModulePrefetching")),
0316       skipFirstEvent_(config.getUntrackedParameter<bool>("skipFirstEvent")) {
0317   // make sure that CUDA is initialised, and that the CUDAInterface destructor is called after this service's destructor
0318   edm::Service<CUDAInterface> cuda;
0319   if (not cuda or not cuda->enabled())
0320     return;
0321
0322   std::sort(highlightModules_.begin(), highlightModules_.end());
0323
0324   // create the NVTX domain for global EDM transitions
0325   global_domain_ = nvtxDomainCreate("EDM Global");
0326
0327   // enables profile collection; if profiling is already enabled it has no effect
0328   if (not skipFirstEvent_) {
0329     cudaProfilerStart();
0330   }
0331
0332   registry.watchPreallocate(this, &NVProfilerService::preallocate);
0333
0334   // these signal pair are NOT guaranteed to be called by the same thread
0335   registry.watchPreBeginJob(this, &NVProfilerService::preBeginJob);
0336   registry.watchPostBeginJob(this, &NVProfilerService::postBeginJob);
0337
0338   registry.watchLookupInitializationComplete(this, &NVProfilerService::lookupInitializationComplete);
0339
0340   // there is no preEndJob() signal
0341   registry.watchPostEndJob(this, &NVProfilerService::postEndJob);
0342
0343   // these signal pair are NOT guaranteed to be called by the same thread
0344   registry.watchPreGlobalBeginRun(this, &NVProfilerService::preGlobalBeginRun);
0345   registry.watchPostGlobalBeginRun(this, &NVProfilerService::postGlobalBeginRun);
0346
0347   // these signal pair are NOT guaranteed to be called by the same thread
0348   registry.watchPreGlobalEndRun(this, &NVProfilerService::preGlobalEndRun);
0349   registry.watchPostGlobalEndRun(this, &NVProfilerService::postGlobalEndRun);
0350
0351   // these signal pair are NOT guaranteed to be called by the same thread
0352   registry.watchPreStreamBeginRun(this, &NVProfilerService::preStreamBeginRun);
0353   registry.watchPostStreamBeginRun(this, &NVProfilerService::postStreamBeginRun);
0354
0355   // these signal pair are NOT guaranteed to be called by the same thread
0356   registry.watchPreStreamEndRun(this, &NVProfilerService::preStreamEndRun);
0357   registry.watchPostStreamEndRun(this, &NVProfilerService::postStreamEndRun);
0358
0359   // these signal pair are NOT guaranteed to be called by the same thread
0360   registry.watchPreGlobalBeginLumi(this, &NVProfilerService::preGlobalBeginLumi);
0361   registry.watchPostGlobalBeginLumi(this, &NVProfilerService::postGlobalBeginLumi);
0362
0363   // these signal pair are NOT guaranteed to be called by the same thread
0364   registry.watchPreGlobalEndLumi(this, &NVProfilerService::preGlobalEndLumi);
0365   registry.watchPostGlobalEndLumi(this, &NVProfilerService::postGlobalEndLumi);
0366
0367   // these signal pair are NOT guaranteed to be called by the same thread
0368   registry.watchPreStreamBeginLumi(this, &NVProfilerService::preStreamBeginLumi);
0369   registry.watchPostStreamBeginLumi(this, &NVProfilerService::postStreamBeginLumi);
0370
0371   // these signal pair are NOT guaranteed to be called by the same thread
0372   registry.watchPreStreamEndLumi(this, &NVProfilerService::preStreamEndLumi);
0373   registry.watchPostStreamEndLumi(this, &NVProfilerService::postStreamEndLumi);
0374
0375   // these signal pair are NOT guaranteed to be called by the same thread
0376   registry.watchPreEvent(this, &NVProfilerService::preEvent);
0377   registry.watchPostEvent(this, &NVProfilerService::postEvent);
0378
0379   // these signal pair are NOT guaranteed to be called by the same thread
0380   registry.watchPrePathEvent(this, &NVProfilerService::prePathEvent);
0381   registry.watchPostPathEvent(this, &NVProfilerService::postPathEvent);
0382
0383   if (showModulePrefetching_) {
0384     // these signal pair are NOT guaranteed to be called by the same thread
0385     registry.watchPreModuleEventPrefetching(this, &NVProfilerService::preModuleEventPrefetching);
0386     registry.watchPostModuleEventPrefetching(this, &NVProfilerService::postModuleEventPrefetching);
0387   }
0388
0389   // these signal pair are guaranteed to be called by the same thread
0390   registry.watchPreOpenFile(this, &NVProfilerService::preOpenFile);
0391   registry.watchPostOpenFile(this, &NVProfilerService::postOpenFile);
0392
0393   // these signal pair are guaranteed to be called by the same thread
0394   registry.watchPreCloseFile(this, &NVProfilerService::preCloseFile);
0395   registry.watchPostCloseFile(this, &NVProfilerService::postCloseFile);
0396
0397   // these signal pair are guaranteed to be called by the same thread
0398   registry.watchPreSourceConstruction(this, &NVProfilerService::preSourceConstruction);
0399   registry.watchPostSourceConstruction(this, &NVProfilerService::postSourceConstruction);
0400
0401   // these signal pair are guaranteed to be called by the same thread
0402   registry.watchPreSourceRun(this, &NVProfilerService::preSourceRun);
0403   registry.watchPostSourceRun(this, &NVProfilerService::postSourceRun);
0404
0405   // these signal pair are guaranteed to be called by the same thread
0406   registry.watchPreSourceLumi(this, &NVProfilerService::preSourceLumi);
0407   registry.watchPostSourceLumi(this, &NVProfilerService::postSourceLumi);
0408
0409   // these signal pair are guaranteed to be called by the same thread
0410   registry.watchPreSourceEvent(this, &NVProfilerService::preSourceEvent);
0411   registry.watchPostSourceEvent(this, &NVProfilerService::postSourceEvent);
0412
0413   // these signal pair are guaranteed to be called by the same thread
0414   registry.watchPreModuleConstruction(this, &NVProfilerService::preModuleConstruction);
0415   registry.watchPostModuleConstruction(this, &NVProfilerService::postModuleConstruction);
0416
0417   // these signal pair are guaranteed to be called by the same thread
0418   registry.watchPreModuleDestruction(this, &NVProfilerService::preModuleDestruction);
0419   registry.watchPostModuleDestruction(this, &NVProfilerService::postModuleDestruction);
0420
0421   // these signal pair are guaranteed to be called by the same thread
0422   registry.watchPreModuleBeginJob(this, &NVProfilerService::preModuleBeginJob);
0423   registry.watchPostModuleBeginJob(this, &NVProfilerService::postModuleBeginJob);
0424
0425   // these signal pair are guaranteed to be called by the same thread
0426   registry.watchPreModuleEndJob(this, &NVProfilerService::preModuleEndJob);
0427   registry.watchPostModuleEndJob(this, &NVProfilerService::postModuleEndJob);
0428
0429   // these signal pair are guaranteed to be called by the same thread
0430   registry.watchPreModuleBeginStream(this, &NVProfilerService::preModuleBeginStream);
0431   registry.watchPostModuleBeginStream(this, &NVProfilerService::postModuleBeginStream);
0432
0433   // these signal pair are guaranteed to be called by the same thread
0434   registry.watchPreModuleEndStream(this, &NVProfilerService::preModuleEndStream);
0435   registry.watchPostModuleEndStream(this, &NVProfilerService::postModuleEndStream);
0436
0437   // these signal pair are guaranteed to be called by the same thread
0438   registry.watchPreModuleGlobalBeginRun(this, &NVProfilerService::preModuleGlobalBeginRun);
0439   registry.watchPostModuleGlobalBeginRun(this, &NVProfilerService::postModuleGlobalBeginRun);
0440
0441   // these signal pair are guaranteed to be called by the same thread
0442   registry.watchPreModuleGlobalEndRun(this, &NVProfilerService::preModuleGlobalEndRun);
0443   registry.watchPostModuleGlobalEndRun(this, &NVProfilerService::postModuleGlobalEndRun);
0444
0445   // these signal pair are guaranteed to be called by the same thread
0446   registry.watchPreModuleGlobalBeginLumi(this, &NVProfilerService::preModuleGlobalBeginLumi);
0447   registry.watchPostModuleGlobalBeginLumi(this, &NVProfilerService::postModuleGlobalBeginLumi);
0448
0449   // these signal pair are guaranteed to be called by the same thread
0450   registry.watchPreModuleGlobalEndLumi(this, &NVProfilerService::preModuleGlobalEndLumi);
0451   registry.watchPostModuleGlobalEndLumi(this, &NVProfilerService::postModuleGlobalEndLumi);
0452
0453   // these signal pair are guaranteed to be called by the same thread
0454   registry.watchPreModuleStreamBeginRun(this, &NVProfilerService::preModuleStreamBeginRun);
0455   registry.watchPostModuleStreamBeginRun(this, &NVProfilerService::postModuleStreamBeginRun);
0456
0457   // these signal pair are guaranteed to be called by the same thread
0458   registry.watchPreModuleStreamEndRun(this, &NVProfilerService::preModuleStreamEndRun);
0459   registry.watchPostModuleStreamEndRun(this, &NVProfilerService::postModuleStreamEndRun);
0460
0461   // these signal pair are guaranteed to be called by the same thread
0462   registry.watchPreModuleStreamBeginLumi(this, &NVProfilerService::preModuleStreamBeginLumi);
0463   registry.watchPostModuleStreamBeginLumi(this, &NVProfilerService::postModuleStreamBeginLumi);
0464
0465   // these signal pair are guaranteed to be called by the same thread
0466   registry.watchPreModuleStreamEndLumi(this, &NVProfilerService::preModuleStreamEndLumi);
0467   registry.watchPostModuleStreamEndLumi(this, &NVProfilerService::postModuleStreamEndLumi);
0468
0469   // these signal pair are guaranteed to be called by the same thread
0470   registry.watchPreModuleEventAcquire(this, &NVProfilerService::preModuleEventAcquire);
0471   registry.watchPostModuleEventAcquire(this, &NVProfilerService::postModuleEventAcquire);
0472
0473   // these signal pair are guaranteed to be called by the same thread
0474   registry.watchPreModuleEvent(this, &NVProfilerService::preModuleEvent);
0475   registry.watchPostModuleEvent(this, &NVProfilerService::postModuleEvent);
0476
0477   // these signal pair are guaranteed to be called by the same thread
0478   registry.watchPreModuleEventDelayedGet(this, &NVProfilerService::preModuleEventDelayedGet);
0479   registry.watchPostModuleEventDelayedGet(this, &NVProfilerService::postModuleEventDelayedGet);
0480
0481   // these signal pair are guaranteed to be called by the same thread
0482   registry.watchPreEventReadFromSource(this, &NVProfilerService::preEventReadFromSource);
0483   registry.watchPostEventReadFromSource(this, &NVProfilerService::postEventReadFromSource);
0484
0485   if (showModulePrefetching_) {
0486     // these signal pair are NOT guaranteed to be called by the same thread
0487     registry.watchPreModuleTransformPrefetching(this, &NVProfilerService::preModuleTransformPrefetching);
0488     registry.watchPostModuleTransformPrefetching(this, &NVProfilerService::postModuleTransformPrefetching);
0489   }
0490
0491   // these signal pair are guaranteed to be called by the same thread
0492   registry.watchPreModuleTransform(this, &NVProfilerService::preModuleTransform);
0493   registry.watchPostModuleTransform(this, &NVProfilerService::postModuleTransform);
0494
0495   // these signal pair are guaranteed to be called by the same thread
0496   registry.watchPreModuleTransformAcquiring(this, &NVProfilerService::preModuleTransformAcquiring);
0497   registry.watchPostModuleTransformAcquiring(this, &NVProfilerService::postModuleTransformAcquiring);
0498 }
0499
0500 NVProfilerService::~NVProfilerService() {
0501   for (unsigned int sid = 0; sid < stream_domain_.size(); ++sid) {
0502     nvtxDomainDestroy(stream_domain_[sid]);
0503   }
0504   nvtxDomainDestroy(global_domain_);
0505   cudaProfilerStop();
0506 }
0507
0508 void NVProfilerService::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0509   edm::ParameterSetDescription desc;
0510   desc.addUntracked<std::vector<std::string>>("highlightModules", {})->setComment("");
0511   desc.addUntracked<bool>("showModulePrefetching", false)
0512       ->setComment("Show the stack of dependencies that requested to run a module.");
0513   desc.addUntracked<bool>("skipFirstEvent", false)
0514       ->setComment(
0515           "Start profiling after the first event has completed.\nWith multiple streams, ignore transitions belonging "
0516           "to events started in parallel to the first event.\nRequires running nvprof with the '--profile-from-start "
0517           "off' option.");
0518   descriptions.add("NVProfilerService", desc);
0519   descriptions.setComment(R"(This Service provides CMSSW-aware annotations to nvprof/nvvm.
0520
0521 Notes on nvprof options:
0522   - the option '--profile-from-start off' should be used if skipFirstEvent is True.
0523   - the option '--cpu-profiling on' currently results in cmsRun being stuck at the beginning of the job.
0524   - the option '--cpu-thread-tracing on' is not compatible with jemalloc, and should only be used with cmsRunGlibC.)");
0525 }
0526
0527 void NVProfilerService::preallocate(edm::service::SystemBounds const& bounds) {
0528   std::stringstream out;
0529   out << "preallocate: " << bounds.maxNumberOfConcurrentRuns() << " concurrent runs, "
0530       << bounds.maxNumberOfConcurrentLuminosityBlocks() << " luminosity sections, " << bounds.maxNumberOfStreams()
0531       << " streams\nrunning on " << bounds.maxNumberOfThreads() << " threads";
0532   nvtxDomainMark(global_domain_, out.str().c_str());
0533
0534   auto concurrentStreams = bounds.maxNumberOfStreams();
0535   // create the NVTX domains for per-EDM-stream transitions
0536   stream_domain_.resize(concurrentStreams);
0537   for (unsigned int sid = 0; sid < concurrentStreams; ++sid) {
0538     stream_domain_[sid] = nvtxDomainCreate(fmt::sprintf("EDM Stream %d", sid).c_str());
0539   }
0540
0541   event_.resize(concurrentStreams);
0542   stream_modules_.resize(concurrentStreams);
0543   for (auto& modulesForOneStream : stream_modules_) {
0544     modulesForOneStream.resize(global_modules_.size(), nvtxInvalidRangeId);
0545   }
0546
0547   if (skipFirstEvent_) {
0548     globalFirstEventDone_ = false;
0549     std::vector<std::atomic<bool>> tmp(concurrentStreams);
0550     for (auto& element : tmp)
0551       std::atomic_init(&element, false);
0552     streamFirstEventDone_ = std::move(tmp);
0553   }
0554 }
0555
0556 void NVProfilerService::preBeginJob(edm::ProcessContext const& context) {
0557   nvtxDomainMark(global_domain_, "preBeginJob");
0558 }
0559
0560 void NVProfilerService::postBeginJob() {
0561   if (not skipFirstEvent_ or globalFirstEventDone_) {
0562     nvtxDomainMark(global_domain_, "postBeginJob");
0563   }
0564 }
0565
0566 void NVProfilerService::lookupInitializationComplete(edm::PathsAndConsumesOfModulesBase const&,
0567                                                      edm::ProcessContext const&) {
0568   nvtxDomainMark(global_domain_, "lookupInitializationComplete");
0569 }
0570
0571 void NVProfilerService::postEndJob() {
0572   if (not skipFirstEvent_ or globalFirstEventDone_) {
0573     nvtxDomainMark(global_domain_, "postEndJob");
0574   }
0575 }
0576
0577 void NVProfilerService::preSourceEvent(edm::StreamID sid) {
0578   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0579     nvtxDomainRangePush(stream_domain_[sid], "source");
0580   }
0581 }
0582
0583 void NVProfilerService::postSourceEvent(edm::StreamID sid) {
0584   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0585     nvtxDomainRangePop(stream_domain_[sid]);
0586   }
0587 }
0588
0589 void NVProfilerService::preSourceLumi(edm::LuminosityBlockIndex index) {
0590   if (not skipFirstEvent_ or globalFirstEventDone_) {
0591     nvtxDomainRangePush(global_domain_, "source lumi");
0592   }
0593 }
0594
0595 void NVProfilerService::postSourceLumi(edm::LuminosityBlockIndex index) {
0596   if (not skipFirstEvent_ or globalFirstEventDone_) {
0597     nvtxDomainRangePop(global_domain_);
0598   }
0599 }
0600
0601 void NVProfilerService::preSourceRun(edm::RunIndex index) {
0602   if (not skipFirstEvent_ or globalFirstEventDone_) {
0603     nvtxDomainRangePush(global_domain_, "source run");
0604   }
0605 }
0606
0607 void NVProfilerService::postSourceRun(edm::RunIndex index) {
0608   if (not skipFirstEvent_ or globalFirstEventDone_) {
0609     nvtxDomainRangePop(global_domain_);
0610   }
0611 }
0612
0613 void NVProfilerService::preOpenFile(std::string const& lfn) {
0614   if (not skipFirstEvent_ or globalFirstEventDone_) {
0615     nvtxDomainRangePush(global_domain_, ("open file "s + lfn).c_str());
0616   }
0617 }
0618
0619 void NVProfilerService::postOpenFile(std::string const& lfn) {
0620   if (not skipFirstEvent_ or globalFirstEventDone_) {
0621     nvtxDomainRangePop(global_domain_);
0622   }
0623 }
0624
0625 void NVProfilerService::preCloseFile(std::string const& lfn) {
0626   if (not skipFirstEvent_ or globalFirstEventDone_) {
0627     nvtxDomainRangePush(global_domain_, ("close file "s + lfn).c_str());
0628   }
0629 }
0630
0631 void NVProfilerService::postCloseFile(std::string const& lfn) {
0632   if (not skipFirstEvent_ or globalFirstEventDone_) {
0633     nvtxDomainRangePop(global_domain_);
0634   }
0635 }
0636
0637 void NVProfilerService::preModuleBeginStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0638   auto sid = sc.streamID();
0639   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0640     auto mid = mcc.moduleDescription()->id();
0641     auto const& label = mcc.moduleDescription()->moduleLabel();
0642     auto const& msg = label + " begin stream";
0643     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0644     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0645   }
0646 }
0647
0648 void NVProfilerService::postModuleBeginStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0649   auto sid = sc.streamID();
0650   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0651     auto mid = mcc.moduleDescription()->id();
0652     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0653     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0654   }
0655 }
0656
0657 void NVProfilerService::preModuleEndStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0658   auto sid = sc.streamID();
0659   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0660     auto mid = mcc.moduleDescription()->id();
0661     auto const& label = mcc.moduleDescription()->moduleLabel();
0662     auto const& msg = label + " end stream";
0663     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0664     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0665   }
0666 }
0667
0668 void NVProfilerService::postModuleEndStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0669   auto sid = sc.streamID();
0670   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0671     auto mid = mcc.moduleDescription()->id();
0672     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0673     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0674   }
0675 }
0676
0677 void NVProfilerService::preGlobalBeginRun(edm::GlobalContext const& gc) {
0678   if (not skipFirstEvent_ or globalFirstEventDone_) {
0679     nvtxDomainRangePush(global_domain_, "global begin run");
0680   }
0681 }
0682
0683 void NVProfilerService::postGlobalBeginRun(edm::GlobalContext const& gc) {
0684   if (not skipFirstEvent_ or globalFirstEventDone_) {
0685     nvtxDomainRangePop(global_domain_);
0686   }
0687 }
0688
0689 void NVProfilerService::preGlobalEndRun(edm::GlobalContext const& gc) {
0690   if (not skipFirstEvent_ or globalFirstEventDone_) {
0691     nvtxDomainRangePush(global_domain_, "global end run");
0692   }
0693 }
0694
0695 void NVProfilerService::postGlobalEndRun(edm::GlobalContext const& gc) {
0696   if (not skipFirstEvent_ or globalFirstEventDone_) {
0697     nvtxDomainRangePop(global_domain_);
0698   }
0699 }
0700
0701 void NVProfilerService::preStreamBeginRun(edm::StreamContext const& sc) {
0702   auto sid = sc.streamID();
0703   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0704     nvtxDomainRangePush(stream_domain_[sid], "stream begin run");
0705   }
0706 }
0707
0708 void NVProfilerService::postStreamBeginRun(edm::StreamContext const& sc) {
0709   auto sid = sc.streamID();
0710   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0711     nvtxDomainRangePop(stream_domain_[sid]);
0712   }
0713 }
0714
0715 void NVProfilerService::preStreamEndRun(edm::StreamContext const& sc) {
0716   auto sid = sc.streamID();
0717   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0718     nvtxDomainRangePush(stream_domain_[sid], "stream end run");
0719   }
0720 }
0721
0722 void NVProfilerService::postStreamEndRun(edm::StreamContext const& sc) {
0723   auto sid = sc.streamID();
0724   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0725     nvtxDomainRangePop(stream_domain_[sid]);
0726   }
0727 }
0728
0729 void NVProfilerService::preGlobalBeginLumi(edm::GlobalContext const& gc) {
0730   if (not skipFirstEvent_ or globalFirstEventDone_) {
0731     nvtxDomainRangePush(global_domain_, "global begin lumi");
0732   }
0733 }
0734
0735 void NVProfilerService::postGlobalBeginLumi(edm::GlobalContext const& gc) {
0736   if (not skipFirstEvent_ or globalFirstEventDone_) {
0737     nvtxDomainRangePop(global_domain_);
0738   }
0739 }
0740
0741 void NVProfilerService::preGlobalEndLumi(edm::GlobalContext const& gc) {
0742   if (not skipFirstEvent_ or globalFirstEventDone_) {
0743     nvtxDomainRangePush(global_domain_, "global end lumi");
0744   }
0745 }
0746
0747 void NVProfilerService::postGlobalEndLumi(edm::GlobalContext const& gc) {
0748   if (not skipFirstEvent_ or globalFirstEventDone_) {
0749     nvtxDomainRangePop(global_domain_);
0750   }
0751 }
0752
0753 void NVProfilerService::preStreamBeginLumi(edm::StreamContext const& sc) {
0754   auto sid = sc.streamID();
0755   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0756     nvtxDomainRangePush(stream_domain_[sid], "stream begin lumi");
0757   }
0758 }
0759
0760 void NVProfilerService::postStreamBeginLumi(edm::StreamContext const& sc) {
0761   auto sid = sc.streamID();
0762   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0763     nvtxDomainRangePop(stream_domain_[sid]);
0764   }
0765 }
0766
0767 void NVProfilerService::preStreamEndLumi(edm::StreamContext const& sc) {
0768   auto sid = sc.streamID();
0769   nvtxDomainRangePush(stream_domain_[sid], "stream end lumi");
0770 }
0771
0772 void NVProfilerService::postStreamEndLumi(edm::StreamContext const& sc) {
0773   auto sid = sc.streamID();
0774   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0775     nvtxDomainRangePop(stream_domain_[sid]);
0776   }
0777 }
0778
0779 void NVProfilerService::preEvent(edm::StreamContext const& sc) {
0780   auto sid = sc.streamID();
0781   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0782     event_[sid] = nvtxDomainRangeStartColor(stream_domain_[sid], "event", nvtxDarkGreen);
0783   }
0784 }
0785
0786 void NVProfilerService::postEvent(edm::StreamContext const& sc) {
0787   auto sid = sc.streamID();
0788   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0789     nvtxDomainRangeEnd(stream_domain_[sid], event_[sid]);
0790     event_[sid] = nvtxInvalidRangeId;
0791   } else {
0792     streamFirstEventDone_[sid] = true;
0793     auto identity = [](bool x) { return x; };
0794     if (std::all_of(streamFirstEventDone_.begin(), streamFirstEventDone_.end(), identity)) {
0795       bool expected = false;
0796       if (globalFirstEventDone_.compare_exchange_strong(expected, true))
0797         cudaProfilerStart();
0798     }
0799   }
0800 }
0801
0802 void NVProfilerService::prePathEvent(edm::StreamContext const& sc, edm::PathContext const& pc) {
0803   auto sid = sc.streamID();
0804   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0805     nvtxDomainMark(global_domain_, ("before path "s + pc.pathName()).c_str());
0806   }
0807 }
0808
0809 void NVProfilerService::postPathEvent(edm::StreamContext const& sc,
0810                                       edm::PathContext const& pc,
0811                                       edm::HLTPathStatus const& hlts) {
0812   auto sid = sc.streamID();
0813   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0814     nvtxDomainMark(global_domain_, ("after path "s + pc.pathName()).c_str());
0815   }
0816 }
0817
0818 void NVProfilerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0819   auto sid = sc.streamID();
0820   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0821     auto mid = mcc.moduleDescription()->id();
0822     auto const& label = mcc.moduleDescription()->moduleLabel();
0823     auto const& msg = label + " prefetching";
0824     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0825     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColorLight(label));
0826   }
0827 }
0828
0829 void NVProfilerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0830   auto sid = sc.streamID();
0831   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0832     auto mid = mcc.moduleDescription()->id();
0833     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0834     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0835   }
0836 }
0837
0838 void NVProfilerService::preModuleConstruction(edm::ModuleDescription const& desc) {
0839   auto mid = desc.id();
0840   global_modules_.grow_to_at_least(mid + 1);
0841
0842   // This normally does nothing because stream_modules_ is empty when
0843   // called. But there is a rare case when a looper is used that replacement
0844   // modules can be constructed at end of loop. I'm not sure if that feature
0845   // is ever actually used but just to be safe...
0846   for (auto& modulesForOneStream : stream_modules_) {
0847     modulesForOneStream.resize(global_modules_.size(), nvtxInvalidRangeId);
0848   }
0849
0850   if (not skipFirstEvent_) {
0851     auto const& label = desc.moduleLabel();
0852     auto const& msg = label + " construction";
0853     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0854   }
0855 }
0856
0857 void NVProfilerService::postModuleConstruction(edm::ModuleDescription const& desc) {
0858   if (not skipFirstEvent_) {
0859     auto mid = desc.id();
0860     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0861     global_modules_[mid] = nvtxInvalidRangeId;
0862   }
0863 }
0864
0865 void NVProfilerService::preModuleDestruction(edm::ModuleDescription const& desc) {
0866   if (not skipFirstEvent_) {
0867     auto mid = desc.id();
0868     auto const& label = desc.moduleLabel();
0869     auto const& msg = label + " destruction";
0870     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0871   }
0872 }
0873
0874 void NVProfilerService::postModuleDestruction(edm::ModuleDescription const& desc) {
0875   if (not skipFirstEvent_) {
0876     auto mid = desc.id();
0877     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0878     global_modules_[mid] = nvtxInvalidRangeId;
0879   }
0880 }
0881
0882 void NVProfilerService::preModuleBeginJob(edm::ModuleDescription const& desc) {
0883   if (not skipFirstEvent_) {
0884     auto mid = desc.id();
0885     auto const& label = desc.moduleLabel();
0886     auto const& msg = label + " begin job";
0887     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0888   }
0889 }
0890
0891 void NVProfilerService::postModuleBeginJob(edm::ModuleDescription const& desc) {
0892   if (not skipFirstEvent_) {
0893     auto mid = desc.id();
0894     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0895     global_modules_[mid] = nvtxInvalidRangeId;
0896   }
0897 }
0898
0899 void NVProfilerService::preModuleEndJob(edm::ModuleDescription const& desc) {
0900   if (not skipFirstEvent_ or globalFirstEventDone_) {
0901     auto mid = desc.id();
0902     auto const& label = desc.moduleLabel();
0903     auto const& msg = label + " end job";
0904     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0905   }
0906 }
0907
0908 void NVProfilerService::postModuleEndJob(edm::ModuleDescription const& desc) {
0909   if (not skipFirstEvent_ or globalFirstEventDone_) {
0910     auto mid = desc.id();
0911     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0912     global_modules_[mid] = nvtxInvalidRangeId;
0913   }
0914 }
0915
0916 void NVProfilerService::preModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0917   auto sid = sc.streamID();
0918   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0919     auto mid = mcc.moduleDescription()->id();
0920     auto const& label = mcc.moduleDescription()->moduleLabel();
0921     auto const& msg = label + " acquire";
0922     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0923     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0924   }
0925 }
0926
0927 void NVProfilerService::postModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0928   auto sid = sc.streamID();
0929   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0930     auto mid = mcc.moduleDescription()->id();
0931     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0932     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0933   }
0934 }
0935
0936 void NVProfilerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0937   auto sid = sc.streamID();
0938   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0939     auto mid = mcc.moduleDescription()->id();
0940     auto const& label = mcc.moduleDescription()->moduleLabel();
0941     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0942     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], label.c_str(), labelColor(label));
0943   }
0944 }
0945
0946 void NVProfilerService::postModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0947   auto sid = sc.streamID();
0948   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0949     auto mid = mcc.moduleDescription()->id();
0950     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0951     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0952   }
0953 }
0954
0955 void NVProfilerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0956   /* FIXME
0957   auto sid = sc.streamID();
0958   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0959     auto mid = mcc.moduleDescription()->id();
0960     auto const & label = mcc.moduleDescription()->moduleLabel();
0961     auto const & msg = label + " delayed get";
0962     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0963     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], label.c_str(), labelColorLight(label));
0964   }
0965   */
0966 }
0967
0968 void NVProfilerService::postModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0969   /* FIXME
0970   auto sid = sc.streamID();
0971   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0972     auto mid = mcc.moduleDescription()->id();
0973     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0974     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0975   }
0976   */
0977 }
0978
0979 void NVProfilerService::preEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0980   /* FIXME
0981   auto sid = sc.streamID();
0982   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0983     auto mid = mcc.moduleDescription()->id();
0984     auto const & label = mcc.moduleDescription()->moduleLabel();
0985     auto const & msg = label + " read from source";
0986     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0987     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColorLight(label));
0988   }
0989   */
0990 }
0991
0992 void NVProfilerService::postEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0993   /* FIXME
0994   auto sid = sc.streamID();
0995   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0996     auto mid = mcc.moduleDescription()->id();
0997     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0998     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0999   }
1000   */
1001 }
1002
1003 void NVProfilerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1004   auto sid = sc.streamID();
1005   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1006     auto mid = mcc.moduleDescription()->id();
1007     auto const& label = mcc.moduleDescription()->moduleLabel();
1008     auto const& msg = label + " stream begin run";
1009     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1010     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1011   }
1012 }
1013
1014 void NVProfilerService::postModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1015   auto sid = sc.streamID();
1016   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1017     auto mid = mcc.moduleDescription()->id();
1018     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1019     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1020   }
1021 }
1022
1023 void NVProfilerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1024   auto sid = sc.streamID();
1025   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1026     auto mid = mcc.moduleDescription()->id();
1027     auto const& label = mcc.moduleDescription()->moduleLabel();
1028     auto const& msg = label + " stream end run";
1029     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1030     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1031   }
1032 }
1033
1034 void NVProfilerService::postModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1035   auto sid = sc.streamID();
1036   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1037     auto mid = mcc.moduleDescription()->id();
1038     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1039     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1040   }
1041 }
1042
1043 void NVProfilerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1044   auto sid = sc.streamID();
1045   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1046     auto mid = mcc.moduleDescription()->id();
1047     auto const& label = mcc.moduleDescription()->moduleLabel();
1048     auto const& msg = label + " stream begin lumi";
1049     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1050     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1051   }
1052 }
1053
1054 void NVProfilerService::postModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1055   auto sid = sc.streamID();
1056   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1057     auto mid = mcc.moduleDescription()->id();
1058     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1059     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1060   }
1061 }
1062
1063 void NVProfilerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1064   auto sid = sc.streamID();
1065   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1066     auto mid = mcc.moduleDescription()->id();
1067     auto const& label = mcc.moduleDescription()->moduleLabel();
1068     auto const& msg = label + " stream end lumi";
1069     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1070     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1071   }
1072 }
1073
1074 void NVProfilerService::postModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1075   auto sid = sc.streamID();
1076   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1077     auto mid = mcc.moduleDescription()->id();
1078     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1079     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1080   }
1081 }
1082
1083 void NVProfilerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1084   if (not skipFirstEvent_ or globalFirstEventDone_) {
1085     auto mid = mcc.moduleDescription()->id();
1086     auto const& label = mcc.moduleDescription()->moduleLabel();
1087     auto const& msg = label + " global begin run";
1088     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1089   }
1090 }
1091
1092 void NVProfilerService::postModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1093   if (not skipFirstEvent_ or globalFirstEventDone_) {
1094     auto mid = mcc.moduleDescription()->id();
1095     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1096     global_modules_[mid] = nvtxInvalidRangeId;
1097   }
1098 }
1099
1100 void NVProfilerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1101   if (not skipFirstEvent_ or globalFirstEventDone_) {
1102     auto mid = mcc.moduleDescription()->id();
1103     auto const& label = mcc.moduleDescription()->moduleLabel();
1104     auto const& msg = label + " global end run";
1105     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1106   }
1107 }
1108
1109 void NVProfilerService::postModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1110   if (not skipFirstEvent_ or globalFirstEventDone_) {
1111     auto mid = mcc.moduleDescription()->id();
1112     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1113     global_modules_[mid] = nvtxInvalidRangeId;
1114   }
1115 }
1116
1117 void NVProfilerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1118   if (not skipFirstEvent_ or globalFirstEventDone_) {
1119     auto mid = mcc.moduleDescription()->id();
1120     auto const& label = mcc.moduleDescription()->moduleLabel();
1121     auto const& msg = label + " global begin lumi";
1122     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1123   }
1124 }
1125
1126 void NVProfilerService::postModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1127   if (not skipFirstEvent_ or globalFirstEventDone_) {
1128     auto mid = mcc.moduleDescription()->id();
1129     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1130     global_modules_[mid] = nvtxInvalidRangeId;
1131   }
1132 }
1133
1134 void NVProfilerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1135   if (not skipFirstEvent_ or globalFirstEventDone_) {
1136     auto mid = mcc.moduleDescription()->id();
1137     auto const& label = mcc.moduleDescription()->moduleLabel();
1138     auto const& msg = label + " global end lumi";
1139     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1140   }
1141 }
1142
1143 void NVProfilerService::postModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1144   if (not skipFirstEvent_ or globalFirstEventDone_) {
1145     auto mid = mcc.moduleDescription()->id();
1146     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1147     global_modules_[mid] = nvtxInvalidRangeId;
1148   }
1149 }
1150
1151 void NVProfilerService::preSourceConstruction(edm::ModuleDescription const& desc) {
1152   auto mid = desc.id();
1153   global_modules_.grow_to_at_least(mid + 1);
1154
1155   if (not skipFirstEvent_) {
1156     auto const& label = desc.moduleLabel();
1157     auto const& msg = label + " construction";
1158     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1159   }
1160 }
1161
1162 void NVProfilerService::postSourceConstruction(edm::ModuleDescription const& desc) {
1163   if (not skipFirstEvent_) {
1164     auto mid = desc.id();
1165     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1166     global_modules_[mid] = nvtxInvalidRangeId;
1167   }
1168 }
1169
1170 void NVProfilerService::preModuleTransformPrefetching(edm::StreamContext const& sc,
1171                                                       edm::ModuleCallingContext const& mcc) {
1172   auto sid = sc.streamID();
1173   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1174     auto mid = mcc.moduleDescription()->id();
1175     auto const& label = mcc.moduleDescription()->moduleLabel();
1176     auto const& msg = label + " transform prefetching";
1177     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1178     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColorLight(label));
1179   }
1180 }
1181
1182 void NVProfilerService::postModuleTransformPrefetching(edm::StreamContext const& sc,
1183                                                        edm::ModuleCallingContext const& mcc) {
1184   auto sid = sc.streamID();
1185   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1186     auto mid = mcc.moduleDescription()->id();
1187     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1188     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1189   }
1190 }
1191
1192 void NVProfilerService::preModuleTransformAcquiring(edm::StreamContext const& sc,
1193                                                     edm::ModuleCallingContext const& mcc) {
1194   auto sid = sc.streamID();
1195   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1196     auto mid = mcc.moduleDescription()->id();
1197     auto const& label = mcc.moduleDescription()->moduleLabel();
1198     auto const& msg = label + " transform acquire";
1199     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1200     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1201   }
1202 }
1203
1204 void NVProfilerService::postModuleTransformAcquiring(edm::StreamContext const& sc,
1205                                                      edm::ModuleCallingContext const& mcc) {
1206   auto sid = sc.streamID();
1207   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1208     auto mid = mcc.moduleDescription()->id();
1209     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1210     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1211   }
1212 }
1213
1214 void NVProfilerService::preModuleTransform(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1215   auto sid = sc.streamID();
1216   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1217     auto mid = mcc.moduleDescription()->id();
1218     auto const& label = mcc.moduleDescription()->moduleLabel();
1219     auto const& msg = label + " transform";
1220     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1221     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1222   }
1223 }
1224
1225 void NVProfilerService::postModuleTransform(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1226   auto sid = sc.streamID();
1227   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1228     auto mid = mcc.moduleDescription()->id();
1229     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1230     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1231   }
1232 }
1233
1234 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
1235 DEFINE_FWK_SERVICE(NVProfilerService);