CUDAServices/plugins/NVProfilerService.cc

0001 // -*- C++ -*-
0002 //
0003 // Package:     HeterogeneousCore/CUDAServices
0004 // Class  :     NVProfilerService
0005
0006 #include <algorithm>
0007 #include <iostream>
0008 #include <sstream>
0009 #include <string>
0010 #include <vector>
0011
0012 #include <oneapi/tbb/concurrent_vector.h>
0013
0014 #include <fmt/printf.h>
0015
0016 #include <cuda_profiler_api.h>
0017 #include <nvToolsExt.h>
0018
0019 #include "DataFormats/Common/interface/HLTPathStatus.h"
0020 #include "DataFormats/Provenance/interface/EventID.h"
0021 #include "DataFormats/Provenance/interface/LuminosityBlockID.h"
0022 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0023 #include "DataFormats/Provenance/interface/RunID.h"
0024 #include "DataFormats/Provenance/interface/Timestamp.h"
0025 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0026 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0027 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0028 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0029 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0030 #include "FWCore/ServiceRegistry/interface/ConsumesInfo.h"
0031 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0032 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0033 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0034 #include "FWCore/ServiceRegistry/interface/PathsAndConsumesOfModulesBase.h"
0035 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0036 #include "FWCore/ServiceRegistry/interface/Service.h"
0037 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0038 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0039 #include "FWCore/Utilities/interface/BranchType.h"
0040 #include "FWCore/Utilities/interface/Exception.h"
0041 #include "FWCore/Utilities/interface/ProductKindOfType.h"
0042 #include "FWCore/Utilities/interface/TimeOfDay.h"
0043 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0044
0045 using namespace std::string_literals;
0046
0047 namespace {
0048   int nvtxDomainRangePush(nvtxDomainHandle_t domain, const char* message) {
0049     nvtxEventAttributes_t eventAttrib = {};
0050     eventAttrib.version = NVTX_VERSION;
0051     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0052     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0053     eventAttrib.message.ascii = message;
0054     return nvtxDomainRangePushEx(domain, &eventAttrib);
0055   }
0056
0057   __attribute__((unused)) int nvtxDomainRangePushColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0058     nvtxEventAttributes_t eventAttrib = {};
0059     eventAttrib.version = NVTX_VERSION;
0060     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0061     eventAttrib.colorType = NVTX_COLOR_ARGB;
0062     eventAttrib.color = color;
0063     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0064     eventAttrib.message.ascii = message;
0065     return nvtxDomainRangePushEx(domain, &eventAttrib);
0066   }
0067
0068   __attribute__((unused)) nvtxRangeId_t nvtxDomainRangeStart(nvtxDomainHandle_t domain, const char* message) {
0069     nvtxEventAttributes_t eventAttrib = {};
0070     eventAttrib.version = NVTX_VERSION;
0071     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0072     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0073     eventAttrib.message.ascii = message;
0074     return nvtxDomainRangeStartEx(domain, &eventAttrib);
0075   }
0076
0077   nvtxRangeId_t nvtxDomainRangeStartColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0078     nvtxEventAttributes_t eventAttrib = {};
0079     eventAttrib.version = NVTX_VERSION;
0080     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0081     eventAttrib.colorType = NVTX_COLOR_ARGB;
0082     eventAttrib.color = color;
0083     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0084     eventAttrib.message.ascii = message;
0085     return nvtxDomainRangeStartEx(domain, &eventAttrib);
0086   }
0087
0088   void nvtxDomainMark(nvtxDomainHandle_t domain, const char* message) {
0089     nvtxEventAttributes_t eventAttrib = {};
0090     eventAttrib.version = NVTX_VERSION;
0091     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0092     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0093     eventAttrib.message.ascii = message;
0094     nvtxDomainMarkEx(domain, &eventAttrib);
0095   }
0096
0097   __attribute__((unused)) void nvtxDomainMarkColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0098     nvtxEventAttributes_t eventAttrib = {};
0099     eventAttrib.version = NVTX_VERSION;
0100     eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0101     eventAttrib.colorType = NVTX_COLOR_ARGB;
0102     eventAttrib.color = color;
0103     eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0104     eventAttrib.message.ascii = message;
0105     nvtxDomainMarkEx(domain, &eventAttrib);
0106   }
0107
0108   enum {
0109     nvtxBlack = 0x00000000,
0110     nvtxRed = 0x00ff0000,
0111     nvtxDarkGreen = 0x00009900,
0112     nvtxGreen = 0x0000ff00,
0113     nvtxLightGreen = 0x00ccffcc,
0114     nvtxBlue = 0x000000ff,
0115     nvtxAmber = 0x00ffbf00,
0116     nvtxLightAmber = 0x00fff2cc,
0117     nvtxWhite = 0x00ffffff
0118   };
0119
0120   constexpr nvtxRangeId_t nvtxInvalidRangeId = 0xfffffffffffffffful;
0121 }  // namespace
0122
0123 class NVProfilerService {
0124 public:
0125   NVProfilerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0126   ~NVProfilerService();
0127
0128   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0129
0130   void preallocate(edm::service::SystemBounds const&);
0131
0132   // these signal pair are NOT guaranteed to be called by the same thread
0133   void preBeginJob(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0134   void postBeginJob();
0135
0136   // there is no preEndJob() signal
0137   void postEndJob();
0138
0139   // these signal pair are NOT guaranteed to be called by the same thread
0140   void preGlobalBeginRun(edm::GlobalContext const&);
0141   void postGlobalBeginRun(edm::GlobalContext const&);
0142
0143   // these signal pair are NOT guaranteed to be called by the same thread
0144   void preGlobalEndRun(edm::GlobalContext const&);
0145   void postGlobalEndRun(edm::GlobalContext const&);
0146
0147   // these signal pair are NOT guaranteed to be called by the same thread
0148   void preStreamBeginRun(edm::StreamContext const&);
0149   void postStreamBeginRun(edm::StreamContext const&);
0150
0151   // these signal pair are NOT guaranteed to be called by the same thread
0152   void preStreamEndRun(edm::StreamContext const&);
0153   void postStreamEndRun(edm::StreamContext const&);
0154
0155   // these signal pair are NOT guaranteed to be called by the same thread
0156   void preGlobalBeginLumi(edm::GlobalContext const&);
0157   void postGlobalBeginLumi(edm::GlobalContext const&);
0158
0159   // these signal pair are NOT guaranteed to be called by the same thread
0160   void preGlobalEndLumi(edm::GlobalContext const&);
0161   void postGlobalEndLumi(edm::GlobalContext const&);
0162
0163   // these signal pair are NOT guaranteed to be called by the same thread
0164   void preStreamBeginLumi(edm::StreamContext const&);
0165   void postStreamBeginLumi(edm::StreamContext const&);
0166
0167   // these signal pair are NOT guaranteed to be called by the same thread
0168   void preStreamEndLumi(edm::StreamContext const&);
0169   void postStreamEndLumi(edm::StreamContext const&);
0170
0171   // these signal pair are NOT guaranteed to be called by the same thread
0172   void preEvent(edm::StreamContext const&);
0173   void postEvent(edm::StreamContext const&);
0174
0175   // these signal pair are NOT guaranteed to be called by the same thread
0176   void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0177   void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0178
0179   // these signal pair are NOT guaranteed to be called by the same thread
0180   void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0181   void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0182
0183   // these signal pair are guaranteed to be called by the same thread
0184   void preOpenFile(std::string const&);
0185   void postOpenFile(std::string const&);
0186
0187   // these signal pair are guaranteed to be called by the same thread
0188   void preCloseFile(std::string const&);
0189   void postCloseFile(std::string const&);
0190
0191   // these signal pair are guaranteed to be called by the same thread
0192   void preSourceConstruction(edm::ModuleDescription const&);
0193   void postSourceConstruction(edm::ModuleDescription const&);
0194
0195   // these signal pair are guaranteed to be called by the same thread
0196   void preSourceRun(edm::RunIndex);
0197   void postSourceRun(edm::RunIndex);
0198
0199   // these signal pair are guaranteed to be called by the same thread
0200   void preSourceLumi(edm::LuminosityBlockIndex);
0201   void postSourceLumi(edm::LuminosityBlockIndex);
0202
0203   // these signal pair are guaranteed to be called by the same thread
0204   void preSourceEvent(edm::StreamID);
0205   void postSourceEvent(edm::StreamID);
0206
0207   // these signal pair are guaranteed to be called by the same thread
0208   void preModuleConstruction(edm::ModuleDescription const&);
0209   void postModuleConstruction(edm::ModuleDescription const&);
0210
0211   // these signal pair are guaranteed to be called by the same thread
0212   void preModuleDestruction(edm::ModuleDescription const&);
0213   void postModuleDestruction(edm::ModuleDescription const&);
0214
0215   // these signal pair are guaranteed to be called by the same thread
0216   void preModuleBeginJob(edm::ModuleDescription const&);
0217   void postModuleBeginJob(edm::ModuleDescription const&);
0218
0219   // these signal pair are guaranteed to be called by the same thread
0220   void preModuleEndJob(edm::ModuleDescription const&);
0221   void postModuleEndJob(edm::ModuleDescription const&);
0222
0223   // these signal pair are guaranteed to be called by the same thread
0224   void preModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0225   void postModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0226
0227   // these signal pair are guaranteed to be called by the same thread
0228   void preModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0229   void postModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0230
0231   // these signal pair are guaranteed to be called by the same thread
0232   void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0233   void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0234
0235   // these signal pair are guaranteed to be called by the same thread
0236   void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0237   void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0238
0239   // these signal pair are guaranteed to be called by the same thread
0240   void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0241   void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0242
0243   // these signal pair are guaranteed to be called by the same thread
0244   void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0245   void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0246
0247   // these signal pair are guaranteed to be called by the same thread
0248   void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0249   void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0250
0251   // these signal pair are guaranteed to be called by the same thread
0252   void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0253   void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0254
0255   // these signal pair are guaranteed to be called by the same thread
0256   void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0257   void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0258
0259   // these signal pair are guaranteed to be called by the same thread
0260   void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0261   void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0262
0263   // these signal pair are guaranteed to be called by the same thread
0264   void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0265   void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0266
0267   // these signal pair are guaranteed to be called by the same thread
0268   void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0269   void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0270
0271   // these signal pair are guaranteed to be called by the same thread
0272   void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0273   void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0274
0275   // these signal pair are guaranteed to be called by the same thread
0276   void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0277   void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0278
0279 private:
0280   bool highlight(std::string const& label) const {
0281     return (std::binary_search(highlightModules_.begin(), highlightModules_.end(), label));
0282   }
0283
0284   uint32_t labelColor(std::string const& label) const { return highlight(label) ? nvtxAmber : nvtxGreen; }
0285
0286   uint32_t labelColorLight(std::string const& label) const {
0287     return highlight(label) ? nvtxLightAmber : nvtxLightGreen;
0288   }
0289
0290   std::vector<std::string> highlightModules_;
0291   const bool showModulePrefetching_;
0292   const bool skipFirstEvent_;
0293
0294   std::atomic<bool> globalFirstEventDone_ = false;
0295   std::vector<std::atomic<bool>> streamFirstEventDone_;
0296   std::vector<nvtxRangeId_t> event_;                        // per-stream event ranges
0297   std::vector<std::vector<nvtxRangeId_t>> stream_modules_;  // per-stream, per-module ranges
0298   // use a tbb::concurrent_vector rather than an std::vector because its final size is not known
0299   tbb::concurrent_vector<nvtxRangeId_t> global_modules_;  // global per-module events
0300
0301   nvtxDomainHandle_t global_domain_;               // NVTX domain for global EDM transitions
0302   std::vector<nvtxDomainHandle_t> stream_domain_;  // NVTX domains for per-EDM-stream transitions
0303 };
0304
0305 NVProfilerService::NVProfilerService(edm::ParameterSet const& config, edm::ActivityRegistry& registry)
0306     : highlightModules_(config.getUntrackedParameter<std::vector<std::string>>("highlightModules")),
0307       showModulePrefetching_(config.getUntrackedParameter<bool>("showModulePrefetching")),
0308       skipFirstEvent_(config.getUntrackedParameter<bool>("skipFirstEvent")) {
0309   // make sure that CUDA is initialised, and that the CUDAService destructor is called after this service's destructor
0310   edm::Service<CUDAService> cudaService;
0311
0312   std::sort(highlightModules_.begin(), highlightModules_.end());
0313
0314   // create the NVTX domain for global EDM transitions
0315   global_domain_ = nvtxDomainCreate("EDM Global");
0316
0317   // enables profile collection; if profiling is already enabled it has no effect
0318   if (not skipFirstEvent_) {
0319     cudaProfilerStart();
0320   }
0321
0322   registry.watchPreallocate(this, &NVProfilerService::preallocate);
0323
0324   // these signal pair are NOT guaranteed to be called by the same thread
0325   registry.watchPreBeginJob(this, &NVProfilerService::preBeginJob);
0326   registry.watchPostBeginJob(this, &NVProfilerService::postBeginJob);
0327
0328   // there is no preEndJob() signal
0329   registry.watchPostEndJob(this, &NVProfilerService::postEndJob);
0330
0331   // these signal pair are NOT guaranteed to be called by the same thread
0332   registry.watchPreGlobalBeginRun(this, &NVProfilerService::preGlobalBeginRun);
0333   registry.watchPostGlobalBeginRun(this, &NVProfilerService::postGlobalBeginRun);
0334
0335   // these signal pair are NOT guaranteed to be called by the same thread
0336   registry.watchPreGlobalEndRun(this, &NVProfilerService::preGlobalEndRun);
0337   registry.watchPostGlobalEndRun(this, &NVProfilerService::postGlobalEndRun);
0338
0339   // these signal pair are NOT guaranteed to be called by the same thread
0340   registry.watchPreStreamBeginRun(this, &NVProfilerService::preStreamBeginRun);
0341   registry.watchPostStreamBeginRun(this, &NVProfilerService::postStreamBeginRun);
0342
0343   // these signal pair are NOT guaranteed to be called by the same thread
0344   registry.watchPreStreamEndRun(this, &NVProfilerService::preStreamEndRun);
0345   registry.watchPostStreamEndRun(this, &NVProfilerService::postStreamEndRun);
0346
0347   // these signal pair are NOT guaranteed to be called by the same thread
0348   registry.watchPreGlobalBeginLumi(this, &NVProfilerService::preGlobalBeginLumi);
0349   registry.watchPostGlobalBeginLumi(this, &NVProfilerService::postGlobalBeginLumi);
0350
0351   // these signal pair are NOT guaranteed to be called by the same thread
0352   registry.watchPreGlobalEndLumi(this, &NVProfilerService::preGlobalEndLumi);
0353   registry.watchPostGlobalEndLumi(this, &NVProfilerService::postGlobalEndLumi);
0354
0355   // these signal pair are NOT guaranteed to be called by the same thread
0356   registry.watchPreStreamBeginLumi(this, &NVProfilerService::preStreamBeginLumi);
0357   registry.watchPostStreamBeginLumi(this, &NVProfilerService::postStreamBeginLumi);
0358
0359   // these signal pair are NOT guaranteed to be called by the same thread
0360   registry.watchPreStreamEndLumi(this, &NVProfilerService::preStreamEndLumi);
0361   registry.watchPostStreamEndLumi(this, &NVProfilerService::postStreamEndLumi);
0362
0363   // these signal pair are NOT guaranteed to be called by the same thread
0364   registry.watchPreEvent(this, &NVProfilerService::preEvent);
0365   registry.watchPostEvent(this, &NVProfilerService::postEvent);
0366
0367   // these signal pair are NOT guaranteed to be called by the same thread
0368   registry.watchPrePathEvent(this, &NVProfilerService::prePathEvent);
0369   registry.watchPostPathEvent(this, &NVProfilerService::postPathEvent);
0370
0371   if (showModulePrefetching_) {
0372     // these signal pair are NOT guaranteed to be called by the same thread
0373     registry.watchPreModuleEventPrefetching(this, &NVProfilerService::preModuleEventPrefetching);
0374     registry.watchPostModuleEventPrefetching(this, &NVProfilerService::postModuleEventPrefetching);
0375   }
0376
0377   // these signal pair are guaranteed to be called by the same thread
0378   registry.watchPreOpenFile(this, &NVProfilerService::preOpenFile);
0379   registry.watchPostOpenFile(this, &NVProfilerService::postOpenFile);
0380
0381   // these signal pair are guaranteed to be called by the same thread
0382   registry.watchPreCloseFile(this, &NVProfilerService::preCloseFile);
0383   registry.watchPostCloseFile(this, &NVProfilerService::postCloseFile);
0384
0385   // these signal pair are guaranteed to be called by the same thread
0386   registry.watchPreSourceConstruction(this, &NVProfilerService::preSourceConstruction);
0387   registry.watchPostSourceConstruction(this, &NVProfilerService::postSourceConstruction);
0388
0389   // these signal pair are guaranteed to be called by the same thread
0390   registry.watchPreSourceRun(this, &NVProfilerService::preSourceRun);
0391   registry.watchPostSourceRun(this, &NVProfilerService::postSourceRun);
0392
0393   // these signal pair are guaranteed to be called by the same thread
0394   registry.watchPreSourceLumi(this, &NVProfilerService::preSourceLumi);
0395   registry.watchPostSourceLumi(this, &NVProfilerService::postSourceLumi);
0396
0397   // these signal pair are guaranteed to be called by the same thread
0398   registry.watchPreSourceEvent(this, &NVProfilerService::preSourceEvent);
0399   registry.watchPostSourceEvent(this, &NVProfilerService::postSourceEvent);
0400
0401   // these signal pair are guaranteed to be called by the same thread
0402   registry.watchPreModuleConstruction(this, &NVProfilerService::preModuleConstruction);
0403   registry.watchPostModuleConstruction(this, &NVProfilerService::postModuleConstruction);
0404
0405   // these signal pair are guaranteed to be called by the same thread
0406   registry.watchPreModuleDestruction(this, &NVProfilerService::preModuleDestruction);
0407   registry.watchPostModuleDestruction(this, &NVProfilerService::postModuleDestruction);
0408
0409   // these signal pair are guaranteed to be called by the same thread
0410   registry.watchPreModuleBeginJob(this, &NVProfilerService::preModuleBeginJob);
0411   registry.watchPostModuleBeginJob(this, &NVProfilerService::postModuleBeginJob);
0412
0413   // these signal pair are guaranteed to be called by the same thread
0414   registry.watchPreModuleEndJob(this, &NVProfilerService::preModuleEndJob);
0415   registry.watchPostModuleEndJob(this, &NVProfilerService::postModuleEndJob);
0416
0417   // these signal pair are guaranteed to be called by the same thread
0418   registry.watchPreModuleBeginStream(this, &NVProfilerService::preModuleBeginStream);
0419   registry.watchPostModuleBeginStream(this, &NVProfilerService::postModuleBeginStream);
0420
0421   // these signal pair are guaranteed to be called by the same thread
0422   registry.watchPreModuleEndStream(this, &NVProfilerService::preModuleEndStream);
0423   registry.watchPostModuleEndStream(this, &NVProfilerService::postModuleEndStream);
0424
0425   // these signal pair are guaranteed to be called by the same thread
0426   registry.watchPreModuleGlobalBeginRun(this, &NVProfilerService::preModuleGlobalBeginRun);
0427   registry.watchPostModuleGlobalBeginRun(this, &NVProfilerService::postModuleGlobalBeginRun);
0428
0429   // these signal pair are guaranteed to be called by the same thread
0430   registry.watchPreModuleGlobalEndRun(this, &NVProfilerService::preModuleGlobalEndRun);
0431   registry.watchPostModuleGlobalEndRun(this, &NVProfilerService::postModuleGlobalEndRun);
0432
0433   // these signal pair are guaranteed to be called by the same thread
0434   registry.watchPreModuleGlobalBeginLumi(this, &NVProfilerService::preModuleGlobalBeginLumi);
0435   registry.watchPostModuleGlobalBeginLumi(this, &NVProfilerService::postModuleGlobalBeginLumi);
0436
0437   // these signal pair are guaranteed to be called by the same thread
0438   registry.watchPreModuleGlobalEndLumi(this, &NVProfilerService::preModuleGlobalEndLumi);
0439   registry.watchPostModuleGlobalEndLumi(this, &NVProfilerService::postModuleGlobalEndLumi);
0440
0441   // these signal pair are guaranteed to be called by the same thread
0442   registry.watchPreModuleStreamBeginRun(this, &NVProfilerService::preModuleStreamBeginRun);
0443   registry.watchPostModuleStreamBeginRun(this, &NVProfilerService::postModuleStreamBeginRun);
0444
0445   // these signal pair are guaranteed to be called by the same thread
0446   registry.watchPreModuleStreamEndRun(this, &NVProfilerService::preModuleStreamEndRun);
0447   registry.watchPostModuleStreamEndRun(this, &NVProfilerService::postModuleStreamEndRun);
0448
0449   // these signal pair are guaranteed to be called by the same thread
0450   registry.watchPreModuleStreamBeginLumi(this, &NVProfilerService::preModuleStreamBeginLumi);
0451   registry.watchPostModuleStreamBeginLumi(this, &NVProfilerService::postModuleStreamBeginLumi);
0452
0453   // these signal pair are guaranteed to be called by the same thread
0454   registry.watchPreModuleStreamEndLumi(this, &NVProfilerService::preModuleStreamEndLumi);
0455   registry.watchPostModuleStreamEndLumi(this, &NVProfilerService::postModuleStreamEndLumi);
0456
0457   // these signal pair are guaranteed to be called by the same thread
0458   registry.watchPreModuleEventAcquire(this, &NVProfilerService::preModuleEventAcquire);
0459   registry.watchPostModuleEventAcquire(this, &NVProfilerService::postModuleEventAcquire);
0460
0461   // these signal pair are guaranteed to be called by the same thread
0462   registry.watchPreModuleEvent(this, &NVProfilerService::preModuleEvent);
0463   registry.watchPostModuleEvent(this, &NVProfilerService::postModuleEvent);
0464
0465   // these signal pair are guaranteed to be called by the same thread
0466   registry.watchPreModuleEventDelayedGet(this, &NVProfilerService::preModuleEventDelayedGet);
0467   registry.watchPostModuleEventDelayedGet(this, &NVProfilerService::postModuleEventDelayedGet);
0468
0469   // these signal pair are guaranteed to be called by the same thread
0470   registry.watchPreEventReadFromSource(this, &NVProfilerService::preEventReadFromSource);
0471   registry.watchPostEventReadFromSource(this, &NVProfilerService::postEventReadFromSource);
0472 }
0473
0474 NVProfilerService::~NVProfilerService() {
0475   for (unsigned int sid = 0; sid < stream_domain_.size(); ++sid) {
0476     nvtxDomainDestroy(stream_domain_[sid]);
0477   }
0478   nvtxDomainDestroy(global_domain_);
0479   cudaProfilerStop();
0480 }
0481
0482 void NVProfilerService::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0483   edm::ParameterSetDescription desc;
0484   desc.addUntracked<std::vector<std::string>>("highlightModules", {})->setComment("");
0485   desc.addUntracked<bool>("showModulePrefetching", false)
0486       ->setComment("Show the stack of dependencies that requested to run a module.");
0487   desc.addUntracked<bool>("skipFirstEvent", false)
0488       ->setComment(
0489           "Start profiling after the first event has completed.\nWith multiple streams, ignore transitions belonging "
0490           "to events started in parallel to the first event.\nRequires running nvprof with the '--profile-from-start "
0491           "off' option.");
0492   descriptions.add("NVProfilerService", desc);
0493   descriptions.setComment(R"(This Service provides CMSSW-aware annotations to nvprof/nvvm.
0494
0495 Notes on nvprof options:
0496   - the option '--profile-from-start off' should be used if skipFirstEvent is True.
0497   - the option '--cpu-profiling on' currently results in cmsRun being stuck at the beginning of the job.
0498   - the option '--cpu-thread-tracing on' is not compatible with jemalloc, and should only be used with cmsRunGlibC.)");
0499 }
0500
0501 void NVProfilerService::preallocate(edm::service::SystemBounds const& bounds) {
0502   std::stringstream out;
0503   out << "preallocate: " << bounds.maxNumberOfConcurrentRuns() << " concurrent runs, "
0504       << bounds.maxNumberOfConcurrentLuminosityBlocks() << " luminosity sections, " << bounds.maxNumberOfStreams()
0505       << " streams\nrunning on" << bounds.maxNumberOfThreads() << " threads";
0506   nvtxDomainMark(global_domain_, out.str().c_str());
0507
0508   auto concurrentStreams = bounds.maxNumberOfStreams();
0509   // create the NVTX domains for per-EDM-stream transitions
0510   stream_domain_.resize(concurrentStreams);
0511   for (unsigned int sid = 0; sid < concurrentStreams; ++sid) {
0512     stream_domain_[sid] = nvtxDomainCreate(fmt::sprintf("EDM Stream %d", sid).c_str());
0513   }
0514
0515   event_.resize(concurrentStreams);
0516   stream_modules_.resize(concurrentStreams);
0517   if (skipFirstEvent_) {
0518     globalFirstEventDone_ = false;
0519     std::vector<std::atomic<bool>> tmp(concurrentStreams);
0520     for (auto& element : tmp)
0521       std::atomic_init(&element, false);
0522     streamFirstEventDone_ = std::move(tmp);
0523   }
0524 }
0525
0526 void NVProfilerService::preBeginJob(edm::PathsAndConsumesOfModulesBase const& pathsAndConsumes,
0527                                     edm::ProcessContext const& pc) {
0528   nvtxDomainMark(global_domain_, "preBeginJob");
0529
0530   // FIXME this probably works only in the absence of subprocesses
0531   // size() + 1 because pathsAndConsumes.allModules() does not include the source
0532   unsigned int modules = pathsAndConsumes.allModules().size() + 1;
0533   global_modules_.resize(modules, nvtxInvalidRangeId);
0534   for (unsigned int sid = 0; sid < stream_modules_.size(); ++sid) {
0535     stream_modules_[sid].resize(modules, nvtxInvalidRangeId);
0536   }
0537 }
0538
0539 void NVProfilerService::postBeginJob() {
0540   if (not skipFirstEvent_ or globalFirstEventDone_) {
0541     nvtxDomainMark(global_domain_, "postBeginJob");
0542   }
0543 }
0544
0545 void NVProfilerService::postEndJob() {
0546   if (not skipFirstEvent_ or globalFirstEventDone_) {
0547     nvtxDomainMark(global_domain_, "postEndJob");
0548   }
0549 }
0550
0551 void NVProfilerService::preSourceEvent(edm::StreamID sid) {
0552   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0553     nvtxDomainRangePush(stream_domain_[sid], "source");
0554   }
0555 }
0556
0557 void NVProfilerService::postSourceEvent(edm::StreamID sid) {
0558   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0559     nvtxDomainRangePop(stream_domain_[sid]);
0560   }
0561 }
0562
0563 void NVProfilerService::preSourceLumi(edm::LuminosityBlockIndex index) {
0564   if (not skipFirstEvent_ or globalFirstEventDone_) {
0565     nvtxDomainRangePush(global_domain_, "source lumi");
0566   }
0567 }
0568
0569 void NVProfilerService::postSourceLumi(edm::LuminosityBlockIndex index) {
0570   if (not skipFirstEvent_ or globalFirstEventDone_) {
0571     nvtxDomainRangePop(global_domain_);
0572   }
0573 }
0574
0575 void NVProfilerService::preSourceRun(edm::RunIndex index) {
0576   if (not skipFirstEvent_ or globalFirstEventDone_) {
0577     nvtxDomainRangePush(global_domain_, "source run");
0578   }
0579 }
0580
0581 void NVProfilerService::postSourceRun(edm::RunIndex index) {
0582   if (not skipFirstEvent_ or globalFirstEventDone_) {
0583     nvtxDomainRangePop(global_domain_);
0584   }
0585 }
0586
0587 void NVProfilerService::preOpenFile(std::string const& lfn) {
0588   if (not skipFirstEvent_ or globalFirstEventDone_) {
0589     nvtxDomainRangePush(global_domain_, ("open file "s + lfn).c_str());
0590   }
0591 }
0592
0593 void NVProfilerService::postOpenFile(std::string const& lfn) {
0594   if (not skipFirstEvent_ or globalFirstEventDone_) {
0595     nvtxDomainRangePop(global_domain_);
0596   }
0597 }
0598
0599 void NVProfilerService::preCloseFile(std::string const& lfn) {
0600   if (not skipFirstEvent_ or globalFirstEventDone_) {
0601     nvtxDomainRangePush(global_domain_, ("close file "s + lfn).c_str());
0602   }
0603 }
0604
0605 void NVProfilerService::postCloseFile(std::string const& lfn) {
0606   if (not skipFirstEvent_ or globalFirstEventDone_) {
0607     nvtxDomainRangePop(global_domain_);
0608   }
0609 }
0610
0611 void NVProfilerService::preModuleBeginStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0612   auto sid = sc.streamID();
0613   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0614     auto mid = mcc.moduleDescription()->id();
0615     auto const& label = mcc.moduleDescription()->moduleLabel();
0616     auto const& msg = label + " begin stream";
0617     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0618     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0619   }
0620 }
0621
0622 void NVProfilerService::postModuleBeginStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0623   auto sid = sc.streamID();
0624   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0625     auto mid = mcc.moduleDescription()->id();
0626     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0627     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0628   }
0629 }
0630
0631 void NVProfilerService::preModuleEndStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0632   auto sid = sc.streamID();
0633   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0634     auto mid = mcc.moduleDescription()->id();
0635     auto const& label = mcc.moduleDescription()->moduleLabel();
0636     auto const& msg = label + " end stream";
0637     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0638     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0639   }
0640 }
0641
0642 void NVProfilerService::postModuleEndStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0643   auto sid = sc.streamID();
0644   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0645     auto mid = mcc.moduleDescription()->id();
0646     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0647     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0648   }
0649 }
0650
0651 void NVProfilerService::preGlobalBeginRun(edm::GlobalContext const& gc) {
0652   if (not skipFirstEvent_ or globalFirstEventDone_) {
0653     nvtxDomainRangePush(global_domain_, "global begin run");
0654   }
0655 }
0656
0657 void NVProfilerService::postGlobalBeginRun(edm::GlobalContext const& gc) {
0658   if (not skipFirstEvent_ or globalFirstEventDone_) {
0659     nvtxDomainRangePop(global_domain_);
0660   }
0661 }
0662
0663 void NVProfilerService::preGlobalEndRun(edm::GlobalContext const& gc) {
0664   if (not skipFirstEvent_ or globalFirstEventDone_) {
0665     nvtxDomainRangePush(global_domain_, "global end run");
0666   }
0667 }
0668
0669 void NVProfilerService::postGlobalEndRun(edm::GlobalContext const& gc) {
0670   if (not skipFirstEvent_ or globalFirstEventDone_) {
0671     nvtxDomainRangePop(global_domain_);
0672   }
0673 }
0674
0675 void NVProfilerService::preStreamBeginRun(edm::StreamContext const& sc) {
0676   auto sid = sc.streamID();
0677   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0678     nvtxDomainRangePush(stream_domain_[sid], "stream begin run");
0679   }
0680 }
0681
0682 void NVProfilerService::postStreamBeginRun(edm::StreamContext const& sc) {
0683   auto sid = sc.streamID();
0684   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0685     nvtxDomainRangePop(stream_domain_[sid]);
0686   }
0687 }
0688
0689 void NVProfilerService::preStreamEndRun(edm::StreamContext const& sc) {
0690   auto sid = sc.streamID();
0691   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0692     nvtxDomainRangePush(stream_domain_[sid], "stream end run");
0693   }
0694 }
0695
0696 void NVProfilerService::postStreamEndRun(edm::StreamContext const& sc) {
0697   auto sid = sc.streamID();
0698   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0699     nvtxDomainRangePop(stream_domain_[sid]);
0700   }
0701 }
0702
0703 void NVProfilerService::preGlobalBeginLumi(edm::GlobalContext const& gc) {
0704   if (not skipFirstEvent_ or globalFirstEventDone_) {
0705     nvtxDomainRangePush(global_domain_, "global begin lumi");
0706   }
0707 }
0708
0709 void NVProfilerService::postGlobalBeginLumi(edm::GlobalContext const& gc) {
0710   if (not skipFirstEvent_ or globalFirstEventDone_) {
0711     nvtxDomainRangePop(global_domain_);
0712   }
0713 }
0714
0715 void NVProfilerService::preGlobalEndLumi(edm::GlobalContext const& gc) {
0716   if (not skipFirstEvent_ or globalFirstEventDone_) {
0717     nvtxDomainRangePush(global_domain_, "global end lumi");
0718   }
0719 }
0720
0721 void NVProfilerService::postGlobalEndLumi(edm::GlobalContext const& gc) {
0722   if (not skipFirstEvent_ or globalFirstEventDone_) {
0723     nvtxDomainRangePop(global_domain_);
0724   }
0725 }
0726
0727 void NVProfilerService::preStreamBeginLumi(edm::StreamContext const& sc) {
0728   auto sid = sc.streamID();
0729   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0730     nvtxDomainRangePush(stream_domain_[sid], "stream begin lumi");
0731   }
0732 }
0733
0734 void NVProfilerService::postStreamBeginLumi(edm::StreamContext const& sc) {
0735   auto sid = sc.streamID();
0736   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0737     nvtxDomainRangePop(stream_domain_[sid]);
0738   }
0739 }
0740
0741 void NVProfilerService::preStreamEndLumi(edm::StreamContext const& sc) {
0742   auto sid = sc.streamID();
0743   nvtxDomainRangePush(stream_domain_[sid], "stream end lumi");
0744 }
0745
0746 void NVProfilerService::postStreamEndLumi(edm::StreamContext const& sc) {
0747   auto sid = sc.streamID();
0748   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0749     nvtxDomainRangePop(stream_domain_[sid]);
0750   }
0751 }
0752
0753 void NVProfilerService::preEvent(edm::StreamContext const& sc) {
0754   auto sid = sc.streamID();
0755   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0756     event_[sid] = nvtxDomainRangeStartColor(stream_domain_[sid], "event", nvtxDarkGreen);
0757   }
0758 }
0759
0760 void NVProfilerService::postEvent(edm::StreamContext const& sc) {
0761   auto sid = sc.streamID();
0762   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0763     nvtxDomainRangeEnd(stream_domain_[sid], event_[sid]);
0764     event_[sid] = nvtxInvalidRangeId;
0765   } else {
0766     streamFirstEventDone_[sid] = true;
0767     auto identity = [](bool x) { return x; };
0768     if (std::all_of(streamFirstEventDone_.begin(), streamFirstEventDone_.end(), identity)) {
0769       bool expected = false;
0770       if (globalFirstEventDone_.compare_exchange_strong(expected, true))
0771         cudaProfilerStart();
0772     }
0773   }
0774 }
0775
0776 void NVProfilerService::prePathEvent(edm::StreamContext const& sc, edm::PathContext const& pc) {
0777   auto sid = sc.streamID();
0778   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0779     nvtxDomainMark(global_domain_, ("before path "s + pc.pathName()).c_str());
0780   }
0781 }
0782
0783 void NVProfilerService::postPathEvent(edm::StreamContext const& sc,
0784                                       edm::PathContext const& pc,
0785                                       edm::HLTPathStatus const& hlts) {
0786   auto sid = sc.streamID();
0787   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0788     nvtxDomainMark(global_domain_, ("after path "s + pc.pathName()).c_str());
0789   }
0790 }
0791
0792 void NVProfilerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0793   auto sid = sc.streamID();
0794   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0795     auto mid = mcc.moduleDescription()->id();
0796     auto const& label = mcc.moduleDescription()->moduleLabel();
0797     auto const& msg = label + " prefetching";
0798     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0799     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColorLight(label));
0800   }
0801 }
0802
0803 void NVProfilerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0804   auto sid = sc.streamID();
0805   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0806     auto mid = mcc.moduleDescription()->id();
0807     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0808     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0809   }
0810 }
0811
0812 void NVProfilerService::preModuleConstruction(edm::ModuleDescription const& desc) {
0813   if (not skipFirstEvent_) {
0814     auto mid = desc.id();
0815     global_modules_.grow_to_at_least(mid + 1);
0816     auto const& label = desc.moduleLabel();
0817     auto const& msg = label + " construction";
0818     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0819   }
0820 }
0821
0822 void NVProfilerService::postModuleConstruction(edm::ModuleDescription const& desc) {
0823   if (not skipFirstEvent_) {
0824     auto mid = desc.id();
0825     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0826     global_modules_[mid] = nvtxInvalidRangeId;
0827   }
0828 }
0829
0830 void NVProfilerService::preModuleDestruction(edm::ModuleDescription const& desc) {
0831   if (not skipFirstEvent_) {
0832     auto mid = desc.id();
0833     global_modules_.grow_to_at_least(mid + 1);
0834     auto const& label = desc.moduleLabel();
0835     auto const& msg = label + " destruction";
0836     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0837   }
0838 }
0839
0840 void NVProfilerService::postModuleDestruction(edm::ModuleDescription const& desc) {
0841   if (not skipFirstEvent_) {
0842     auto mid = desc.id();
0843     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0844     global_modules_[mid] = nvtxInvalidRangeId;
0845   }
0846 }
0847
0848 void NVProfilerService::preModuleBeginJob(edm::ModuleDescription const& desc) {
0849   if (not skipFirstEvent_) {
0850     auto mid = desc.id();
0851     auto const& label = desc.moduleLabel();
0852     auto const& msg = label + " begin job";
0853     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0854   }
0855 }
0856
0857 void NVProfilerService::postModuleBeginJob(edm::ModuleDescription const& desc) {
0858   if (not skipFirstEvent_) {
0859     auto mid = desc.id();
0860     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0861     global_modules_[mid] = nvtxInvalidRangeId;
0862   }
0863 }
0864
0865 void NVProfilerService::preModuleEndJob(edm::ModuleDescription const& desc) {
0866   if (not skipFirstEvent_ or globalFirstEventDone_) {
0867     auto mid = desc.id();
0868     auto const& label = desc.moduleLabel();
0869     auto const& msg = label + " end job";
0870     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0871   }
0872 }
0873
0874 void NVProfilerService::postModuleEndJob(edm::ModuleDescription const& desc) {
0875   if (not skipFirstEvent_ or globalFirstEventDone_) {
0876     auto mid = desc.id();
0877     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0878     global_modules_[mid] = nvtxInvalidRangeId;
0879   }
0880 }
0881
0882 void NVProfilerService::preModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0883   auto sid = sc.streamID();
0884   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0885     auto mid = mcc.moduleDescription()->id();
0886     auto const& label = mcc.moduleDescription()->moduleLabel();
0887     auto const& msg = label + " acquire";
0888     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0889     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0890   }
0891 }
0892
0893 void NVProfilerService::postModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0894   auto sid = sc.streamID();
0895   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0896     auto mid = mcc.moduleDescription()->id();
0897     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0898     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0899   }
0900 }
0901
0902 void NVProfilerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0903   auto sid = sc.streamID();
0904   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0905     auto mid = mcc.moduleDescription()->id();
0906     auto const& label = mcc.moduleDescription()->moduleLabel();
0907     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0908     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], label.c_str(), labelColor(label));
0909   }
0910 }
0911
0912 void NVProfilerService::postModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0913   auto sid = sc.streamID();
0914   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0915     auto mid = mcc.moduleDescription()->id();
0916     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0917     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0918   }
0919 }
0920
0921 void NVProfilerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0922   /* FIXME
0923   auto sid = sc.streamID();
0924   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0925     auto mid = mcc.moduleDescription()->id();
0926     auto const & label = mcc.moduleDescription()->moduleLabel();
0927     auto const & msg = label + " delayed get";
0928     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0929     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], label.c_str(), labelColorLight(label));
0930   }
0931   */
0932 }
0933
0934 void NVProfilerService::postModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0935   /* FIXME
0936   auto sid = sc.streamID();
0937   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0938     auto mid = mcc.moduleDescription()->id();
0939     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0940     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0941   }
0942   */
0943 }
0944
0945 void NVProfilerService::preEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0946   /* FIXME
0947   auto sid = sc.streamID();
0948   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0949     auto mid = mcc.moduleDescription()->id();
0950     auto const & label = mcc.moduleDescription()->moduleLabel();
0951     auto const & msg = label + " read from source";
0952     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0953     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColorLight(label));
0954   }
0955   */
0956 }
0957
0958 void NVProfilerService::postEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0959   /* FIXME
0960   auto sid = sc.streamID();
0961   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0962     auto mid = mcc.moduleDescription()->id();
0963     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0964     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0965   }
0966   */
0967 }
0968
0969 void NVProfilerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0970   auto sid = sc.streamID();
0971   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0972     auto mid = mcc.moduleDescription()->id();
0973     auto const& label = mcc.moduleDescription()->moduleLabel();
0974     auto const& msg = label + " stream begin run";
0975     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0976     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0977   }
0978 }
0979
0980 void NVProfilerService::postModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0981   auto sid = sc.streamID();
0982   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0983     auto mid = mcc.moduleDescription()->id();
0984     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0985     stream_modules_[sid][mid] = nvtxInvalidRangeId;
0986   }
0987 }
0988
0989 void NVProfilerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0990   auto sid = sc.streamID();
0991   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0992     auto mid = mcc.moduleDescription()->id();
0993     auto const& label = mcc.moduleDescription()->moduleLabel();
0994     auto const& msg = label + " stream end run";
0995     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0996     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0997   }
0998 }
0999
1000 void NVProfilerService::postModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1001   auto sid = sc.streamID();
1002   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1003     auto mid = mcc.moduleDescription()->id();
1004     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1005     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1006   }
1007 }
1008
1009 void NVProfilerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1010   auto sid = sc.streamID();
1011   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1012     auto mid = mcc.moduleDescription()->id();
1013     auto const& label = mcc.moduleDescription()->moduleLabel();
1014     auto const& msg = label + " stream begin lumi";
1015     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1016     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1017   }
1018 }
1019
1020 void NVProfilerService::postModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1021   auto sid = sc.streamID();
1022   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1023     auto mid = mcc.moduleDescription()->id();
1024     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1025     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1026   }
1027 }
1028
1029 void NVProfilerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1030   auto sid = sc.streamID();
1031   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1032     auto mid = mcc.moduleDescription()->id();
1033     auto const& label = mcc.moduleDescription()->moduleLabel();
1034     auto const& msg = label + " stream end lumi";
1035     assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1036     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1037   }
1038 }
1039
1040 void NVProfilerService::postModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1041   auto sid = sc.streamID();
1042   if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1043     auto mid = mcc.moduleDescription()->id();
1044     nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1045     stream_modules_[sid][mid] = nvtxInvalidRangeId;
1046   }
1047 }
1048
1049 void NVProfilerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1050   if (not skipFirstEvent_ or globalFirstEventDone_) {
1051     auto mid = mcc.moduleDescription()->id();
1052     auto const& label = mcc.moduleDescription()->moduleLabel();
1053     auto const& msg = label + " global begin run";
1054     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1055   }
1056 }
1057
1058 void NVProfilerService::postModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1059   if (not skipFirstEvent_ or globalFirstEventDone_) {
1060     auto mid = mcc.moduleDescription()->id();
1061     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1062     global_modules_[mid] = nvtxInvalidRangeId;
1063   }
1064 }
1065
1066 void NVProfilerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1067   if (not skipFirstEvent_ or globalFirstEventDone_) {
1068     auto mid = mcc.moduleDescription()->id();
1069     auto const& label = mcc.moduleDescription()->moduleLabel();
1070     auto const& msg = label + " global end run";
1071     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1072   }
1073 }
1074
1075 void NVProfilerService::postModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1076   if (not skipFirstEvent_ or globalFirstEventDone_) {
1077     auto mid = mcc.moduleDescription()->id();
1078     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1079     global_modules_[mid] = nvtxInvalidRangeId;
1080   }
1081 }
1082
1083 void NVProfilerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1084   if (not skipFirstEvent_ or globalFirstEventDone_) {
1085     auto mid = mcc.moduleDescription()->id();
1086     auto const& label = mcc.moduleDescription()->moduleLabel();
1087     auto const& msg = label + " global begin lumi";
1088     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1089   }
1090 }
1091
1092 void NVProfilerService::postModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1093   if (not skipFirstEvent_ or globalFirstEventDone_) {
1094     auto mid = mcc.moduleDescription()->id();
1095     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1096     global_modules_[mid] = nvtxInvalidRangeId;
1097   }
1098 }
1099
1100 void NVProfilerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1101   if (not skipFirstEvent_ or globalFirstEventDone_) {
1102     auto mid = mcc.moduleDescription()->id();
1103     auto const& label = mcc.moduleDescription()->moduleLabel();
1104     auto const& msg = label + " global end lumi";
1105     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1106   }
1107 }
1108
1109 void NVProfilerService::postModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1110   if (not skipFirstEvent_ or globalFirstEventDone_) {
1111     auto mid = mcc.moduleDescription()->id();
1112     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1113     global_modules_[mid] = nvtxInvalidRangeId;
1114   }
1115 }
1116
1117 void NVProfilerService::preSourceConstruction(edm::ModuleDescription const& desc) {
1118   if (not skipFirstEvent_) {
1119     auto mid = desc.id();
1120     global_modules_.grow_to_at_least(mid + 1);
1121     auto const& label = desc.moduleLabel();
1122     auto const& msg = label + " construction";
1123     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1124   }
1125 }
1126
1127 void NVProfilerService::postSourceConstruction(edm::ModuleDescription const& desc) {
1128   if (not skipFirstEvent_) {
1129     auto mid = desc.id();
1130     nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1131     global_modules_[mid] = nvtxInvalidRangeId;
1132   }
1133 }
1134
1135 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
1136 DEFINE_FWK_SERVICE(NVProfilerService);