File indexing completed on 2022-01-13 01:43:45
0001
0002
0003
0004
0005
0006 #include <algorithm>
0007 #include <iostream>
0008 #include <sstream>
0009 #include <string>
0010 #include <vector>
0011
0012 #include <oneapi/tbb/concurrent_vector.h>
0013
0014 #include <fmt/printf.h>
0015
0016 #include <cuda_profiler_api.h>
0017 #include <nvToolsExt.h>
0018
0019 #include "DataFormats/Common/interface/HLTPathStatus.h"
0020 #include "DataFormats/Provenance/interface/EventID.h"
0021 #include "DataFormats/Provenance/interface/LuminosityBlockID.h"
0022 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0023 #include "DataFormats/Provenance/interface/RunID.h"
0024 #include "DataFormats/Provenance/interface/Timestamp.h"
0025 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0026 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0027 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0028 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0029 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0030 #include "FWCore/ServiceRegistry/interface/ConsumesInfo.h"
0031 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0032 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0033 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0034 #include "FWCore/ServiceRegistry/interface/PathsAndConsumesOfModulesBase.h"
0035 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0036 #include "FWCore/ServiceRegistry/interface/Service.h"
0037 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0038 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0039 #include "FWCore/Utilities/interface/BranchType.h"
0040 #include "FWCore/Utilities/interface/Exception.h"
0041 #include "FWCore/Utilities/interface/ProductKindOfType.h"
0042 #include "FWCore/Utilities/interface/TimeOfDay.h"
0043 #include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
0044
0045 using namespace std::string_literals;
0046
0047 namespace {
0048 int nvtxDomainRangePush(nvtxDomainHandle_t domain, const char* message) {
0049 nvtxEventAttributes_t eventAttrib = {};
0050 eventAttrib.version = NVTX_VERSION;
0051 eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0052 eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0053 eventAttrib.message.ascii = message;
0054 return nvtxDomainRangePushEx(domain, &eventAttrib);
0055 }
0056
0057 __attribute__((unused)) int nvtxDomainRangePushColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0058 nvtxEventAttributes_t eventAttrib = {};
0059 eventAttrib.version = NVTX_VERSION;
0060 eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0061 eventAttrib.colorType = NVTX_COLOR_ARGB;
0062 eventAttrib.color = color;
0063 eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0064 eventAttrib.message.ascii = message;
0065 return nvtxDomainRangePushEx(domain, &eventAttrib);
0066 }
0067
0068 __attribute__((unused)) nvtxRangeId_t nvtxDomainRangeStart(nvtxDomainHandle_t domain, const char* message) {
0069 nvtxEventAttributes_t eventAttrib = {};
0070 eventAttrib.version = NVTX_VERSION;
0071 eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0072 eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0073 eventAttrib.message.ascii = message;
0074 return nvtxDomainRangeStartEx(domain, &eventAttrib);
0075 }
0076
0077 nvtxRangeId_t nvtxDomainRangeStartColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0078 nvtxEventAttributes_t eventAttrib = {};
0079 eventAttrib.version = NVTX_VERSION;
0080 eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0081 eventAttrib.colorType = NVTX_COLOR_ARGB;
0082 eventAttrib.color = color;
0083 eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0084 eventAttrib.message.ascii = message;
0085 return nvtxDomainRangeStartEx(domain, &eventAttrib);
0086 }
0087
0088 void nvtxDomainMark(nvtxDomainHandle_t domain, const char* message) {
0089 nvtxEventAttributes_t eventAttrib = {};
0090 eventAttrib.version = NVTX_VERSION;
0091 eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0092 eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0093 eventAttrib.message.ascii = message;
0094 nvtxDomainMarkEx(domain, &eventAttrib);
0095 }
0096
0097 __attribute__((unused)) void nvtxDomainMarkColor(nvtxDomainHandle_t domain, const char* message, uint32_t color) {
0098 nvtxEventAttributes_t eventAttrib = {};
0099 eventAttrib.version = NVTX_VERSION;
0100 eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE;
0101 eventAttrib.colorType = NVTX_COLOR_ARGB;
0102 eventAttrib.color = color;
0103 eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII;
0104 eventAttrib.message.ascii = message;
0105 nvtxDomainMarkEx(domain, &eventAttrib);
0106 }
0107
0108 enum {
0109 nvtxBlack = 0x00000000,
0110 nvtxRed = 0x00ff0000,
0111 nvtxDarkGreen = 0x00009900,
0112 nvtxGreen = 0x0000ff00,
0113 nvtxLightGreen = 0x00ccffcc,
0114 nvtxBlue = 0x000000ff,
0115 nvtxAmber = 0x00ffbf00,
0116 nvtxLightAmber = 0x00fff2cc,
0117 nvtxWhite = 0x00ffffff
0118 };
0119
0120 constexpr nvtxRangeId_t nvtxInvalidRangeId = 0xfffffffffffffffful;
0121 }
0122
0123 class NVProfilerService {
0124 public:
0125 NVProfilerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0126 ~NVProfilerService();
0127
0128 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0129
0130 void preallocate(edm::service::SystemBounds const&);
0131
0132
0133 void preBeginJob(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0134 void postBeginJob();
0135
0136
0137 void postEndJob();
0138
0139
0140 void preGlobalBeginRun(edm::GlobalContext const&);
0141 void postGlobalBeginRun(edm::GlobalContext const&);
0142
0143
0144 void preGlobalEndRun(edm::GlobalContext const&);
0145 void postGlobalEndRun(edm::GlobalContext const&);
0146
0147
0148 void preStreamBeginRun(edm::StreamContext const&);
0149 void postStreamBeginRun(edm::StreamContext const&);
0150
0151
0152 void preStreamEndRun(edm::StreamContext const&);
0153 void postStreamEndRun(edm::StreamContext const&);
0154
0155
0156 void preGlobalBeginLumi(edm::GlobalContext const&);
0157 void postGlobalBeginLumi(edm::GlobalContext const&);
0158
0159
0160 void preGlobalEndLumi(edm::GlobalContext const&);
0161 void postGlobalEndLumi(edm::GlobalContext const&);
0162
0163
0164 void preStreamBeginLumi(edm::StreamContext const&);
0165 void postStreamBeginLumi(edm::StreamContext const&);
0166
0167
0168 void preStreamEndLumi(edm::StreamContext const&);
0169 void postStreamEndLumi(edm::StreamContext const&);
0170
0171
0172 void preEvent(edm::StreamContext const&);
0173 void postEvent(edm::StreamContext const&);
0174
0175
0176 void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0177 void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0178
0179
0180 void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0181 void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0182
0183
0184 void preOpenFile(std::string const&);
0185 void postOpenFile(std::string const&);
0186
0187
0188 void preCloseFile(std::string const&);
0189 void postCloseFile(std::string const&);
0190
0191
0192 void preSourceConstruction(edm::ModuleDescription const&);
0193 void postSourceConstruction(edm::ModuleDescription const&);
0194
0195
0196 void preSourceRun(edm::RunIndex);
0197 void postSourceRun(edm::RunIndex);
0198
0199
0200 void preSourceLumi(edm::LuminosityBlockIndex);
0201 void postSourceLumi(edm::LuminosityBlockIndex);
0202
0203
0204 void preSourceEvent(edm::StreamID);
0205 void postSourceEvent(edm::StreamID);
0206
0207
0208 void preModuleConstruction(edm::ModuleDescription const&);
0209 void postModuleConstruction(edm::ModuleDescription const&);
0210
0211
0212 void preModuleDestruction(edm::ModuleDescription const&);
0213 void postModuleDestruction(edm::ModuleDescription const&);
0214
0215
0216 void preModuleBeginJob(edm::ModuleDescription const&);
0217 void postModuleBeginJob(edm::ModuleDescription const&);
0218
0219
0220 void preModuleEndJob(edm::ModuleDescription const&);
0221 void postModuleEndJob(edm::ModuleDescription const&);
0222
0223
0224 void preModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0225 void postModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0226
0227
0228 void preModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0229 void postModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0230
0231
0232 void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0233 void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0234
0235
0236 void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0237 void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0238
0239
0240 void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0241 void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0242
0243
0244 void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0245 void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0246
0247
0248 void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0249 void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0250
0251
0252 void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0253 void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0254
0255
0256 void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0257 void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0258
0259
0260 void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0261 void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0262
0263
0264 void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0265 void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0266
0267
0268 void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0269 void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0270
0271
0272 void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0273 void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0274
0275
0276 void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0277 void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0278
0279 private:
0280 bool highlight(std::string const& label) const {
0281 return (std::binary_search(highlightModules_.begin(), highlightModules_.end(), label));
0282 }
0283
0284 uint32_t labelColor(std::string const& label) const { return highlight(label) ? nvtxAmber : nvtxGreen; }
0285
0286 uint32_t labelColorLight(std::string const& label) const {
0287 return highlight(label) ? nvtxLightAmber : nvtxLightGreen;
0288 }
0289
0290 std::vector<std::string> highlightModules_;
0291 const bool showModulePrefetching_;
0292 const bool skipFirstEvent_;
0293
0294 std::atomic<bool> globalFirstEventDone_ = false;
0295 std::vector<std::atomic<bool>> streamFirstEventDone_;
0296 std::vector<nvtxRangeId_t> event_;
0297 std::vector<std::vector<nvtxRangeId_t>> stream_modules_;
0298
0299 tbb::concurrent_vector<nvtxRangeId_t> global_modules_;
0300
0301 nvtxDomainHandle_t global_domain_;
0302 std::vector<nvtxDomainHandle_t> stream_domain_;
0303 };
0304
0305 NVProfilerService::NVProfilerService(edm::ParameterSet const& config, edm::ActivityRegistry& registry)
0306 : highlightModules_(config.getUntrackedParameter<std::vector<std::string>>("highlightModules")),
0307 showModulePrefetching_(config.getUntrackedParameter<bool>("showModulePrefetching")),
0308 skipFirstEvent_(config.getUntrackedParameter<bool>("skipFirstEvent")) {
0309
0310 edm::Service<CUDAService> cudaService;
0311
0312 std::sort(highlightModules_.begin(), highlightModules_.end());
0313
0314
0315 global_domain_ = nvtxDomainCreate("EDM Global");
0316
0317
0318 if (not skipFirstEvent_) {
0319 cudaProfilerStart();
0320 }
0321
0322 registry.watchPreallocate(this, &NVProfilerService::preallocate);
0323
0324
0325 registry.watchPreBeginJob(this, &NVProfilerService::preBeginJob);
0326 registry.watchPostBeginJob(this, &NVProfilerService::postBeginJob);
0327
0328
0329 registry.watchPostEndJob(this, &NVProfilerService::postEndJob);
0330
0331
0332 registry.watchPreGlobalBeginRun(this, &NVProfilerService::preGlobalBeginRun);
0333 registry.watchPostGlobalBeginRun(this, &NVProfilerService::postGlobalBeginRun);
0334
0335
0336 registry.watchPreGlobalEndRun(this, &NVProfilerService::preGlobalEndRun);
0337 registry.watchPostGlobalEndRun(this, &NVProfilerService::postGlobalEndRun);
0338
0339
0340 registry.watchPreStreamBeginRun(this, &NVProfilerService::preStreamBeginRun);
0341 registry.watchPostStreamBeginRun(this, &NVProfilerService::postStreamBeginRun);
0342
0343
0344 registry.watchPreStreamEndRun(this, &NVProfilerService::preStreamEndRun);
0345 registry.watchPostStreamEndRun(this, &NVProfilerService::postStreamEndRun);
0346
0347
0348 registry.watchPreGlobalBeginLumi(this, &NVProfilerService::preGlobalBeginLumi);
0349 registry.watchPostGlobalBeginLumi(this, &NVProfilerService::postGlobalBeginLumi);
0350
0351
0352 registry.watchPreGlobalEndLumi(this, &NVProfilerService::preGlobalEndLumi);
0353 registry.watchPostGlobalEndLumi(this, &NVProfilerService::postGlobalEndLumi);
0354
0355
0356 registry.watchPreStreamBeginLumi(this, &NVProfilerService::preStreamBeginLumi);
0357 registry.watchPostStreamBeginLumi(this, &NVProfilerService::postStreamBeginLumi);
0358
0359
0360 registry.watchPreStreamEndLumi(this, &NVProfilerService::preStreamEndLumi);
0361 registry.watchPostStreamEndLumi(this, &NVProfilerService::postStreamEndLumi);
0362
0363
0364 registry.watchPreEvent(this, &NVProfilerService::preEvent);
0365 registry.watchPostEvent(this, &NVProfilerService::postEvent);
0366
0367
0368 registry.watchPrePathEvent(this, &NVProfilerService::prePathEvent);
0369 registry.watchPostPathEvent(this, &NVProfilerService::postPathEvent);
0370
0371 if (showModulePrefetching_) {
0372
0373 registry.watchPreModuleEventPrefetching(this, &NVProfilerService::preModuleEventPrefetching);
0374 registry.watchPostModuleEventPrefetching(this, &NVProfilerService::postModuleEventPrefetching);
0375 }
0376
0377
0378 registry.watchPreOpenFile(this, &NVProfilerService::preOpenFile);
0379 registry.watchPostOpenFile(this, &NVProfilerService::postOpenFile);
0380
0381
0382 registry.watchPreCloseFile(this, &NVProfilerService::preCloseFile);
0383 registry.watchPostCloseFile(this, &NVProfilerService::postCloseFile);
0384
0385
0386 registry.watchPreSourceConstruction(this, &NVProfilerService::preSourceConstruction);
0387 registry.watchPostSourceConstruction(this, &NVProfilerService::postSourceConstruction);
0388
0389
0390 registry.watchPreSourceRun(this, &NVProfilerService::preSourceRun);
0391 registry.watchPostSourceRun(this, &NVProfilerService::postSourceRun);
0392
0393
0394 registry.watchPreSourceLumi(this, &NVProfilerService::preSourceLumi);
0395 registry.watchPostSourceLumi(this, &NVProfilerService::postSourceLumi);
0396
0397
0398 registry.watchPreSourceEvent(this, &NVProfilerService::preSourceEvent);
0399 registry.watchPostSourceEvent(this, &NVProfilerService::postSourceEvent);
0400
0401
0402 registry.watchPreModuleConstruction(this, &NVProfilerService::preModuleConstruction);
0403 registry.watchPostModuleConstruction(this, &NVProfilerService::postModuleConstruction);
0404
0405
0406 registry.watchPreModuleDestruction(this, &NVProfilerService::preModuleDestruction);
0407 registry.watchPostModuleDestruction(this, &NVProfilerService::postModuleDestruction);
0408
0409
0410 registry.watchPreModuleBeginJob(this, &NVProfilerService::preModuleBeginJob);
0411 registry.watchPostModuleBeginJob(this, &NVProfilerService::postModuleBeginJob);
0412
0413
0414 registry.watchPreModuleEndJob(this, &NVProfilerService::preModuleEndJob);
0415 registry.watchPostModuleEndJob(this, &NVProfilerService::postModuleEndJob);
0416
0417
0418 registry.watchPreModuleBeginStream(this, &NVProfilerService::preModuleBeginStream);
0419 registry.watchPostModuleBeginStream(this, &NVProfilerService::postModuleBeginStream);
0420
0421
0422 registry.watchPreModuleEndStream(this, &NVProfilerService::preModuleEndStream);
0423 registry.watchPostModuleEndStream(this, &NVProfilerService::postModuleEndStream);
0424
0425
0426 registry.watchPreModuleGlobalBeginRun(this, &NVProfilerService::preModuleGlobalBeginRun);
0427 registry.watchPostModuleGlobalBeginRun(this, &NVProfilerService::postModuleGlobalBeginRun);
0428
0429
0430 registry.watchPreModuleGlobalEndRun(this, &NVProfilerService::preModuleGlobalEndRun);
0431 registry.watchPostModuleGlobalEndRun(this, &NVProfilerService::postModuleGlobalEndRun);
0432
0433
0434 registry.watchPreModuleGlobalBeginLumi(this, &NVProfilerService::preModuleGlobalBeginLumi);
0435 registry.watchPostModuleGlobalBeginLumi(this, &NVProfilerService::postModuleGlobalBeginLumi);
0436
0437
0438 registry.watchPreModuleGlobalEndLumi(this, &NVProfilerService::preModuleGlobalEndLumi);
0439 registry.watchPostModuleGlobalEndLumi(this, &NVProfilerService::postModuleGlobalEndLumi);
0440
0441
0442 registry.watchPreModuleStreamBeginRun(this, &NVProfilerService::preModuleStreamBeginRun);
0443 registry.watchPostModuleStreamBeginRun(this, &NVProfilerService::postModuleStreamBeginRun);
0444
0445
0446 registry.watchPreModuleStreamEndRun(this, &NVProfilerService::preModuleStreamEndRun);
0447 registry.watchPostModuleStreamEndRun(this, &NVProfilerService::postModuleStreamEndRun);
0448
0449
0450 registry.watchPreModuleStreamBeginLumi(this, &NVProfilerService::preModuleStreamBeginLumi);
0451 registry.watchPostModuleStreamBeginLumi(this, &NVProfilerService::postModuleStreamBeginLumi);
0452
0453
0454 registry.watchPreModuleStreamEndLumi(this, &NVProfilerService::preModuleStreamEndLumi);
0455 registry.watchPostModuleStreamEndLumi(this, &NVProfilerService::postModuleStreamEndLumi);
0456
0457
0458 registry.watchPreModuleEventAcquire(this, &NVProfilerService::preModuleEventAcquire);
0459 registry.watchPostModuleEventAcquire(this, &NVProfilerService::postModuleEventAcquire);
0460
0461
0462 registry.watchPreModuleEvent(this, &NVProfilerService::preModuleEvent);
0463 registry.watchPostModuleEvent(this, &NVProfilerService::postModuleEvent);
0464
0465
0466 registry.watchPreModuleEventDelayedGet(this, &NVProfilerService::preModuleEventDelayedGet);
0467 registry.watchPostModuleEventDelayedGet(this, &NVProfilerService::postModuleEventDelayedGet);
0468
0469
0470 registry.watchPreEventReadFromSource(this, &NVProfilerService::preEventReadFromSource);
0471 registry.watchPostEventReadFromSource(this, &NVProfilerService::postEventReadFromSource);
0472 }
0473
0474 NVProfilerService::~NVProfilerService() {
0475 for (unsigned int sid = 0; sid < stream_domain_.size(); ++sid) {
0476 nvtxDomainDestroy(stream_domain_[sid]);
0477 }
0478 nvtxDomainDestroy(global_domain_);
0479 cudaProfilerStop();
0480 }
0481
0482 void NVProfilerService::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0483 edm::ParameterSetDescription desc;
0484 desc.addUntracked<std::vector<std::string>>("highlightModules", {})->setComment("");
0485 desc.addUntracked<bool>("showModulePrefetching", false)
0486 ->setComment("Show the stack of dependencies that requested to run a module.");
0487 desc.addUntracked<bool>("skipFirstEvent", false)
0488 ->setComment(
0489 "Start profiling after the first event has completed.\nWith multiple streams, ignore transitions belonging "
0490 "to events started in parallel to the first event.\nRequires running nvprof with the '--profile-from-start "
0491 "off' option.");
0492 descriptions.add("NVProfilerService", desc);
0493 descriptions.setComment(R"(This Service provides CMSSW-aware annotations to nvprof/nvvm.
0494
0495 Notes on nvprof options:
0496 - the option '--profile-from-start off' should be used if skipFirstEvent is True.
0497 - the option '--cpu-profiling on' currently results in cmsRun being stuck at the beginning of the job.
0498 - the option '--cpu-thread-tracing on' is not compatible with jemalloc, and should only be used with cmsRunGlibC.)");
0499 }
0500
0501 void NVProfilerService::preallocate(edm::service::SystemBounds const& bounds) {
0502 std::stringstream out;
0503 out << "preallocate: " << bounds.maxNumberOfConcurrentRuns() << " concurrent runs, "
0504 << bounds.maxNumberOfConcurrentLuminosityBlocks() << " luminosity sections, " << bounds.maxNumberOfStreams()
0505 << " streams\nrunning on" << bounds.maxNumberOfThreads() << " threads";
0506 nvtxDomainMark(global_domain_, out.str().c_str());
0507
0508 auto concurrentStreams = bounds.maxNumberOfStreams();
0509
0510 stream_domain_.resize(concurrentStreams);
0511 for (unsigned int sid = 0; sid < concurrentStreams; ++sid) {
0512 stream_domain_[sid] = nvtxDomainCreate(fmt::sprintf("EDM Stream %d", sid).c_str());
0513 }
0514
0515 event_.resize(concurrentStreams);
0516 stream_modules_.resize(concurrentStreams);
0517 if (skipFirstEvent_) {
0518 globalFirstEventDone_ = false;
0519 std::vector<std::atomic<bool>> tmp(concurrentStreams);
0520 for (auto& element : tmp)
0521 std::atomic_init(&element, false);
0522 streamFirstEventDone_ = std::move(tmp);
0523 }
0524 }
0525
0526 void NVProfilerService::preBeginJob(edm::PathsAndConsumesOfModulesBase const& pathsAndConsumes,
0527 edm::ProcessContext const& pc) {
0528 nvtxDomainMark(global_domain_, "preBeginJob");
0529
0530
0531
0532 unsigned int modules = pathsAndConsumes.allModules().size() + 1;
0533 global_modules_.resize(modules, nvtxInvalidRangeId);
0534 for (unsigned int sid = 0; sid < stream_modules_.size(); ++sid) {
0535 stream_modules_[sid].resize(modules, nvtxInvalidRangeId);
0536 }
0537 }
0538
0539 void NVProfilerService::postBeginJob() {
0540 if (not skipFirstEvent_ or globalFirstEventDone_) {
0541 nvtxDomainMark(global_domain_, "postBeginJob");
0542 }
0543 }
0544
0545 void NVProfilerService::postEndJob() {
0546 if (not skipFirstEvent_ or globalFirstEventDone_) {
0547 nvtxDomainMark(global_domain_, "postEndJob");
0548 }
0549 }
0550
0551 void NVProfilerService::preSourceEvent(edm::StreamID sid) {
0552 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0553 nvtxDomainRangePush(stream_domain_[sid], "source");
0554 }
0555 }
0556
0557 void NVProfilerService::postSourceEvent(edm::StreamID sid) {
0558 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0559 nvtxDomainRangePop(stream_domain_[sid]);
0560 }
0561 }
0562
0563 void NVProfilerService::preSourceLumi(edm::LuminosityBlockIndex index) {
0564 if (not skipFirstEvent_ or globalFirstEventDone_) {
0565 nvtxDomainRangePush(global_domain_, "source lumi");
0566 }
0567 }
0568
0569 void NVProfilerService::postSourceLumi(edm::LuminosityBlockIndex index) {
0570 if (not skipFirstEvent_ or globalFirstEventDone_) {
0571 nvtxDomainRangePop(global_domain_);
0572 }
0573 }
0574
0575 void NVProfilerService::preSourceRun(edm::RunIndex index) {
0576 if (not skipFirstEvent_ or globalFirstEventDone_) {
0577 nvtxDomainRangePush(global_domain_, "source run");
0578 }
0579 }
0580
0581 void NVProfilerService::postSourceRun(edm::RunIndex index) {
0582 if (not skipFirstEvent_ or globalFirstEventDone_) {
0583 nvtxDomainRangePop(global_domain_);
0584 }
0585 }
0586
0587 void NVProfilerService::preOpenFile(std::string const& lfn) {
0588 if (not skipFirstEvent_ or globalFirstEventDone_) {
0589 nvtxDomainRangePush(global_domain_, ("open file "s + lfn).c_str());
0590 }
0591 }
0592
0593 void NVProfilerService::postOpenFile(std::string const& lfn) {
0594 if (not skipFirstEvent_ or globalFirstEventDone_) {
0595 nvtxDomainRangePop(global_domain_);
0596 }
0597 }
0598
0599 void NVProfilerService::preCloseFile(std::string const& lfn) {
0600 if (not skipFirstEvent_ or globalFirstEventDone_) {
0601 nvtxDomainRangePush(global_domain_, ("close file "s + lfn).c_str());
0602 }
0603 }
0604
0605 void NVProfilerService::postCloseFile(std::string const& lfn) {
0606 if (not skipFirstEvent_ or globalFirstEventDone_) {
0607 nvtxDomainRangePop(global_domain_);
0608 }
0609 }
0610
0611 void NVProfilerService::preModuleBeginStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0612 auto sid = sc.streamID();
0613 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0614 auto mid = mcc.moduleDescription()->id();
0615 auto const& label = mcc.moduleDescription()->moduleLabel();
0616 auto const& msg = label + " begin stream";
0617 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0618 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0619 }
0620 }
0621
0622 void NVProfilerService::postModuleBeginStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0623 auto sid = sc.streamID();
0624 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0625 auto mid = mcc.moduleDescription()->id();
0626 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0627 stream_modules_[sid][mid] = nvtxInvalidRangeId;
0628 }
0629 }
0630
0631 void NVProfilerService::preModuleEndStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0632 auto sid = sc.streamID();
0633 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0634 auto mid = mcc.moduleDescription()->id();
0635 auto const& label = mcc.moduleDescription()->moduleLabel();
0636 auto const& msg = label + " end stream";
0637 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0638 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0639 }
0640 }
0641
0642 void NVProfilerService::postModuleEndStream(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0643 auto sid = sc.streamID();
0644 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0645 auto mid = mcc.moduleDescription()->id();
0646 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0647 stream_modules_[sid][mid] = nvtxInvalidRangeId;
0648 }
0649 }
0650
0651 void NVProfilerService::preGlobalBeginRun(edm::GlobalContext const& gc) {
0652 if (not skipFirstEvent_ or globalFirstEventDone_) {
0653 nvtxDomainRangePush(global_domain_, "global begin run");
0654 }
0655 }
0656
0657 void NVProfilerService::postGlobalBeginRun(edm::GlobalContext const& gc) {
0658 if (not skipFirstEvent_ or globalFirstEventDone_) {
0659 nvtxDomainRangePop(global_domain_);
0660 }
0661 }
0662
0663 void NVProfilerService::preGlobalEndRun(edm::GlobalContext const& gc) {
0664 if (not skipFirstEvent_ or globalFirstEventDone_) {
0665 nvtxDomainRangePush(global_domain_, "global end run");
0666 }
0667 }
0668
0669 void NVProfilerService::postGlobalEndRun(edm::GlobalContext const& gc) {
0670 if (not skipFirstEvent_ or globalFirstEventDone_) {
0671 nvtxDomainRangePop(global_domain_);
0672 }
0673 }
0674
0675 void NVProfilerService::preStreamBeginRun(edm::StreamContext const& sc) {
0676 auto sid = sc.streamID();
0677 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0678 nvtxDomainRangePush(stream_domain_[sid], "stream begin run");
0679 }
0680 }
0681
0682 void NVProfilerService::postStreamBeginRun(edm::StreamContext const& sc) {
0683 auto sid = sc.streamID();
0684 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0685 nvtxDomainRangePop(stream_domain_[sid]);
0686 }
0687 }
0688
0689 void NVProfilerService::preStreamEndRun(edm::StreamContext const& sc) {
0690 auto sid = sc.streamID();
0691 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0692 nvtxDomainRangePush(stream_domain_[sid], "stream end run");
0693 }
0694 }
0695
0696 void NVProfilerService::postStreamEndRun(edm::StreamContext const& sc) {
0697 auto sid = sc.streamID();
0698 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0699 nvtxDomainRangePop(stream_domain_[sid]);
0700 }
0701 }
0702
0703 void NVProfilerService::preGlobalBeginLumi(edm::GlobalContext const& gc) {
0704 if (not skipFirstEvent_ or globalFirstEventDone_) {
0705 nvtxDomainRangePush(global_domain_, "global begin lumi");
0706 }
0707 }
0708
0709 void NVProfilerService::postGlobalBeginLumi(edm::GlobalContext const& gc) {
0710 if (not skipFirstEvent_ or globalFirstEventDone_) {
0711 nvtxDomainRangePop(global_domain_);
0712 }
0713 }
0714
0715 void NVProfilerService::preGlobalEndLumi(edm::GlobalContext const& gc) {
0716 if (not skipFirstEvent_ or globalFirstEventDone_) {
0717 nvtxDomainRangePush(global_domain_, "global end lumi");
0718 }
0719 }
0720
0721 void NVProfilerService::postGlobalEndLumi(edm::GlobalContext const& gc) {
0722 if (not skipFirstEvent_ or globalFirstEventDone_) {
0723 nvtxDomainRangePop(global_domain_);
0724 }
0725 }
0726
0727 void NVProfilerService::preStreamBeginLumi(edm::StreamContext const& sc) {
0728 auto sid = sc.streamID();
0729 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0730 nvtxDomainRangePush(stream_domain_[sid], "stream begin lumi");
0731 }
0732 }
0733
0734 void NVProfilerService::postStreamBeginLumi(edm::StreamContext const& sc) {
0735 auto sid = sc.streamID();
0736 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0737 nvtxDomainRangePop(stream_domain_[sid]);
0738 }
0739 }
0740
0741 void NVProfilerService::preStreamEndLumi(edm::StreamContext const& sc) {
0742 auto sid = sc.streamID();
0743 nvtxDomainRangePush(stream_domain_[sid], "stream end lumi");
0744 }
0745
0746 void NVProfilerService::postStreamEndLumi(edm::StreamContext const& sc) {
0747 auto sid = sc.streamID();
0748 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0749 nvtxDomainRangePop(stream_domain_[sid]);
0750 }
0751 }
0752
0753 void NVProfilerService::preEvent(edm::StreamContext const& sc) {
0754 auto sid = sc.streamID();
0755 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0756 event_[sid] = nvtxDomainRangeStartColor(stream_domain_[sid], "event", nvtxDarkGreen);
0757 }
0758 }
0759
0760 void NVProfilerService::postEvent(edm::StreamContext const& sc) {
0761 auto sid = sc.streamID();
0762 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0763 nvtxDomainRangeEnd(stream_domain_[sid], event_[sid]);
0764 event_[sid] = nvtxInvalidRangeId;
0765 } else {
0766 streamFirstEventDone_[sid] = true;
0767 auto identity = [](bool x) { return x; };
0768 if (std::all_of(streamFirstEventDone_.begin(), streamFirstEventDone_.end(), identity)) {
0769 bool expected = false;
0770 if (globalFirstEventDone_.compare_exchange_strong(expected, true))
0771 cudaProfilerStart();
0772 }
0773 }
0774 }
0775
0776 void NVProfilerService::prePathEvent(edm::StreamContext const& sc, edm::PathContext const& pc) {
0777 auto sid = sc.streamID();
0778 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0779 nvtxDomainMark(global_domain_, ("before path "s + pc.pathName()).c_str());
0780 }
0781 }
0782
0783 void NVProfilerService::postPathEvent(edm::StreamContext const& sc,
0784 edm::PathContext const& pc,
0785 edm::HLTPathStatus const& hlts) {
0786 auto sid = sc.streamID();
0787 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0788 nvtxDomainMark(global_domain_, ("after path "s + pc.pathName()).c_str());
0789 }
0790 }
0791
0792 void NVProfilerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0793 auto sid = sc.streamID();
0794 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0795 auto mid = mcc.moduleDescription()->id();
0796 auto const& label = mcc.moduleDescription()->moduleLabel();
0797 auto const& msg = label + " prefetching";
0798 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0799 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColorLight(label));
0800 }
0801 }
0802
0803 void NVProfilerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0804 auto sid = sc.streamID();
0805 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0806 auto mid = mcc.moduleDescription()->id();
0807 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0808 stream_modules_[sid][mid] = nvtxInvalidRangeId;
0809 }
0810 }
0811
0812 void NVProfilerService::preModuleConstruction(edm::ModuleDescription const& desc) {
0813 if (not skipFirstEvent_) {
0814 auto mid = desc.id();
0815 global_modules_.grow_to_at_least(mid + 1);
0816 auto const& label = desc.moduleLabel();
0817 auto const& msg = label + " construction";
0818 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0819 }
0820 }
0821
0822 void NVProfilerService::postModuleConstruction(edm::ModuleDescription const& desc) {
0823 if (not skipFirstEvent_) {
0824 auto mid = desc.id();
0825 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0826 global_modules_[mid] = nvtxInvalidRangeId;
0827 }
0828 }
0829
0830 void NVProfilerService::preModuleDestruction(edm::ModuleDescription const& desc) {
0831 if (not skipFirstEvent_) {
0832 auto mid = desc.id();
0833 global_modules_.grow_to_at_least(mid + 1);
0834 auto const& label = desc.moduleLabel();
0835 auto const& msg = label + " destruction";
0836 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0837 }
0838 }
0839
0840 void NVProfilerService::postModuleDestruction(edm::ModuleDescription const& desc) {
0841 if (not skipFirstEvent_) {
0842 auto mid = desc.id();
0843 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0844 global_modules_[mid] = nvtxInvalidRangeId;
0845 }
0846 }
0847
0848 void NVProfilerService::preModuleBeginJob(edm::ModuleDescription const& desc) {
0849 if (not skipFirstEvent_) {
0850 auto mid = desc.id();
0851 auto const& label = desc.moduleLabel();
0852 auto const& msg = label + " begin job";
0853 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0854 }
0855 }
0856
0857 void NVProfilerService::postModuleBeginJob(edm::ModuleDescription const& desc) {
0858 if (not skipFirstEvent_) {
0859 auto mid = desc.id();
0860 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0861 global_modules_[mid] = nvtxInvalidRangeId;
0862 }
0863 }
0864
0865 void NVProfilerService::preModuleEndJob(edm::ModuleDescription const& desc) {
0866 if (not skipFirstEvent_ or globalFirstEventDone_) {
0867 auto mid = desc.id();
0868 auto const& label = desc.moduleLabel();
0869 auto const& msg = label + " end job";
0870 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
0871 }
0872 }
0873
0874 void NVProfilerService::postModuleEndJob(edm::ModuleDescription const& desc) {
0875 if (not skipFirstEvent_ or globalFirstEventDone_) {
0876 auto mid = desc.id();
0877 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
0878 global_modules_[mid] = nvtxInvalidRangeId;
0879 }
0880 }
0881
0882 void NVProfilerService::preModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0883 auto sid = sc.streamID();
0884 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0885 auto mid = mcc.moduleDescription()->id();
0886 auto const& label = mcc.moduleDescription()->moduleLabel();
0887 auto const& msg = label + " acquire";
0888 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0889 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0890 }
0891 }
0892
0893 void NVProfilerService::postModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0894 auto sid = sc.streamID();
0895 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0896 auto mid = mcc.moduleDescription()->id();
0897 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0898 stream_modules_[sid][mid] = nvtxInvalidRangeId;
0899 }
0900 }
0901
0902 void NVProfilerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0903 auto sid = sc.streamID();
0904 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0905 auto mid = mcc.moduleDescription()->id();
0906 auto const& label = mcc.moduleDescription()->moduleLabel();
0907 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0908 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], label.c_str(), labelColor(label));
0909 }
0910 }
0911
0912 void NVProfilerService::postModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0913 auto sid = sc.streamID();
0914 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0915 auto mid = mcc.moduleDescription()->id();
0916 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0917 stream_modules_[sid][mid] = nvtxInvalidRangeId;
0918 }
0919 }
0920
0921 void NVProfilerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0922
0923
0924
0925
0926
0927
0928
0929
0930
0931
0932 }
0933
0934 void NVProfilerService::postModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0935
0936
0937
0938
0939
0940
0941
0942
0943 }
0944
0945 void NVProfilerService::preEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0946
0947
0948
0949
0950
0951
0952
0953
0954
0955
0956 }
0957
0958 void NVProfilerService::postEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0959
0960
0961
0962
0963
0964
0965
0966
0967 }
0968
0969 void NVProfilerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0970 auto sid = sc.streamID();
0971 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0972 auto mid = mcc.moduleDescription()->id();
0973 auto const& label = mcc.moduleDescription()->moduleLabel();
0974 auto const& msg = label + " stream begin run";
0975 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0976 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0977 }
0978 }
0979
0980 void NVProfilerService::postModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0981 auto sid = sc.streamID();
0982 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0983 auto mid = mcc.moduleDescription()->id();
0984 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
0985 stream_modules_[sid][mid] = nvtxInvalidRangeId;
0986 }
0987 }
0988
0989 void NVProfilerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
0990 auto sid = sc.streamID();
0991 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
0992 auto mid = mcc.moduleDescription()->id();
0993 auto const& label = mcc.moduleDescription()->moduleLabel();
0994 auto const& msg = label + " stream end run";
0995 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
0996 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
0997 }
0998 }
0999
1000 void NVProfilerService::postModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1001 auto sid = sc.streamID();
1002 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1003 auto mid = mcc.moduleDescription()->id();
1004 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1005 stream_modules_[sid][mid] = nvtxInvalidRangeId;
1006 }
1007 }
1008
1009 void NVProfilerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1010 auto sid = sc.streamID();
1011 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1012 auto mid = mcc.moduleDescription()->id();
1013 auto const& label = mcc.moduleDescription()->moduleLabel();
1014 auto const& msg = label + " stream begin lumi";
1015 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1016 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1017 }
1018 }
1019
1020 void NVProfilerService::postModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1021 auto sid = sc.streamID();
1022 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1023 auto mid = mcc.moduleDescription()->id();
1024 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1025 stream_modules_[sid][mid] = nvtxInvalidRangeId;
1026 }
1027 }
1028
1029 void NVProfilerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1030 auto sid = sc.streamID();
1031 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1032 auto mid = mcc.moduleDescription()->id();
1033 auto const& label = mcc.moduleDescription()->moduleLabel();
1034 auto const& msg = label + " stream end lumi";
1035 assert(stream_modules_[sid][mid] == nvtxInvalidRangeId);
1036 stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain_[sid], msg.c_str(), labelColor(label));
1037 }
1038 }
1039
1040 void NVProfilerService::postModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1041 auto sid = sc.streamID();
1042 if (not skipFirstEvent_ or streamFirstEventDone_[sid]) {
1043 auto mid = mcc.moduleDescription()->id();
1044 nvtxDomainRangeEnd(stream_domain_[sid], stream_modules_[sid][mid]);
1045 stream_modules_[sid][mid] = nvtxInvalidRangeId;
1046 }
1047 }
1048
1049 void NVProfilerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1050 if (not skipFirstEvent_ or globalFirstEventDone_) {
1051 auto mid = mcc.moduleDescription()->id();
1052 auto const& label = mcc.moduleDescription()->moduleLabel();
1053 auto const& msg = label + " global begin run";
1054 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1055 }
1056 }
1057
1058 void NVProfilerService::postModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1059 if (not skipFirstEvent_ or globalFirstEventDone_) {
1060 auto mid = mcc.moduleDescription()->id();
1061 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1062 global_modules_[mid] = nvtxInvalidRangeId;
1063 }
1064 }
1065
1066 void NVProfilerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1067 if (not skipFirstEvent_ or globalFirstEventDone_) {
1068 auto mid = mcc.moduleDescription()->id();
1069 auto const& label = mcc.moduleDescription()->moduleLabel();
1070 auto const& msg = label + " global end run";
1071 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1072 }
1073 }
1074
1075 void NVProfilerService::postModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1076 if (not skipFirstEvent_ or globalFirstEventDone_) {
1077 auto mid = mcc.moduleDescription()->id();
1078 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1079 global_modules_[mid] = nvtxInvalidRangeId;
1080 }
1081 }
1082
1083 void NVProfilerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1084 if (not skipFirstEvent_ or globalFirstEventDone_) {
1085 auto mid = mcc.moduleDescription()->id();
1086 auto const& label = mcc.moduleDescription()->moduleLabel();
1087 auto const& msg = label + " global begin lumi";
1088 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1089 }
1090 }
1091
1092 void NVProfilerService::postModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1093 if (not skipFirstEvent_ or globalFirstEventDone_) {
1094 auto mid = mcc.moduleDescription()->id();
1095 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1096 global_modules_[mid] = nvtxInvalidRangeId;
1097 }
1098 }
1099
1100 void NVProfilerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1101 if (not skipFirstEvent_ or globalFirstEventDone_) {
1102 auto mid = mcc.moduleDescription()->id();
1103 auto const& label = mcc.moduleDescription()->moduleLabel();
1104 auto const& msg = label + " global end lumi";
1105 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1106 }
1107 }
1108
1109 void NVProfilerService::postModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1110 if (not skipFirstEvent_ or globalFirstEventDone_) {
1111 auto mid = mcc.moduleDescription()->id();
1112 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1113 global_modules_[mid] = nvtxInvalidRangeId;
1114 }
1115 }
1116
1117 void NVProfilerService::preSourceConstruction(edm::ModuleDescription const& desc) {
1118 if (not skipFirstEvent_) {
1119 auto mid = desc.id();
1120 global_modules_.grow_to_at_least(mid + 1);
1121 auto const& label = desc.moduleLabel();
1122 auto const& msg = label + " construction";
1123 global_modules_[mid] = nvtxDomainRangeStartColor(global_domain_, msg.c_str(), labelColor(label));
1124 }
1125 }
1126
1127 void NVProfilerService::postSourceConstruction(edm::ModuleDescription const& desc) {
1128 if (not skipFirstEvent_) {
1129 auto mid = desc.id();
1130 nvtxDomainRangeEnd(global_domain_, global_modules_[mid]);
1131 global_modules_[mid] = nvtxInvalidRangeId;
1132 }
1133 }
1134
1135 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
1136 DEFINE_FWK_SERVICE(NVProfilerService);