Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2025-06-04 22:36:17

0001 // C++ headers
0002 #include <cmath>
0003 #include <fstream>
0004 #include <iomanip>
0005 #include <iostream>
0006 #include <limits>
0007 #include <mutex>
0008 #include <sstream>
0009 #include <string>
0010 
0011 // boost headers
0012 #include <boost/range/irange.hpp>
0013 
0014 // {fmt} headers
0015 #include <fmt/printf.h>
0016 
0017 // JSON headers
0018 #include <nlohmann/json.hpp>
0019 using json = nlohmann::json;
0020 
0021 // CMSSW headers
0022 #include "DQMServices/Core/interface/DQMStore.h"
0023 #include "DataFormats/Common/interface/HLTPathStatus.h"
0024 #include "DataFormats/Provenance/interface/EventID.h"
0025 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0026 #include "DataFormats/Scalers/interface/LumiScalers.h"
0027 #include "FWCore/Framework/interface/Event.h"
0028 #include "FWCore/Framework/interface/LuminosityBlock.h"
0029 #include "FWCore/ServiceRegistry/interface/PlaceInPathContext.h"
0030 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0031 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0032 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0033 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0034 #include "FWCore/Utilities/interface/Exception.h"
0035 #include "FWCore/Utilities/interface/StreamID.h"
0036 #include "HLTrigger/Timer/interface/memory_usage.h"
0037 #include "HLTrigger/Timer/interface/processor_model.h"
0038 #include "FastTimerService.h"
0039 
0040 using namespace std::literals;
0041 
0042 ///////////////////////////////////////////////////////////////////////////////
0043 
0044 namespace {
0045 
0046   // convert any std::chrono::duration to milliseconds
0047   template <class Rep, class Period>
0048   double ms(std::chrono::duration<Rep, Period> duration) {
0049     return std::chrono::duration_cast<std::chrono::duration<double, std::milli>>(duration).count();
0050   }
0051 
0052   // convert any boost::chrono::duration to milliseconds
0053   template <class Rep, class Period>
0054   double ms(boost::chrono::duration<Rep, Period> duration) {
0055     return boost::chrono::duration_cast<boost::chrono::duration<double, boost::milli>>(duration).count();
0056   }
0057 
0058   // convert a std::atomic<boost::chrono::nanoseconds::rep> to milliseconds
0059   double ms(std::atomic<boost::chrono::nanoseconds::rep> const& duration) {
0060     return boost::chrono::duration_cast<boost::chrono::duration<double, boost::milli>>(
0061                boost::chrono::nanoseconds(duration.load()))
0062         .count();
0063   }
0064 
0065   // convert from bytes to kilobytes, rounding down
0066   uint64_t kB(uint64_t bytes) { return bytes / 1024; }
0067 
0068   // convert from bytes to kilobytes, rounding down
0069   uint64_t kB(std::atomic<uint64_t> const& bytes) { return bytes.load() / 1024; }
0070 
0071 }  // namespace
0072 
0073 ///////////////////////////////////////////////////////////////////////////////
0074 
0075 // resources being monitored by the service
0076 
0077 // Resources
0078 
0079 FastTimerService::Resources::Resources()
0080     : time_thread(boost::chrono::nanoseconds::zero()),
0081       time_real(boost::chrono::nanoseconds::zero()),
0082       allocated(0ul),
0083       deallocated(0ul) {}
0084 
0085 void FastTimerService::Resources::reset() {
0086   time_thread = boost::chrono::nanoseconds::zero();
0087   time_real = boost::chrono::nanoseconds::zero();
0088   allocated = 0ul;
0089   deallocated = 0ul;
0090 }
0091 
0092 FastTimerService::Resources& FastTimerService::Resources::operator+=(Resources const& other) {
0093   time_thread += other.time_thread;
0094   time_real += other.time_real;
0095   allocated += other.allocated;
0096   deallocated += other.deallocated;
0097   return *this;
0098 }
0099 
0100 FastTimerService::Resources& FastTimerService::Resources::operator+=(AtomicResources const& other) {
0101   time_thread += boost::chrono::nanoseconds(other.time_thread.load());
0102   time_real += boost::chrono::nanoseconds(other.time_real.load());
0103   allocated += other.allocated.load();
0104   deallocated += other.deallocated.load();
0105   return *this;
0106 }
0107 
0108 FastTimerService::Resources FastTimerService::Resources::operator+(Resources const& other) const {
0109   Resources result(*this);
0110   result += other;
0111   return result;
0112 }
0113 
0114 FastTimerService::Resources FastTimerService::Resources::operator+(AtomicResources const& other) const {
0115   Resources result(*this);
0116   result += other;
0117   return result;
0118 }
0119 
0120 // AtomicResources
0121 // operation on the whole object are not atomic, as the operations
0122 // on the individual fields could be interleaved; however, accumulation
0123 // of results should yield the correct result.
0124 
0125 FastTimerService::AtomicResources::AtomicResources()
0126     : time_thread(0ul), time_real(0ul), allocated(0ul), deallocated(0ul) {}
0127 
0128 FastTimerService::AtomicResources::AtomicResources(AtomicResources const& other)
0129     : time_thread(other.time_thread.load()),
0130       time_real(other.time_real.load()),
0131       allocated(other.allocated.load()),
0132       deallocated(other.deallocated.load()) {}
0133 
0134 void FastTimerService::AtomicResources::reset() {
0135   time_thread = 0ul;
0136   time_real = 0ul;
0137   allocated = 0ul;
0138   deallocated = 0ul;
0139 }
0140 
0141 FastTimerService::AtomicResources& FastTimerService::AtomicResources::operator=(AtomicResources const& other) {
0142   time_thread = other.time_thread.load();
0143   time_real = other.time_real.load();
0144   allocated = other.allocated.load();
0145   deallocated = other.deallocated.load();
0146   return *this;
0147 }
0148 
0149 FastTimerService::AtomicResources& FastTimerService::AtomicResources::operator+=(AtomicResources const& other) {
0150   time_thread += other.time_thread.load();
0151   time_real += other.time_real.load();
0152   allocated += other.allocated.load();
0153   deallocated += other.deallocated.load();
0154   return *this;
0155 }
0156 
0157 FastTimerService::AtomicResources& FastTimerService::AtomicResources::operator+=(Resources const& other) {
0158   time_thread += other.time_thread.count();
0159   time_real += other.time_real.count();
0160   allocated += other.allocated;
0161   deallocated += other.deallocated;
0162   return *this;
0163 }
0164 
0165 FastTimerService::AtomicResources FastTimerService::AtomicResources::operator+(AtomicResources const& other) const {
0166   AtomicResources result(*this);
0167   result += other;
0168   return result;
0169 }
0170 
0171 FastTimerService::Resources FastTimerService::AtomicResources::operator+(Resources const& other) const {
0172   return other + *this;
0173 }
0174 
0175 // ResourcesPerModule
0176 
0177 FastTimerService::ResourcesPerModule::ResourcesPerModule() noexcept : total(), events(0), has_acquire(false) {}
0178 
0179 void FastTimerService::ResourcesPerModule::reset() noexcept {
0180   total.reset();
0181   events = 0;
0182   has_acquire = false;
0183 }
0184 
0185 FastTimerService::ResourcesPerModule& FastTimerService::ResourcesPerModule::operator+=(ResourcesPerModule const& other) {
0186   total += other.total;
0187   events += other.events;
0188   has_acquire = has_acquire or other.has_acquire;
0189   return *this;
0190 }
0191 
0192 FastTimerService::ResourcesPerModule FastTimerService::ResourcesPerModule::operator+(
0193     ResourcesPerModule const& other) const {
0194   ResourcesPerModule result(*this);
0195   result += other;
0196   return result;
0197 }
0198 
0199 // ResourcesPerPath
0200 
0201 void FastTimerService::ResourcesPerPath::reset() {
0202   active.reset();
0203   total.reset();
0204   last = 0;
0205   status = false;
0206 }
0207 
0208 FastTimerService::ResourcesPerPath& FastTimerService::ResourcesPerPath::operator+=(ResourcesPerPath const& other) {
0209   active += other.active;
0210   total += other.total;
0211   last = 0;  // summing these makes no sense, reset them instead
0212   status = false;
0213   return *this;
0214 }
0215 
0216 FastTimerService::ResourcesPerPath FastTimerService::ResourcesPerPath::operator+(ResourcesPerPath const& other) const {
0217   ResourcesPerPath result(*this);
0218   result += other;
0219   return result;
0220 }
0221 
0222 // ResourcesPerProcess
0223 
0224 FastTimerService::ResourcesPerProcess::ResourcesPerProcess(ProcessCallGraph::ProcessType const& process)
0225     : total(), paths(process.paths_.size()), endpaths(process.endPaths_.size()) {}
0226 
0227 void FastTimerService::ResourcesPerProcess::reset() {
0228   total.reset();
0229   for (auto& path : paths)
0230     path.reset();
0231   for (auto& path : endpaths)
0232     path.reset();
0233 }
0234 
0235 FastTimerService::ResourcesPerProcess& FastTimerService::ResourcesPerProcess::operator+=(
0236     ResourcesPerProcess const& other) {
0237   total += other.total;
0238   assert(paths.size() == other.paths.size());
0239   for (unsigned int i : boost::irange(0ul, paths.size()))
0240     paths[i] += other.paths[i];
0241   assert(endpaths.size() == other.endpaths.size());
0242   for (unsigned int i : boost::irange(0ul, endpaths.size()))
0243     endpaths[i] += other.endpaths[i];
0244   return *this;
0245 }
0246 
0247 FastTimerService::ResourcesPerProcess FastTimerService::ResourcesPerProcess::operator+(
0248     ResourcesPerProcess const& other) const {
0249   ResourcesPerProcess result(*this);
0250   result += other;
0251   return result;
0252 }
0253 
0254 // ResourcesPerJob
0255 
0256 FastTimerService::ResourcesPerJob::ResourcesPerJob(ProcessCallGraph const& job,
0257                                                    std::vector<GroupOfModules> const& groups)
0258     : highlight{groups.size()}, modules{job.size()}, process{job.processDescription()}, events{0} {}
0259 
0260 void FastTimerService::ResourcesPerJob::reset() {
0261   total.reset();
0262   overhead.reset();
0263   idle.reset();
0264   eventsetup.reset();
0265   event.reset();
0266   for (auto& module : highlight)
0267     module.reset();
0268   for (auto& module : modules)
0269     module.reset();
0270   process.reset();
0271   events = 0;
0272 }
0273 
0274 FastTimerService::ResourcesPerJob& FastTimerService::ResourcesPerJob::operator+=(ResourcesPerJob const& other) {
0275   total += other.total;
0276   overhead += other.overhead;
0277   idle += other.idle;
0278   eventsetup += other.eventsetup;
0279   event += other.event;
0280   assert(highlight.size() == other.highlight.size());
0281   for (unsigned int i : boost::irange(0ul, highlight.size()))
0282     highlight[i] += other.highlight[i];
0283   assert(modules.size() == other.modules.size());
0284   for (unsigned int i : boost::irange(0ul, modules.size()))
0285     modules[i] += other.modules[i];
0286   process += other.process;
0287   events += other.events;
0288   return *this;
0289 }
0290 
0291 FastTimerService::ResourcesPerJob FastTimerService::ResourcesPerJob::operator+(ResourcesPerJob const& other) const {
0292   ResourcesPerJob result(*this);
0293   result += other;
0294   return result;
0295 }
0296 
0297 // per-thread measurements
0298 
0299 // Measurement
0300 
0301 FastTimerService::Measurement::Measurement() noexcept { measure(); }
0302 
0303 void FastTimerService::Measurement::measure() noexcept {
0304 #ifdef DEBUG_THREAD_CONCURRENCY
0305   id = std::this_thread::get_id();
0306 #endif  // DEBUG_THREAD_CONCURRENCY
0307   time_thread = boost::chrono::thread_clock::now();
0308   time_real = boost::chrono::high_resolution_clock::now();
0309   allocated = memory_usage::allocated();
0310   deallocated = memory_usage::deallocated();
0311 }
0312 
0313 void FastTimerService::Measurement::measure_and_store(Resources& store) noexcept {
0314 #ifdef DEBUG_THREAD_CONCURRENCY
0315   assert(std::this_thread::get_id() == id);
0316 #endif  // DEBUG_THREAD_CONCURRENCY
0317   auto new_time_thread = boost::chrono::thread_clock::now();
0318   auto new_time_real = boost::chrono::high_resolution_clock::now();
0319   auto new_allocated = memory_usage::allocated();
0320   auto new_deallocated = memory_usage::deallocated();
0321   store.time_thread = new_time_thread - time_thread;
0322   store.time_real = new_time_real - time_real;
0323   store.allocated = new_allocated - allocated;
0324   store.deallocated = new_deallocated - deallocated;
0325   time_thread = new_time_thread;
0326   time_real = new_time_real;
0327   allocated = new_allocated;
0328   deallocated = new_deallocated;
0329 }
0330 
0331 void FastTimerService::Measurement::measure_and_accumulate(Resources& store) noexcept {
0332 #ifdef DEBUG_THREAD_CONCURRENCY
0333   assert(std::this_thread::get_id() == id);
0334 #endif  // DEBUG_THREAD_CONCURRENCY
0335   auto new_time_thread = boost::chrono::thread_clock::now();
0336   auto new_time_real = boost::chrono::high_resolution_clock::now();
0337   auto new_allocated = memory_usage::allocated();
0338   auto new_deallocated = memory_usage::deallocated();
0339   store.time_thread += new_time_thread - time_thread;
0340   store.time_real += new_time_real - time_real;
0341   store.allocated += new_allocated - allocated;
0342   store.deallocated += new_deallocated - deallocated;
0343   time_thread = new_time_thread;
0344   time_real = new_time_real;
0345   allocated = new_allocated;
0346   deallocated = new_deallocated;
0347 }
0348 
0349 void FastTimerService::Measurement::measure_and_accumulate(AtomicResources& store) noexcept {
0350 #ifdef DEBUG_THREAD_CONCURRENCY
0351   assert(std::this_thread::get_id() == id);
0352 #endif  // DEBUG_THREAD_CONCURRENCY
0353   auto new_time_thread = boost::chrono::thread_clock::now();
0354   auto new_time_real = boost::chrono::high_resolution_clock::now();
0355   auto new_allocated = memory_usage::allocated();
0356   auto new_deallocated = memory_usage::deallocated();
0357   store.time_thread += boost::chrono::duration_cast<boost::chrono::nanoseconds>(new_time_thread - time_thread).count();
0358   store.time_real += boost::chrono::duration_cast<boost::chrono::nanoseconds>(new_time_real - time_real).count();
0359   store.allocated += new_allocated - allocated;
0360   store.deallocated += new_deallocated - deallocated;
0361   time_thread = new_time_thread;
0362   time_real = new_time_real;
0363   allocated = new_allocated;
0364   deallocated = new_deallocated;
0365 }
0366 
0367 ///////////////////////////////////////////////////////////////////////////////
0368 
0369 void FastTimerService::PlotsPerElement::book(dqm::reco::DQMStore::IBooker& booker,
0370                                              std::string const& name,
0371                                              std::string const& title,
0372                                              PlotRanges const& ranges,
0373                                              unsigned int lumisections,
0374                                              bool byls) {
0375   int time_bins = (int)std::ceil(ranges.time_range / ranges.time_resolution);
0376   int mem_bins = (int)std::ceil(ranges.memory_range / ranges.memory_resolution);
0377   std::string y_title_ms = fmt::sprintf("events / %.1f ms", ranges.time_resolution);
0378   std::string y_title_kB = fmt::sprintf("events / %.1f kB", ranges.memory_resolution);
0379 
0380   // MonitorElement::setStatOverflows(kTRUE) includes underflows and overflows in the computation of mean and RMS
0381   time_thread_ =
0382       booker.book1D(name + " time_thread", title + " processing time (cpu)", time_bins, 0., ranges.time_range);
0383   time_thread_->setXTitle("processing time [ms]");
0384   time_thread_->setYTitle(y_title_ms);
0385   time_thread_->setStatOverflows(kTRUE);
0386 
0387   time_real_ = booker.book1D(name + " time_real", title + " processing time (real)", time_bins, 0., ranges.time_range);
0388   time_real_->setXTitle("processing time [ms]");
0389   time_real_->setYTitle(y_title_ms);
0390   time_real_->setStatOverflows(kTRUE);
0391 
0392   if (memory_usage::is_available()) {
0393     allocated_ = booker.book1D(name + " allocated", title + " allocated memory", mem_bins, 0., ranges.memory_range);
0394     allocated_->setXTitle("memory [kB]");
0395     allocated_->setYTitle(y_title_kB);
0396     allocated_->setStatOverflows(kTRUE);
0397 
0398     deallocated_ =
0399         booker.book1D(name + " deallocated", title + " deallocated memory", mem_bins, 0., ranges.memory_range);
0400     deallocated_->setXTitle("memory [kB]");
0401     deallocated_->setYTitle(y_title_kB);
0402     deallocated_->setStatOverflows(kTRUE);
0403   }
0404 
0405   if (not byls)
0406     return;
0407 
0408   time_thread_byls_ = booker.bookProfile(name + " time_thread_byls",
0409                                          title + " processing time (cpu) vs. lumisection",
0410                                          lumisections,
0411                                          0.5,
0412                                          lumisections + 0.5,
0413                                          time_bins,
0414                                          0.,
0415                                          std::numeric_limits<double>::infinity(),
0416                                          " ");
0417   time_thread_byls_->setXTitle("lumisection");
0418   time_thread_byls_->setYTitle("processing time [ms]");
0419   time_thread_byls_->setStatOverflows(kTRUE);
0420 
0421   time_real_byls_ = booker.bookProfile(name + " time_real_byls",
0422                                        title + " processing time (real) vs. lumisection",
0423                                        lumisections,
0424                                        0.5,
0425                                        lumisections + 0.5,
0426                                        time_bins,
0427                                        0.,
0428                                        std::numeric_limits<double>::infinity(),
0429                                        " ");
0430   time_real_byls_->setXTitle("lumisection");
0431   time_real_byls_->setYTitle("processing time [ms]");
0432   time_real_byls_->setStatOverflows(kTRUE);
0433 
0434   if (memory_usage::is_available()) {
0435     allocated_byls_ = booker.bookProfile(name + " allocated_byls",
0436                                          title + " allocated memory vs. lumisection",
0437                                          lumisections,
0438                                          0.5,
0439                                          lumisections + 0.5,
0440                                          mem_bins,
0441                                          0.,
0442                                          std::numeric_limits<double>::infinity(),
0443                                          " ");
0444     allocated_byls_->setXTitle("lumisection");
0445     allocated_byls_->setYTitle("memory [kB]");
0446     allocated_byls_->setStatOverflows(kTRUE);
0447 
0448     deallocated_byls_ = booker.bookProfile(name + " deallocated_byls",
0449                                            title + " deallocated memory vs. lumisection",
0450                                            lumisections,
0451                                            0.5,
0452                                            lumisections + 0.5,
0453                                            mem_bins,
0454                                            0.,
0455                                            std::numeric_limits<double>::infinity(),
0456                                            " ");
0457     deallocated_byls_->setXTitle("lumisection");
0458     deallocated_byls_->setYTitle("memory [kB]");
0459     deallocated_byls_->setStatOverflows(kTRUE);
0460   }
0461 }
0462 
0463 void FastTimerService::PlotsPerElement::fill(Resources const& data, unsigned int lumisection) {
0464   if (time_thread_)
0465     time_thread_->Fill(ms(data.time_thread));
0466 
0467   if (time_thread_byls_)
0468     time_thread_byls_->Fill(lumisection, ms(data.time_thread));
0469 
0470   if (time_real_)
0471     time_real_->Fill(ms(data.time_real));
0472 
0473   if (time_real_byls_)
0474     time_real_byls_->Fill(lumisection, ms(data.time_real));
0475 
0476   if (allocated_)
0477     allocated_->Fill(kB(data.allocated));
0478 
0479   if (allocated_byls_)
0480     allocated_byls_->Fill(lumisection, kB(data.allocated));
0481 
0482   if (deallocated_)
0483     deallocated_->Fill(kB(data.deallocated));
0484 
0485   if (deallocated_byls_)
0486     deallocated_byls_->Fill(lumisection, kB(data.deallocated));
0487 }
0488 
0489 void FastTimerService::PlotsPerElement::fill(AtomicResources const& data, unsigned int lumisection) {
0490   if (time_thread_)
0491     time_thread_->Fill(ms(boost::chrono::nanoseconds(data.time_thread.load())));
0492 
0493   if (time_thread_byls_)
0494     time_thread_byls_->Fill(lumisection, ms(boost::chrono::nanoseconds(data.time_thread.load())));
0495 
0496   if (time_real_)
0497     time_real_->Fill(ms(boost::chrono::nanoseconds(data.time_real.load())));
0498 
0499   if (time_real_byls_)
0500     time_real_byls_->Fill(lumisection, ms(boost::chrono::nanoseconds(data.time_real.load())));
0501 
0502   if (allocated_)
0503     allocated_->Fill(kB(data.allocated));
0504 
0505   if (allocated_byls_)
0506     allocated_byls_->Fill(lumisection, kB(data.allocated));
0507 
0508   if (deallocated_)
0509     deallocated_->Fill(kB(data.deallocated));
0510 
0511   if (deallocated_byls_)
0512     deallocated_byls_->Fill(lumisection, kB(data.deallocated));
0513 }
0514 
0515 void FastTimerService::PlotsPerElement::fill_fraction(Resources const& data,
0516                                                       Resources const& part,
0517                                                       unsigned int lumisection) {
0518   float total;
0519   float fraction;
0520 
0521   total = ms(data.time_thread);
0522   fraction = (total > 0.) ? (ms(part.time_thread) / total) : 0.;
0523   if (time_thread_)
0524     time_thread_->Fill(total, fraction);
0525 
0526   if (time_thread_byls_)
0527     time_thread_byls_->Fill(lumisection, total, fraction);
0528 
0529   total = ms(data.time_real);
0530   fraction = (total > 0.) ? (ms(part.time_real) / total) : 0.;
0531   if (time_real_)
0532     time_real_->Fill(total, fraction);
0533 
0534   if (time_real_byls_)
0535     time_real_byls_->Fill(lumisection, total, fraction);
0536 
0537   total = kB(data.allocated);
0538   fraction = (total > 0.) ? (kB(part.allocated) / total) : 0.;
0539   if (allocated_)
0540     allocated_->Fill(total, fraction);
0541 
0542   if (allocated_byls_)
0543     allocated_byls_->Fill(lumisection, total, fraction);
0544 
0545   total = kB(data.deallocated);
0546   fraction = (total > 0.) ? (kB(part.deallocated) / total) : 0.;
0547   if (deallocated_)
0548     deallocated_->Fill(total, fraction);
0549 
0550   if (deallocated_byls_)
0551     deallocated_byls_->Fill(lumisection, total, fraction);
0552 }
0553 
0554 void FastTimerService::PlotsPerPath::book(dqm::reco::DQMStore::IBooker& booker,
0555                                           std::string const& prefixDir,
0556                                           ProcessCallGraph const& job,
0557                                           ProcessCallGraph::PathType const& path,
0558                                           PlotRanges const& ranges,
0559                                           unsigned int lumisections,
0560                                           bool byls) {
0561   const std::string basedir = booker.pwd();
0562   std::string folderName = basedir + "/" + prefixDir + path.name_;
0563   fixForDQM(folderName);
0564   booker.setCurrentFolder(folderName);
0565 
0566   total_.book(booker, "path", path.name_, ranges, lumisections, byls);
0567 
0568   // MonitorElement::setStatOverflows(kTRUE) includes underflows and overflows in the computation of mean and RMS
0569   unsigned int bins = path.modules_and_dependencies_.size();
0570   module_counter_ = booker.book1DD("module_counter", "module counter", bins + 1, -0.5, bins + 0.5);
0571   module_counter_->setYTitle("events");
0572   module_counter_->setStatOverflows(kTRUE);
0573   module_time_thread_total_ =
0574       booker.book1DD("module_time_thread_total", "total module time (cpu)", bins, -0.5, bins - 0.5);
0575   module_time_thread_total_->setYTitle("processing time [ms]");
0576   module_time_thread_total_->setStatOverflows(kTRUE);
0577   module_time_real_total_ =
0578       booker.book1DD("module_time_real_total", "total module time (real)", bins, -0.5, bins - 0.5);
0579   module_time_real_total_->setYTitle("processing time [ms]");
0580   module_time_real_total_->setStatOverflows(kTRUE);
0581   if (memory_usage::is_available()) {
0582     module_allocated_total_ =
0583         booker.book1DD("module_allocated_total", "total allocated memory", bins, -0.5, bins - 0.5);
0584     module_allocated_total_->setYTitle("memory [kB]");
0585     module_allocated_total_->setStatOverflows(kTRUE);
0586     module_deallocated_total_ =
0587         booker.book1DD("module_deallocated_total", "total deallocated memory", bins, -0.5, bins - 0.5);
0588     module_deallocated_total_->setYTitle("memory [kB]");
0589     module_deallocated_total_->setStatOverflows(kTRUE);
0590   }
0591   for (unsigned int bin : boost::irange(0u, bins)) {
0592     auto const& module = job[path.modules_and_dependencies_[bin]];
0593     std::string const& label =
0594         module.scheduled_ ? module.module_.moduleLabel() : module.module_.moduleLabel() + " (unscheduled)";
0595     module_counter_->setBinLabel(bin + 1, label);
0596     module_time_thread_total_->setBinLabel(bin + 1, label);
0597     module_time_real_total_->setBinLabel(bin + 1, label);
0598     if (memory_usage::is_available()) {
0599       module_allocated_total_->setBinLabel(bin + 1, label);
0600       module_deallocated_total_->setBinLabel(bin + 1, label);
0601     }
0602   }
0603   module_counter_->setBinLabel(bins + 1, "");
0604 
0605   booker.setCurrentFolder(basedir);
0606 }
0607 
0608 void FastTimerService::PlotsPerPath::fill(ProcessCallGraph::PathType const& description,
0609                                           ResourcesPerJob const& data,
0610                                           ResourcesPerPath const& path,
0611                                           unsigned int ls) {
0612   // fill the total path time
0613   total_.fill(path.total, ls);
0614 
0615   // fill the modules that actually ran and the total time spent in each od them
0616   for (unsigned int i = 0; i < path.last; ++i) {
0617     auto const& module = data.modules[description.modules_and_dependencies_[i]];
0618     if (module_counter_)
0619       module_counter_->Fill(i);
0620 
0621     if (module_time_thread_total_)
0622       module_time_thread_total_->Fill(i, ms(module.total.time_thread));
0623 
0624     if (module_time_real_total_)
0625       module_time_real_total_->Fill(i, ms(module.total.time_real));
0626 
0627     if (module_allocated_total_)
0628       module_allocated_total_->Fill(i, kB(module.total.allocated));
0629 
0630     if (module_deallocated_total_)
0631       module_deallocated_total_->Fill(i, kB(module.total.deallocated));
0632   }
0633   if (module_counter_ and path.status)
0634     module_counter_->Fill(path.last);
0635 }
0636 
0637 FastTimerService::PlotsPerProcess::PlotsPerProcess(ProcessCallGraph::ProcessType const& process)
0638     : event_(), paths_(process.paths_.size()), endpaths_(process.endPaths_.size()) {}
0639 
0640 void FastTimerService::PlotsPerProcess::book(dqm::reco::DQMStore::IBooker& booker,
0641                                              ProcessCallGraph const& job,
0642                                              ProcessCallGraph::ProcessType const& process,
0643                                              PlotRanges const& event_ranges,
0644                                              PlotRanges const& path_ranges,
0645                                              unsigned int lumisections,
0646                                              bool bypath,
0647                                              bool byls) {
0648   const std::string basedir = booker.pwd();
0649   event_.book(booker, "process " + process.name_, "process " + process.name_, event_ranges, lumisections, byls);
0650   if (bypath) {
0651     booker.setCurrentFolder(basedir + "/process " + process.name_ + " paths");
0652     for (unsigned int id : boost::irange(0ul, paths_.size())) {
0653       paths_[id].book(booker, "path ", job, process.paths_[id], path_ranges, lumisections, byls);
0654     }
0655     for (unsigned int id : boost::irange(0ul, endpaths_.size())) {
0656       endpaths_[id].book(booker, "endpath ", job, process.endPaths_[id], path_ranges, lumisections, byls);
0657     }
0658     booker.setCurrentFolder(basedir);
0659   }
0660 }
0661 
0662 void FastTimerService::PlotsPerProcess::fill(ProcessCallGraph::ProcessType const& description,
0663                                              ResourcesPerJob const& data,
0664                                              ResourcesPerProcess const& process,
0665                                              unsigned int ls) {
0666   // fill process event plots
0667   event_.fill(process.total, ls);
0668 
0669   // fill all paths plots
0670   for (unsigned int id : boost::irange(0ul, paths_.size()))
0671     paths_[id].fill(description.paths_[id], data, process.paths[id], ls);
0672 
0673   // fill all endpaths plots
0674   for (unsigned int id : boost::irange(0ul, endpaths_.size()))
0675     endpaths_[id].fill(description.endPaths_[id], data, process.endpaths[id], ls);
0676 }
0677 
0678 FastTimerService::PlotsPerJob::PlotsPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups)
0679     : highlight_{groups.size()}, modules_{job.size()}, process_{job.processDescription()} {}
0680 
0681 void FastTimerService::PlotsPerJob::book(dqm::reco::DQMStore::IBooker& booker,
0682                                          ProcessCallGraph const& job,
0683                                          std::vector<GroupOfModules> const& groups,
0684                                          PlotRanges const& event_ranges,
0685                                          PlotRanges const& path_ranges,
0686                                          PlotRanges const& module_ranges,
0687                                          unsigned int lumisections,
0688                                          bool bymodule,
0689                                          bool bypath,
0690                                          bool byls,
0691                                          bool transitions) {
0692   const std::string basedir = booker.pwd();
0693 
0694   // event summary plots
0695   event_.book(booker, "event", "Event", event_ranges, lumisections, byls);
0696   event_ex_.book(booker, "explicit", "Event (explicit)", event_ranges, lumisections, byls);
0697   overhead_.book(booker, "overhead", "Overhead", event_ranges, lumisections, byls);
0698   idle_.book(booker, "idle", "Idle", event_ranges, lumisections, byls);
0699 
0700   modules_[job.source().id()].book(booker, "source", "Source", module_ranges, lumisections, byls);
0701 
0702   if (transitions) {
0703     lumi_.book(booker, "lumi", "LumiSection transitions", event_ranges, lumisections, byls);
0704 
0705     run_.book(booker, "run", "Run transtions", event_ranges, lumisections, false);
0706   }
0707 
0708   // plot the time spent in few given groups of modules
0709   for (unsigned int group : boost::irange(0ul, groups.size())) {
0710     auto const& label = groups[group].label;
0711     highlight_[group].book(booker, "highlight " + label, "Highlight " + label, event_ranges, lumisections, byls);
0712   }
0713 
0714   // process plots (event, modules, paths and endpaths)
0715   auto const& process = job.processDescription();
0716   process_.book(booker, job, process, event_ranges, path_ranges, lumisections, bypath, byls);
0717 
0718   if (bymodule) {
0719     booker.setCurrentFolder(basedir + "/process " + process.name_ + " modules");
0720     for (unsigned int id : process.modules_) {
0721       std::string const& module_label = job.module(id).moduleLabel();
0722       std::string safe_label = module_label;
0723       fixForDQM(safe_label);
0724       modules_[id].book(booker, safe_label, module_label, module_ranges, lumisections, byls);
0725     }
0726     booker.setCurrentFolder(basedir);
0727   }
0728 }
0729 
0730 void FastTimerService::PlotsPerJob::fill(ProcessCallGraph const& job, ResourcesPerJob const& data, unsigned int ls) {
0731   // fill total event plots
0732   event_.fill(data.total, ls);
0733   event_ex_.fill(data.event, ls);
0734   overhead_.fill(data.overhead, ls);
0735   idle_.fill(data.idle, ls);
0736 
0737   // fill highltight plots
0738   for (unsigned int group : boost::irange(0ul, highlight_.size()))
0739     highlight_[group].fill_fraction(data.total, data.highlight[group], ls);
0740 
0741   // fill modules plots
0742   for (unsigned int id : boost::irange(0ul, modules_.size()))
0743     modules_[id].fill(data.modules[id].total, ls);
0744 
0745   process_.fill(job.processDescription(), data, data.process, ls);
0746 }
0747 
0748 void FastTimerService::PlotsPerJob::fill_run(AtomicResources const& data) {
0749   // fill run transition plots
0750   run_.fill(data, 0);
0751 }
0752 
0753 void FastTimerService::PlotsPerJob::fill_lumi(AtomicResources const& data, unsigned int ls) {
0754   // fill lumisection transition plots
0755   lumi_.fill(data, ls);
0756 }
0757 
0758 ///////////////////////////////////////////////////////////////////////////////
0759 
0760 FastTimerService::FastTimerService(const edm::ParameterSet& config, edm::ActivityRegistry& registry)
0761     :  // configuration
0762       callgraph_(),
0763       // job configuration
0764       concurrent_lumis_(0),
0765       concurrent_runs_(0),
0766       concurrent_streams_(0),
0767       concurrent_threads_(0),
0768       print_event_summary_(config.getUntrackedParameter<bool>("printEventSummary")),
0769       print_run_summary_(config.getUntrackedParameter<bool>("printRunSummary")),
0770       print_job_summary_(config.getUntrackedParameter<bool>("printJobSummary")),
0771       // JSON configuration
0772       //write_json_per_event_(config.getUntrackedParameter<bool>("writeJSONByEvent")),
0773       //write_json_per_ls_(config.getUntrackedParameter<bool>("writeJSONByLumiSection")),
0774       //write_json_per_run_(config.getUntrackedParameter<bool>("writeJSONByRun")),
0775       write_json_summary_(config.getUntrackedParameter<bool>("writeJSONSummary")),
0776       json_filename_(config.getUntrackedParameter<std::string>("jsonFileName")),
0777       // DQM configuration
0778       enable_dqm_(config.getUntrackedParameter<bool>("enableDQM")),
0779       enable_dqm_bymodule_(config.getUntrackedParameter<bool>("enableDQMbyModule")),
0780       enable_dqm_bypath_(config.getUntrackedParameter<bool>("enableDQMbyPath")),
0781       enable_dqm_byls_(config.getUntrackedParameter<bool>("enableDQMbyLumiSection")),
0782       enable_dqm_bynproc_(config.getUntrackedParameter<bool>("enableDQMbyProcesses")),
0783       enable_dqm_transitions_(config.getUntrackedParameter<bool>("enableDQMTransitions")),
0784       dqm_event_ranges_({config.getUntrackedParameter<double>("dqmTimeRange"),                  // ms
0785                          config.getUntrackedParameter<double>("dqmTimeResolution"),             // ms
0786                          config.getUntrackedParameter<double>("dqmMemoryRange"),                // kB
0787                          config.getUntrackedParameter<double>("dqmMemoryResolution")}),         // kB
0788       dqm_path_ranges_({config.getUntrackedParameter<double>("dqmPathTimeRange"),               // ms
0789                         config.getUntrackedParameter<double>("dqmPathTimeResolution"),          // ms
0790                         config.getUntrackedParameter<double>("dqmPathMemoryRange"),             // kB
0791                         config.getUntrackedParameter<double>("dqmPathMemoryResolution")}),      // kB
0792       dqm_module_ranges_({config.getUntrackedParameter<double>("dqmModuleTimeRange"),           // ms
0793                           config.getUntrackedParameter<double>("dqmModuleTimeResolution"),      // ms
0794                           config.getUntrackedParameter<double>("dqmModuleMemoryRange"),         // kB
0795                           config.getUntrackedParameter<double>("dqmModuleMemoryResolution")}),  // kB
0796       dqm_lumisections_range_(config.getUntrackedParameter<unsigned int>("dqmLumiSectionsRange")),
0797       dqm_path_(config.getUntrackedParameter<std::string>("dqmPath")),
0798       // highlight configuration
0799       highlight_module_psets_(config.getUntrackedParameter<std::vector<edm::ParameterSet>>("highlightModules")),
0800       highlight_modules_(highlight_module_psets_.size())  // filled in postBeginJob()
0801 {
0802   // start observing when a thread enters or leaves the TBB global thread arena
0803   tbb::task_scheduler_observer::observe();
0804 
0805   // register EDM call backs
0806   registry.watchPreallocate(this, &FastTimerService::preallocate);
0807   registry.watchLookupInitializationComplete(this, &FastTimerService::lookupInitializationComplete);
0808   registry.watchPostEndJob(this, &FastTimerService::postEndJob);
0809   registry.watchPreGlobalBeginRun(this, &FastTimerService::preGlobalBeginRun);
0810   //registry.watchPostGlobalBeginRun(         this, & FastTimerService::postGlobalBeginRun );
0811   //registry.watchPreGlobalEndRun(            this, & FastTimerService::preGlobalEndRun );
0812   registry.watchPostGlobalEndRun(this, &FastTimerService::postGlobalEndRun);
0813   registry.watchPreStreamBeginRun(this, &FastTimerService::preStreamBeginRun);
0814   //registry.watchPostStreamBeginRun(         this, & FastTimerService::postStreamBeginRun );
0815   //registry.watchPreStreamEndRun(            this, & FastTimerService::preStreamEndRun );
0816   registry.watchPostStreamEndRun(this, &FastTimerService::postStreamEndRun);
0817   registry.watchPreGlobalBeginLumi(this, &FastTimerService::preGlobalBeginLumi);
0818   //registry.watchPostGlobalBeginLumi(        this, & FastTimerService::postGlobalBeginLumi );
0819   //registry.watchPreGlobalEndLumi(           this, & FastTimerService::preGlobalEndLumi );
0820   registry.watchPostGlobalEndLumi(this, &FastTimerService::postGlobalEndLumi);
0821   registry.watchPreStreamBeginLumi(this, &FastTimerService::preStreamBeginLumi);
0822   //registry.watchPostStreamBeginLumi(        this, & FastTimerService::postStreamBeginLumi );
0823   //registry.watchPreStreamEndLumi(           this, & FastTimerService::preStreamEndLumi );
0824   registry.watchPostStreamEndLumi(this, &FastTimerService::postStreamEndLumi);
0825   registry.watchPreEvent(this, &FastTimerService::preEvent);
0826   registry.watchPostEvent(this, &FastTimerService::postEvent);
0827   registry.watchPrePathEvent(this, &FastTimerService::prePathEvent);
0828   registry.watchPostPathEvent(this, &FastTimerService::postPathEvent);
0829   registry.watchPreSourceConstruction(this, &FastTimerService::preSourceConstruction);
0830   //registry.watchPostSourceConstruction(     this, & FastTimerService::postSourceConstruction);
0831   registry.watchPreSourceRun(this, &FastTimerService::preSourceRun);
0832   registry.watchPostSourceRun(this, &FastTimerService::postSourceRun);
0833   registry.watchPreSourceLumi(this, &FastTimerService::preSourceLumi);
0834   registry.watchPostSourceLumi(this, &FastTimerService::postSourceLumi);
0835   registry.watchPreSourceEvent(this, &FastTimerService::preSourceEvent);
0836   registry.watchPostSourceEvent(this, &FastTimerService::postSourceEvent);
0837   //registry.watchPreModuleConstruction(      this, & FastTimerService::preModuleConstruction);
0838   //registry.watchPostModuleConstruction(     this, & FastTimerService::postModuleConstruction);
0839   //registry.watchPreModuleBeginJob(          this, & FastTimerService::preModuleBeginJob );
0840   //registry.watchPostModuleBeginJob(         this, & FastTimerService::postModuleBeginJob );
0841   //registry.watchPreModuleEndJob(            this, & FastTimerService::preModuleEndJob );
0842   //registry.watchPostModuleEndJob(           this, & FastTimerService::postModuleEndJob );
0843   //registry.watchPreModuleBeginStream(       this, & FastTimerService::preModuleBeginStream );
0844   //registry.watchPostModuleBeginStream(      this, & FastTimerService::postModuleBeginStream );
0845   //registry.watchPreModuleEndStream(         this, & FastTimerService::preModuleEndStream );
0846   //registry.watchPostModuleEndStream(        this, & FastTimerService::postModuleEndStream );
0847   registry.watchPreModuleGlobalBeginRun(this, &FastTimerService::preModuleGlobalBeginRun);
0848   registry.watchPostModuleGlobalBeginRun(this, &FastTimerService::postModuleGlobalBeginRun);
0849   registry.watchPreModuleGlobalEndRun(this, &FastTimerService::preModuleGlobalEndRun);
0850   registry.watchPostModuleGlobalEndRun(this, &FastTimerService::postModuleGlobalEndRun);
0851   registry.watchPreModuleGlobalBeginLumi(this, &FastTimerService::preModuleGlobalBeginLumi);
0852   registry.watchPostModuleGlobalBeginLumi(this, &FastTimerService::postModuleGlobalBeginLumi);
0853   registry.watchPreModuleGlobalEndLumi(this, &FastTimerService::preModuleGlobalEndLumi);
0854   registry.watchPostModuleGlobalEndLumi(this, &FastTimerService::postModuleGlobalEndLumi);
0855   registry.watchPreModuleStreamBeginRun(this, &FastTimerService::preModuleStreamBeginRun);
0856   registry.watchPostModuleStreamBeginRun(this, &FastTimerService::postModuleStreamBeginRun);
0857   registry.watchPreModuleStreamEndRun(this, &FastTimerService::preModuleStreamEndRun);
0858   registry.watchPostModuleStreamEndRun(this, &FastTimerService::postModuleStreamEndRun);
0859   registry.watchPreModuleStreamBeginLumi(this, &FastTimerService::preModuleStreamBeginLumi);
0860   registry.watchPostModuleStreamBeginLumi(this, &FastTimerService::postModuleStreamBeginLumi);
0861   registry.watchPreModuleStreamEndLumi(this, &FastTimerService::preModuleStreamEndLumi);
0862   registry.watchPostModuleStreamEndLumi(this, &FastTimerService::postModuleStreamEndLumi);
0863   //registry.watchPreModuleEventPrefetching(  this, & FastTimerService::preModuleEventPrefetching );
0864   //registry.watchPostModuleEventPrefetching( this, & FastTimerService::postModuleEventPrefetching );
0865   registry.watchPreModuleEventAcquire(this, &FastTimerService::preModuleEventAcquire);
0866   registry.watchPostModuleEventAcquire(this, &FastTimerService::postModuleEventAcquire);
0867   registry.watchPreModuleEvent(this, &FastTimerService::preModuleEvent);
0868   registry.watchPostModuleEvent(this, &FastTimerService::postModuleEvent);
0869   registry.watchPreModuleEventDelayedGet(this, &FastTimerService::preModuleEventDelayedGet);
0870   registry.watchPostModuleEventDelayedGet(this, &FastTimerService::postModuleEventDelayedGet);
0871   registry.watchPreEventReadFromSource(this, &FastTimerService::preEventReadFromSource);
0872   registry.watchPostEventReadFromSource(this, &FastTimerService::postEventReadFromSource);
0873   registry.watchPreESModule(this, &FastTimerService::preESModule);
0874   registry.watchPostESModule(this, &FastTimerService::postESModule);
0875 }
0876 
0877 void FastTimerService::ignoredSignal(const std::string& signal) const {
0878   LogDebug("FastTimerService") << "The FastTimerService received is currently not monitoring the signal \"" << signal
0879                                << "\".\n";
0880 }
0881 
0882 void FastTimerService::unsupportedSignal(const std::string& signal) const {
0883   // warn about each signal only once per job
0884   if (unsupported_signals_.insert(signal).second)
0885     edm::LogWarning("FastTimerService") << "The FastTimerService received the unsupported signal \"" << signal
0886                                         << "\".\n"
0887                                         << "Please report how to reproduce the issue to cms-hlt@cern.ch .";
0888 }
0889 
0890 void FastTimerService::preGlobalBeginRun(edm::GlobalContext const& gc) {
0891   ignoredSignal(__func__);
0892 
0893   // reset the run counters
0894   auto index = gc.runIndex();
0895   run_transition_[index].reset();
0896   run_summary_[index].reset();
0897 
0898   // book the DQM plots
0899   if (enable_dqm_) {
0900     // define a callback to book the MonitorElements
0901     auto bookTransactionCallback = [&, this](dqm::reco::DQMStore::IBooker& booker, dqm::reco::DQMStore::IGetter&) {
0902       auto scope = dqm::reco::DQMStore::IBooker::UseRunScope(booker);
0903       // we should really do this, but only DQMStore is allowed to touch it
0904       // We could move to postGlobalBeginRun, then the DQMStore has sure set it up.
0905       //booker.setRunLumi(gc.luminosityBlockID());
0906       booker.setCurrentFolder(dqm_path_);
0907       plots_->book(booker,
0908                    callgraph_,
0909                    highlight_modules_,
0910                    dqm_event_ranges_,
0911                    dqm_path_ranges_,
0912                    dqm_module_ranges_,
0913                    dqm_lumisections_range_,
0914                    enable_dqm_bymodule_,
0915                    enable_dqm_bypath_,
0916                    enable_dqm_byls_,
0917                    enable_dqm_transitions_);
0918     };
0919 
0920     // book MonitorElements for this stream
0921     edm::Service<dqm::legacy::DQMStore>()->meBookerGetter(bookTransactionCallback);
0922   }
0923 }
0924 
0925 void FastTimerService::postGlobalBeginRun(edm::GlobalContext const& gc) { ignoredSignal(__func__); }
0926 
0927 void FastTimerService::preStreamBeginRun(edm::StreamContext const& sc) { ignoredSignal(__func__); }
0928 
0929 void FastTimerService::fixForDQM(std::string& label) {
0930   // clean characters that are deemed unsafe for DQM
0931   // see the definition of `s_safe` in DQMServices/Core/src/DQMStore.cc
0932   static const auto safe_for_dqm = "/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-+=_()# "s;
0933   for (auto& c : label)
0934     if (safe_for_dqm.find(c) == std::string::npos)
0935       c = '_';
0936 }
0937 
0938 void FastTimerService::preallocate(edm::service::SystemBounds const& bounds) {
0939   concurrent_lumis_ = bounds.maxNumberOfConcurrentLuminosityBlocks();
0940   concurrent_runs_ = bounds.maxNumberOfConcurrentRuns();
0941   concurrent_streams_ = bounds.maxNumberOfStreams();
0942   concurrent_threads_ = bounds.maxNumberOfThreads();
0943 
0944   if (enable_dqm_bynproc_)
0945     dqm_path_ += fmt::sprintf(
0946         "/Running on %s with %d streams on %d threads", processor_model, concurrent_streams_, concurrent_threads_);
0947 
0948   // fix the DQM path to avoid invalid characters
0949   fixForDQM(dqm_path_);
0950 
0951   // allocate buffers to keep track of the resources spent in the lumi and run transitions
0952   lumi_transition_.resize(concurrent_lumis_);
0953   run_transition_.resize(concurrent_runs_);
0954 }
0955 
0956 void FastTimerService::preSourceConstruction(edm::ModuleDescription const& module) {
0957   callgraph_.preSourceConstruction(module);
0958 }
0959 
0960 void FastTimerService::lookupInitializationComplete(edm::PathsAndConsumesOfModulesBase const& pathsAndConsumes,
0961                                                     edm::ProcessContext const& context) {
0962   callgraph_.lookupInitializationComplete(pathsAndConsumes, context);
0963   unsigned int modules = callgraph_.size();
0964 
0965   // module highlights
0966   for (unsigned int group : boost::irange(0ul, highlight_module_psets_.size())) {
0967     // copy and sort for faster search via std::binary_search
0968     auto labels = highlight_module_psets_[group].getUntrackedParameter<std::vector<std::string>>("modules");
0969     std::sort(labels.begin(), labels.end());
0970 
0971     highlight_modules_[group].label = highlight_module_psets_[group].getUntrackedParameter<std::string>("label");
0972     highlight_modules_[group].modules.reserve(labels.size());
0973     // convert the module labels in module ids
0974     for (unsigned int i = 0; i < modules; ++i) {
0975       auto const& label = callgraph_.module(i).moduleLabel();
0976       if (std::binary_search(labels.begin(), labels.end(), label))
0977         highlight_modules_[group].modules.push_back(i);
0978     }
0979   }
0980   highlight_module_psets_.clear();
0981 
0982   // allocate the resource counters for each stream, process, path and module
0983   ResourcesPerJob temp(callgraph_, highlight_modules_);
0984   streams_.resize(concurrent_streams_, temp);
0985   run_summary_.resize(concurrent_runs_, temp);
0986   job_summary_ = temp;
0987 
0988   // check that the DQMStore service is available
0989   if (enable_dqm_ and not edm::Service<dqm::legacy::DQMStore>().isAvailable()) {
0990     // the DQMStore is not available, disable all DQM plots
0991     enable_dqm_ = false;
0992     edm::LogWarning("FastTimerService") << "The DQMStore is not avalable, the DQM plots will not be generated";
0993   }
0994 
0995   // allocate the structures to hold pointers to the DQM plots
0996   if (enable_dqm_) {
0997     plots_ = std::make_unique<PlotsPerJob>(callgraph_, highlight_modules_);
0998   }
0999 }
1000 
1001 void FastTimerService::postStreamBeginRun(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1002 
1003 void FastTimerService::preStreamEndRun(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1004 
1005 void FastTimerService::postStreamEndRun(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1006 
1007 void FastTimerService::preGlobalBeginLumi(edm::GlobalContext const& gc) {
1008   ignoredSignal(__func__);
1009 
1010   // reset the lumi counters
1011   auto index = gc.luminosityBlockIndex();
1012   lumi_transition_[index].reset();
1013 }
1014 
1015 void FastTimerService::postGlobalBeginLumi(edm::GlobalContext const& gc) { ignoredSignal(__func__); }
1016 
1017 void FastTimerService::preGlobalEndLumi(edm::GlobalContext const& gc) { ignoredSignal(__func__); }
1018 
1019 void FastTimerService::postGlobalEndLumi(edm::GlobalContext const& gc) {
1020   ignoredSignal(__func__);
1021 
1022   // handle the summaries
1023   auto index = gc.luminosityBlockIndex();
1024 
1025   edm::LogVerbatim out("FastReport");
1026   auto const& label =
1027       fmt::sprintf("run %d, lumisection %d", gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock());
1028   printTransition(out, lumi_transition_[index], label);
1029 
1030   if (enable_dqm_ and enable_dqm_transitions_) {
1031     plots_->fill_lumi(lumi_transition_[index], gc.luminosityBlockID().luminosityBlock());
1032   }
1033 }
1034 
1035 void FastTimerService::preStreamBeginLumi(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1036 
1037 void FastTimerService::postStreamBeginLumi(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1038 
1039 void FastTimerService::preStreamEndLumi(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1040 
1041 void FastTimerService::postStreamEndLumi(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1042 
1043 void FastTimerService::preGlobalEndRun(edm::GlobalContext const& gc) { ignoredSignal(__func__); }
1044 
1045 void FastTimerService::postGlobalEndRun(edm::GlobalContext const& gc) {
1046   ignoredSignal(__func__);
1047 
1048   // handle the summaries
1049   auto index = gc.runIndex();
1050 
1051   edm::LogVerbatim out("FastReport");
1052   auto const& label = fmt::sprintf("run %d", gc.luminosityBlockID().run());
1053   if (print_run_summary_) {
1054     printSummary(out, run_summary_[index], label);
1055   }
1056   printTransition(out, run_transition_[index], label);
1057 
1058   if (enable_dqm_ and enable_dqm_transitions_) {
1059     plots_->fill_run(run_transition_[index]);
1060   }
1061 }
1062 
1063 void FastTimerService::preSourceRun(edm::RunIndex index) { thread().measure_and_accumulate(job_summary_.overhead); }
1064 
1065 void FastTimerService::postSourceRun(edm::RunIndex index) { thread().measure_and_accumulate(run_transition_[index]); }
1066 
1067 void FastTimerService::preSourceLumi(edm::LuminosityBlockIndex index) {
1068   thread().measure_and_accumulate(job_summary_.overhead);
1069 }
1070 
1071 void FastTimerService::postSourceLumi(edm::LuminosityBlockIndex index) {
1072   thread().measure_and_accumulate(lumi_transition_[index]);
1073 }
1074 
1075 void FastTimerService::postEndJob() {
1076   // stop observing to avoid potential race conditions at exit
1077   tbb::task_scheduler_observer::observe(false);
1078   guard_.finalize();
1079   // print the job summaries
1080   if (print_job_summary_) {
1081     edm::LogVerbatim out("FastReport");
1082     printSummary(out, job_summary_, "Job");
1083   }
1084   if (write_json_summary_) {
1085     writeSummaryJSON(job_summary_, json_filename_);
1086   }
1087 }
1088 
1089 template <typename T>
1090 void FastTimerService::printHeader(T& out, std::string const& label) const {
1091   out << "FastReport ";
1092   if (label.size() < 60)
1093     for (unsigned int i = (60 - label.size()) / 2; i > 0; --i)
1094       out << '-';
1095   out << ' ' << label << " Summary ";
1096   if (label.size() < 60)
1097     for (unsigned int i = (59 - label.size()) / 2; i > 0; --i)
1098       out << '-';
1099   out << '\n';
1100 }
1101 
1102 template <typename T>
1103 void FastTimerService::printEventHeader(T& out, std::string const& label) const {
1104   out << "FastReport       CPU time      Real time      Allocated    Deallocated  " << label << "\n";
1105   //      FastReport  ########.# ms  ########.# ms  +######### kB  -######### kB  ...
1106 }
1107 
1108 template <typename T>
1109 void FastTimerService::printEventLine(T& out, Resources const& data, std::string const& label) const {
1110   out << fmt::sprintf("FastReport  %10.1f ms  %10.1f ms  %+10d kB  %+10d kB  %s\n",
1111                       ms(data.time_thread),
1112                       ms(data.time_real),
1113                       +static_cast<int64_t>(kB(data.allocated)),
1114                       -static_cast<int64_t>(kB(data.deallocated)),
1115                       label);
1116 }
1117 
1118 template <typename T>
1119 void FastTimerService::printEventLine(T& out, AtomicResources const& data, std::string const& label) const {
1120   out << fmt::sprintf("FastReport  %10.1f ms  %10.1f ms  %+10d kB  %+10d kB  %s\n",
1121                       ms(boost::chrono::nanoseconds(data.time_thread.load())),
1122                       ms(boost::chrono::nanoseconds(data.time_real.load())),
1123                       +static_cast<int64_t>(kB(data.allocated)),
1124                       -static_cast<int64_t>(kB(data.deallocated)),
1125                       label);
1126 }
1127 
1128 template <typename T>
1129 void FastTimerService::printEvent(T& out, ResourcesPerJob const& data) const {
1130   printHeader(out, "Event");
1131   printEventHeader(out, "Modules");
1132   auto const& source_d = callgraph_.source();
1133   auto const& source = data.modules[source_d.id()];
1134   printEventLine(out, source.total, source_d.moduleLabel());
1135   auto const& proc_d = callgraph_.processDescription();
1136   auto const& proc = data.process;
1137   printEventLine(out, proc.total, "process " + proc_d.name_);
1138   for (unsigned int m : proc_d.modules_) {
1139     auto const& module_d = callgraph_.module(m);
1140     auto const& module = data.modules[m];
1141     printEventLine(out, module.total, "  " + module_d.moduleLabel());
1142   }
1143   printEventLine(out, data.total, "total");
1144   out << '\n';
1145   printEventHeader(out, "Processes and Paths");
1146   printEventLine(out, source.total, source_d.moduleLabel());
1147   printEventLine(out, proc.total, "process " + proc_d.name_);
1148   for (unsigned int p = 0; p < proc.paths.size(); ++p) {
1149     auto const& name = proc_d.paths_[p].name_;
1150     auto const& path = proc.paths[p];
1151     printEventLine(out, path.active, name + " (only scheduled modules)");
1152     printEventLine(out, path.total, name + " (including dependencies)");
1153   }
1154   for (unsigned int p = 0; p < proc.endpaths.size(); ++p) {
1155     auto const& name = proc_d.endPaths_[p].name_;
1156     auto const& path = proc.endpaths[p];
1157     printEventLine(out, path.active, name + " (only scheduled modules)");
1158     printEventLine(out, path.total, name + " (including dependencies)");
1159   }
1160   printEventLine(out, data.total, "total");
1161   out << '\n';
1162   for (unsigned int group : boost::irange(0ul, highlight_modules_.size())) {
1163     printEventHeader(out, "Highlighted modules");
1164     for (unsigned int m : highlight_modules_[group].modules) {
1165       auto const& module_d = callgraph_.module(m);
1166       auto const& module = data.modules[m];
1167       printEventLine(out, module.total, "  " + module_d.moduleLabel());
1168     }
1169     printEventLine(out, data.highlight[group], highlight_modules_[group].label);
1170     out << '\n';
1171   }
1172 }
1173 
1174 template <typename T>
1175 void FastTimerService::printSummaryHeader(T& out, std::string const& label, bool detailed) const {
1176   // clang-format off
1177   if (detailed)
1178     out << "FastReport   CPU time avg.      when run  Real time avg.      when run     Alloc. avg.      when run   Dealloc. avg.      when run  ";
1179   //        FastReport  ########.# ms  ########.# ms  ########.# ms  ########.# ms  +######### kB  +######### kB  -######### kB  -######### kB  ...
1180   else
1181     out << "FastReport   CPU time avg.                Real time avg.                   Alloc. avg.                 Dealloc. avg.                ";
1182   //        FastReport  ########.# ms                 ########.# ms                 +######### kB                 -######### kB                 ...
1183   out << label << '\n';
1184   // clang-format on
1185 }
1186 
1187 template <typename T>
1188 void FastTimerService::printPathSummaryHeader(T& out, std::string const& label) const {
1189   // clang-format off
1190   out << "FastReport     CPU time sched. / depend.    Real time sched. / depend.       Alloc. sched. / depend.     Dealloc. sched. / depend.  ";
1191   //      FastReport  ########.# ms  ########.# ms  ########.# ms  ########.# ms  +######### kB  +######### kB  -######### kB  -######### kB  ...
1192   out << label << '\n';
1193   // clang-format on
1194 }
1195 
1196 template <typename T>
1197 void FastTimerService::printSummaryLine(T& out, Resources const& data, uint64_t events, std::string const& label) const {
1198   out << fmt::sprintf(
1199       // clang-format off
1200       "FastReport  %10.1f ms                 %10.1f ms                 %+10d kB                 %+10d kB                 %s\n",
1201       // clang-format on
1202       (events ? ms(data.time_thread) / events : 0),
1203       (events ? ms(data.time_real) / events : 0),
1204       (events ? +static_cast<int64_t>(kB(data.allocated) / events) : 0),
1205       (events ? -static_cast<int64_t>(kB(data.deallocated) / events) : 0),
1206       label);
1207 }
1208 
1209 template <typename T>
1210 void FastTimerService::printSummaryLine(
1211     T& out, AtomicResources const& data, uint64_t events, uint64_t active, std::string const& label) const {
1212   out << fmt::sprintf(
1213       // clang-format off
1214       "FastReport  %10.1f ms  %10.1f ms  %10.1f ms  %10.1f ms  %+10d kB  %+10d kB  %+10d kB  %+10d kB  %s\n",
1215       // clang-format on
1216       (events ? ms(data.time_thread) / events : 0),
1217       (active ? ms(data.time_thread) / active : 0),
1218       (events ? ms(data.time_real) / events : 0),
1219       (active ? ms(data.time_real) / active : 0),
1220       (events ? +static_cast<int64_t>(kB(data.allocated) / events) : 0),
1221       (active ? +static_cast<int64_t>(kB(data.allocated) / active) : 0),
1222       (events ? -static_cast<int64_t>(kB(data.deallocated) / events) : 0),
1223       (active ? -static_cast<int64_t>(kB(data.deallocated) / active) : 0),
1224       label);
1225 }
1226 
1227 template <typename T>
1228 void FastTimerService::printSummaryLine(T& out,
1229                                         AtomicResources const& data,
1230                                         uint64_t events,
1231                                         std::string const& label) const {
1232   out << fmt::sprintf(
1233       // clang-format off
1234       "FastReport  %10.1f ms                 %10.1f ms                 %+10d kB                 %+10d kB                 %s\n",
1235       // clang-format on
1236       (events ? ms(data.time_thread) / events : 0),
1237       (events ? ms(data.time_real) / events : 0),
1238       (events ? +static_cast<int64_t>(kB(data.allocated) / events) : 0),
1239       (events ? -static_cast<int64_t>(kB(data.deallocated) / events) : 0),
1240       label);
1241 }
1242 
1243 template <typename T>
1244 void FastTimerService::printSummaryLine(
1245     T& out, Resources const& data, uint64_t events, uint64_t active, std::string const& label) const {
1246   out << fmt::sprintf(
1247       "FastReport  %10.1f ms  %10.1f ms  %10.1f ms  %10.1f ms  %+10d kB  %+10d kB  %+10d kB  %+10d kB  %s\n",
1248       (events ? ms(data.time_thread) / events : 0),
1249       (active ? ms(data.time_thread) / active : 0),
1250       (events ? ms(data.time_real) / events : 0),
1251       (active ? ms(data.time_real) / active : 0),
1252       (events ? +static_cast<int64_t>(kB(data.allocated) / events) : 0),
1253       (active ? +static_cast<int64_t>(kB(data.allocated) / active) : 0),
1254       (events ? -static_cast<int64_t>(kB(data.deallocated) / events) : 0),
1255       (active ? -static_cast<int64_t>(kB(data.deallocated) / active) : 0),
1256       label);
1257 }
1258 
1259 template <typename T>
1260 void FastTimerService::printPathSummaryLine(
1261     T& out, Resources const& data, Resources const& total, uint64_t events, std::string const& label) const {
1262   out << fmt::sprintf(
1263       "FastReport  %10.1f ms  %10.1f ms  %10.1f ms  %10.1f ms  %+10d kB  %+10d kB  %+10d kB  %+10d kB  %s\n",
1264       (events ? ms(data.time_thread) / events : 0),
1265       (events ? ms(total.time_thread) / events : 0),
1266       (events ? ms(data.time_real) / events : 0),
1267       (events ? ms(total.time_real) / events : 0),
1268       (events ? +static_cast<int64_t>(kB(data.allocated) / events) : 0),
1269       (events ? +static_cast<int64_t>(kB(total.allocated) / events) : 0),
1270       (events ? -static_cast<int64_t>(kB(data.deallocated) / events) : 0),
1271       (events ? -static_cast<int64_t>(kB(total.deallocated) / events) : 0),
1272       label);
1273 }
1274 
1275 template <typename T>
1276 void FastTimerService::printSummary(T& out, ResourcesPerJob const& data, std::string const& label) const {
1277   printHeader(out, label);
1278   printSummaryHeader(out, "Modules", true);
1279   auto const& source_d = callgraph_.source();
1280   auto const& source = data.modules[source_d.id()];
1281   printSummaryLine(out, source.total, data.events, source.events, source_d.moduleLabel());
1282   auto const& proc_d = callgraph_.processDescription();
1283   auto const& proc = data.process;
1284   printSummaryLine(out, proc.total, data.events, "process " + proc_d.name_);
1285   for (unsigned int m : proc_d.modules_) {
1286     auto const& module_d = callgraph_.module(m);
1287     auto const& module = data.modules[m];
1288     printSummaryLine(out, module.total, data.events, module.events, "  " + module_d.moduleLabel());
1289   }
1290   printSummaryLine(out, data.total, data.events, "total");
1291   printSummaryLine(out, data.eventsetup, data.events, "eventsetup");
1292   printSummaryLine(out, data.overhead, data.events, "other");
1293   printSummaryLine(out, data.idle, data.events, "idle");
1294   out << '\n';
1295   printPathSummaryHeader(out, "Processes and Paths");
1296   printSummaryLine(out, source.total, data.events, source_d.moduleLabel());
1297   printSummaryLine(out, proc.total, data.events, "process " + proc_d.name_);
1298   for (unsigned int p = 0; p < proc.paths.size(); ++p) {
1299     auto const& name = proc_d.paths_[p].name_;
1300     auto const& path = proc.paths[p];
1301     printPathSummaryLine(out, path.active, path.total, data.events, "  " + name);
1302   }
1303   for (unsigned int p = 0; p < proc.endpaths.size(); ++p) {
1304     auto const& name = proc_d.endPaths_[p].name_;
1305     auto const& path = proc.endpaths[p];
1306     printPathSummaryLine(out, path.active, path.total, data.events, "  " + name);
1307   }
1308   printSummaryLine(out, data.total, data.events, "total");
1309   printSummaryLine(out, data.eventsetup, data.events, "eventsetup");
1310   printSummaryLine(out, data.overhead, data.events, "other");
1311   printSummaryLine(out, data.idle, data.events, "idle");
1312   out << '\n';
1313   for (unsigned int group : boost::irange(0ul, highlight_modules_.size())) {
1314     printSummaryHeader(out, "Highlighted modules", true);
1315     for (unsigned int m : highlight_modules_[group].modules) {
1316       auto const& module_d = callgraph_.module(m);
1317       auto const& module = data.modules[m];
1318       printSummaryLine(out, module.total, data.events, module.events, module_d.moduleLabel());
1319     }
1320     printSummaryLine(out, data.highlight[group], data.events, highlight_modules_[group].label);
1321     out << '\n';
1322   }
1323 }
1324 
1325 template <typename T>
1326 void FastTimerService::printTransition(T& out, AtomicResources const& data, std::string const& label) const {
1327   printEventHeader(out, "Transition");
1328   printEventLine(out, data, label);
1329 }
1330 
1331 template <typename T>
1332 json FastTimerService::encodeToJSON(std::string const& type,
1333                                     std::string const& label,
1334                                     unsigned int events,
1335                                     T const& data) const {
1336   return json{{"type", type},
1337               {"label", label},
1338               {"events", events},
1339               {"time_thread", ms(data.time_thread)},
1340               {"time_real", ms(data.time_real)},
1341               {"mem_alloc", kB(data.allocated)},
1342               {"mem_free", kB(data.deallocated)}};
1343 }
1344 
1345 json FastTimerService::encodeToJSON(edm::ModuleDescription const& module, ResourcesPerModule const& data) const {
1346   return encodeToJSON(module.moduleName(), module.moduleLabel(), data.events, data.total);
1347 }
1348 
1349 void FastTimerService::writeSummaryJSON(ResourcesPerJob const& data, std::string const& filename) const {
1350   json j;
1351 
1352   // write a description of the resources
1353   j["resources"] = json::array(
1354       {json{{"name", "time_real"}, {"description", "real time"}, {"unit", "ms"}, {"title", "Time"}},
1355        json{{"name", "time_thread"}, {"description", "cpu time"}, {"unit", "ms"}, {"title", "Time"}},
1356        json{{"name", "mem_alloc"}, {"description", "allocated memory"}, {"unit", "kB"}, {"title", "Memory"}},
1357        json{{"name", "mem_free"}, {"description", "deallocated memory"}, {"unit", "kB"}, {"title", "Memory"}}});
1358 
1359   // write the resources used by the job
1360   j["total"] = encodeToJSON("Job",
1361                             callgraph_.processDescription().name_,
1362                             data.events,
1363                             data.total + data.eventsetup + data.overhead + data.idle);
1364 
1365   // write the resources used by every module
1366   j["modules"] = json::array();
1367   for (unsigned int i = 0; i < callgraph_.size(); ++i) {
1368     auto const& module = callgraph_.module(i);
1369     auto const& data_m = data.modules[i];
1370     j["modules"].push_back(encodeToJSON(module, data_m));
1371   }
1372 
1373   // add an entry for the non-event transitions, modules, and idle states
1374   j["modules"].push_back(encodeToJSON("other", "other", data.events, data.overhead));
1375   j["modules"].push_back(encodeToJSON("eventsetup", "eventsetup", data.events, data.eventsetup));
1376   j["modules"].push_back(encodeToJSON("idle", "idle", data.events, data.idle));
1377 
1378   std::ofstream out(filename);
1379   out << std::setw(2) << j << std::flush;
1380 }
1381 
1382 void FastTimerService::preEvent(edm::StreamContext const& sc) { ignoredSignal(__func__); }
1383 
1384 void FastTimerService::postEvent(edm::StreamContext const& sc) {
1385   ignoredSignal(__func__);
1386 
1387   unsigned int sid = sc.streamID();
1388   auto& stream = streams_[sid];
1389   auto& process = callgraph_.processDescription();
1390 
1391   // measure the event resources as the sum of all modules' resources
1392   auto& data = stream.process.total;
1393   for (unsigned int id : process.modules_)
1394     data += stream.modules[id].total;
1395   stream.total += data;
1396 
1397   // handle the summaries and fill the plots
1398 
1399   // measure the event resources explicitly
1400   stream.event_measurement.measure_and_store(stream.event);
1401 
1402   // add to the event resources those used by source (which is not part of any process)
1403   unsigned int id = 0;
1404   stream.total += stream.modules[id].total;
1405 
1406   // highlighted modules
1407   for (unsigned int group : boost::irange(0ul, highlight_modules_.size()))
1408     for (unsigned int i : highlight_modules_[group].modules)
1409       stream.highlight[group] += stream.modules[i].total;
1410 
1411   // avoid concurrent access to the summary objects
1412   {
1413     std::lock_guard<std::mutex> guard(summary_mutex_);
1414     job_summary_ += stream;
1415     run_summary_[sc.runIndex()] += stream;
1416   }
1417 
1418   if (print_event_summary_) {
1419     edm::LogVerbatim out("FastReport");
1420     printEvent(out, stream);
1421   }
1422 
1423   if (enable_dqm_) {
1424     plots_->fill(callgraph_, stream, sc.eventID().luminosityBlock());
1425   }
1426 }
1427 
1428 void FastTimerService::preSourceEvent(edm::StreamID sid) {
1429   // clear the event counters
1430   auto& stream = streams_[sid];
1431   stream.reset();
1432   ++stream.events;
1433 
1434   // reuse the same measurement for the Source module and for the explicit begin of the Event
1435   auto& measurement = thread();
1436   measurement.measure_and_accumulate(stream.overhead);
1437   stream.event_measurement = measurement;
1438 }
1439 
1440 void FastTimerService::postSourceEvent(edm::StreamID sid) {
1441   edm::ModuleDescription const& md = callgraph_.source();
1442   unsigned int id = md.id();
1443   auto& stream = streams_[sid];
1444   auto& module = stream.modules[id];
1445 
1446   thread().measure_and_store(module.total);
1447   ++stream.modules[id].events;
1448 }
1449 
1450 void FastTimerService::prePathEvent(edm::StreamContext const& sc, edm::PathContext const& pc) {
1451   unsigned int sid = sc.streamID().value();
1452   unsigned int id = pc.pathID();
1453   auto& stream = streams_[sid];
1454   auto& data = pc.isEndPath() ? stream.process.endpaths[id] : stream.process.paths[id];
1455   data.status = false;
1456   data.last = 0;
1457 }
1458 
1459 void FastTimerService::postPathEvent(edm::StreamContext const& sc,
1460                                      edm::PathContext const& pc,
1461                                      edm::HLTPathStatus const& status) {
1462   unsigned int sid = sc.streamID().value();
1463   unsigned int id = pc.pathID();
1464   auto& stream = streams_[sid];
1465   auto& data = pc.isEndPath() ? stream.process.endpaths[id] : stream.process.paths[id];
1466 
1467   auto const& path =
1468       pc.isEndPath() ? callgraph_.processDescription().endPaths_[id] : callgraph_.processDescription().paths_[id];
1469   unsigned int index = path.modules_on_path_.empty() ? 0 : status.index() + 1;
1470   data.last = path.modules_on_path_.empty() ? 0 : path.last_dependency_of_module_[status.index()];
1471 
1472   for (unsigned int i = 0; i < index; ++i) {
1473     auto const& module = stream.modules[path.modules_on_path_[i]];
1474     data.active += module.total;
1475   }
1476   for (unsigned int i = 0; i < data.last; ++i) {
1477     auto const& module = stream.modules[path.modules_and_dependencies_[i]];
1478     data.total += module.total;
1479   }
1480 }
1481 
1482 void FastTimerService::preModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1483   unsigned int sid = sc.streamID().value();
1484   auto& stream = streams_[sid];
1485   thread().measure_and_accumulate(stream.overhead);
1486 }
1487 
1488 void FastTimerService::postModuleEventAcquire(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1489   edm::ModuleDescription const& md = *mcc.moduleDescription();
1490   unsigned int id = md.id();
1491   unsigned int sid = sc.streamID().value();
1492   auto& stream = streams_[sid];
1493   auto& module = stream.modules[id];
1494 
1495   module.has_acquire = true;
1496   thread().measure_and_store(module.total);
1497 }
1498 
1499 void FastTimerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1500   unsigned int sid = sc.streamID().value();
1501   auto& stream = streams_[sid];
1502   thread().measure_and_accumulate(stream.overhead);
1503 }
1504 
1505 void FastTimerService::postModuleEvent(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1506   edm::ModuleDescription const& md = *mcc.moduleDescription();
1507   unsigned int id = md.id();
1508   unsigned int sid = sc.streamID().value();
1509   auto& stream = streams_[sid];
1510   auto& module = stream.modules[id];
1511 
1512   if (module.has_acquire) {
1513     thread().measure_and_accumulate(module.total);
1514   } else {
1515     thread().measure_and_store(module.total);
1516   }
1517   ++module.events;
1518 }
1519 
1520 void FastTimerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1521   unsupportedSignal(__func__);
1522 }
1523 
1524 void FastTimerService::postModuleEventDelayedGet(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1525   unsupportedSignal(__func__);
1526 }
1527 
1528 void FastTimerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1529   ignoredSignal(__func__);
1530 }
1531 
1532 void FastTimerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1533   ignoredSignal(__func__);
1534 }
1535 
1536 void FastTimerService::preEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1537   if (mcc.state() == edm::ModuleCallingContext::State::kPrefetching) {
1538     auto& stream = streams_[sc.streamID()];
1539     thread().measure_and_accumulate(stream.overhead);
1540   }
1541 }
1542 
1543 void FastTimerService::postEventReadFromSource(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1544   if (mcc.state() == edm::ModuleCallingContext::State::kPrefetching) {
1545     edm::ModuleDescription const& md = callgraph_.source();
1546     unsigned int id = md.id();
1547     auto& stream = streams_[sc.streamID()];
1548     auto& module = stream.modules[id];
1549 
1550     thread().measure_and_accumulate(module.total);
1551   }
1552 }
1553 
1554 void FastTimerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1555   thread().measure_and_accumulate(job_summary_.overhead);
1556 }
1557 
1558 void FastTimerService::postModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1559   auto index = gc.runIndex();
1560   thread().measure_and_accumulate(run_transition_[index]);
1561 }
1562 
1563 void FastTimerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1564   thread().measure_and_accumulate(job_summary_.overhead);
1565 }
1566 
1567 void FastTimerService::postModuleGlobalEndRun(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1568   auto index = gc.runIndex();
1569   thread().measure_and_accumulate(run_transition_[index]);
1570 }
1571 
1572 void FastTimerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1573   thread().measure_and_accumulate(job_summary_.overhead);
1574 }
1575 
1576 void FastTimerService::postModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1577   auto index = gc.luminosityBlockIndex();
1578   thread().measure_and_accumulate(lumi_transition_[index]);
1579 }
1580 
1581 void FastTimerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1582   thread().measure_and_accumulate(job_summary_.overhead);
1583 }
1584 
1585 void FastTimerService::postModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::ModuleCallingContext const& mcc) {
1586   auto index = gc.luminosityBlockIndex();
1587   thread().measure_and_accumulate(lumi_transition_[index]);
1588 }
1589 
1590 void FastTimerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1591   thread().measure_and_accumulate(job_summary_.overhead);
1592 }
1593 
1594 void FastTimerService::postModuleStreamBeginRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1595   auto index = sc.runIndex();
1596   thread().measure_and_accumulate(run_transition_[index]);
1597 }
1598 
1599 void FastTimerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1600   thread().measure_and_accumulate(job_summary_.overhead);
1601 }
1602 
1603 void FastTimerService::postModuleStreamEndRun(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1604   auto index = sc.runIndex();
1605   thread().measure_and_accumulate(run_transition_[index]);
1606 }
1607 
1608 void FastTimerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1609   thread().measure_and_accumulate(job_summary_.overhead);
1610 }
1611 
1612 void FastTimerService::postModuleStreamBeginLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1613   auto index = sc.luminosityBlockIndex();
1614   thread().measure_and_accumulate(lumi_transition_[index]);
1615 }
1616 
1617 void FastTimerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1618   thread().measure_and_accumulate(job_summary_.overhead);
1619 }
1620 
1621 void FastTimerService::postModuleStreamEndLumi(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
1622   auto index = sc.luminosityBlockIndex();
1623   thread().measure_and_accumulate(lumi_transition_[index]);
1624 }
1625 
1626 void FastTimerService::preESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const& cc) {
1627   auto top = cc.getTopModuleCallingContext();
1628   if (top->type() == edm::ParentContext::Type::kPlaceInPath) {
1629     //Paths are only used when processing an Event
1630     unsigned int sid = top->parent().placeInPathContext()->pathContext()->streamContext()->streamID().value();
1631     auto& stream = streams_[sid];
1632     thread().measure_and_accumulate(stream.overhead);
1633   }
1634 }
1635 void FastTimerService::postESModule(edm::eventsetup::EventSetupRecordKey const&,
1636                                     edm::ESModuleCallingContext const& cc) {
1637   auto top = cc.getTopModuleCallingContext();
1638   if (top->type() == edm::ParentContext::Type::kPlaceInPath) {
1639     unsigned int sid = top->parent().placeInPathContext()->pathContext()->streamContext()->streamID().value();
1640     auto& stream = streams_[sid];
1641     thread().measure_and_accumulate(stream.eventsetup);
1642   }
1643 }
1644 
1645 FastTimerService::ThreadGuard::ThreadGuard() {
1646   auto err = ::pthread_key_create(&key_, retire_thread);
1647   if (err) {
1648     throw cms::Exception("FastTimerService") << "ThreadGuard key creation failed: " << ::strerror(err);
1649   }
1650 }
1651 
1652 // If this is a new thread, register it and return true
1653 bool FastTimerService::ThreadGuard::register_thread(FastTimerService::AtomicResources& r) {
1654   auto ptr = ::pthread_getspecific(key_);
1655 
1656   if (not ptr) {
1657     auto it = thread_resources_.emplace_back(std::make_shared<specific_t>(r));
1658     auto pp = new std::shared_ptr<specific_t>(*it);
1659     auto err = ::pthread_setspecific(key_, pp);
1660     if (err) {
1661       throw cms::Exception("FastTimerService") << "ThreadGuard pthread_setspecific failed: " << ::strerror(err);
1662     }
1663     return true;
1664   }
1665   return false;
1666 }
1667 
1668 std::shared_ptr<FastTimerService::ThreadGuard::specific_t>* FastTimerService::ThreadGuard::ptr(void* p) {
1669   return static_cast<std::shared_ptr<specific_t>*>(p);
1670 }
1671 
1672 // called when a thread exits
1673 void FastTimerService::ThreadGuard::retire_thread(void* p) {
1674   auto ps = ptr(p);
1675   auto expected = true;
1676   if ((*ps)->live_.compare_exchange_strong(expected, false)) {
1677     // account any resources used or freed by the thread before leaving the TBB pool
1678     (*ps)->measurement_.measure_and_accumulate((*ps)->resource_);
1679   }
1680   delete ps;
1681 }
1682 
1683 // finalize all threads that have not retired
1684 void FastTimerService::ThreadGuard::finalize() {
1685   for (auto& p : thread_resources_) {
1686     auto expected = true;
1687     if (p->live_.compare_exchange_strong(expected, false)) {
1688       p->measurement_.measure_and_accumulate(p->resource_);
1689     }
1690   }
1691 }
1692 
1693 FastTimerService::Measurement& FastTimerService::ThreadGuard::thread() {
1694   return (*ptr(::pthread_getspecific(key_)))->measurement_;
1695 }
1696 
1697 void FastTimerService::on_scheduler_entry(bool worker) {
1698   // The AtomicResources passed to register_thread are used to account the resources
1699   // used or freed by the thread after the last active measurement and before leaving the TBB pool.
1700   if (guard_.register_thread(job_summary_.idle)) {
1701     // initialise the measurement point for a thread that has newly joined the TBB pool
1702     thread().measure();
1703   } else {
1704     // An existing thread is re-joining the TBB pool.
1705     // Note: unsure whether the resources used outside of the TBB pool should be
1706     //   - not accounted:       thread().measure()
1707     //   - considered as idle:  thread().measure_and_accumulate(job_summary_.idle)
1708     //   - considered as other: thread().measure_and_accumulate(job_summary_.overhead)
1709     // FIXME "considered as other" has been seen to produce unreliable results; revert to "not accounted" for the time being.
1710     thread().measure();
1711   }
1712 }
1713 
1714 void FastTimerService::on_scheduler_exit(bool worker) {
1715   // Account for the resources used or freed by the thread after the last active measurement and before leaving the TBB pool.
1716   thread().measure_and_accumulate(job_summary_.idle);
1717 }
1718 
1719 FastTimerService::Measurement& FastTimerService::thread() { return guard_.thread(); }
1720 
1721 // describe the module's configuration
1722 void FastTimerService::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
1723   edm::ParameterSetDescription desc;
1724   desc.addUntracked<bool>("printEventSummary", false);
1725   desc.addUntracked<bool>("printRunSummary", true);
1726   desc.addUntracked<bool>("printJobSummary", true);
1727   // JSON configuration
1728   //desc.addUntracked<bool>("writeJSONByEvent", false);
1729   //desc.addUntracked<bool>("writeJSONByLumiSection", false);
1730   //desc.addUntracked<bool>("writeJSONByRun", false);
1731   desc.addUntracked<bool>("writeJSONSummary", false);
1732   desc.addUntracked<std::string>("jsonFileName", "resources.json");
1733   // DQM configuration
1734   desc.addUntracked<bool>("enableDQM", true);
1735   desc.addUntracked<bool>("enableDQMbyModule", false);
1736   desc.addUntracked<bool>("enableDQMbyPath", false);
1737   desc.addUntracked<bool>("enableDQMbyLumiSection", false);
1738   desc.addUntracked<bool>("enableDQMbyProcesses", false);
1739   desc.addUntracked<bool>("enableDQMTransitions", false);
1740   desc.addUntracked<double>("dqmTimeRange", 1000.);              // ms
1741   desc.addUntracked<double>("dqmTimeResolution", 5.);            // ms
1742   desc.addUntracked<double>("dqmMemoryRange", 1000000.);         // kB
1743   desc.addUntracked<double>("dqmMemoryResolution", 5000.);       // kB
1744   desc.addUntracked<double>("dqmPathTimeRange", 100.);           // ms
1745   desc.addUntracked<double>("dqmPathTimeResolution", 0.5);       // ms
1746   desc.addUntracked<double>("dqmPathMemoryRange", 1000000.);     // kB
1747   desc.addUntracked<double>("dqmPathMemoryResolution", 5000.);   // kB
1748   desc.addUntracked<double>("dqmModuleTimeRange", 40.);          // ms
1749   desc.addUntracked<double>("dqmModuleTimeResolution", 0.2);     // ms
1750   desc.addUntracked<double>("dqmModuleMemoryRange", 100000.);    // kB
1751   desc.addUntracked<double>("dqmModuleMemoryResolution", 500.);  // kB
1752   desc.addUntracked<unsigned>("dqmLumiSectionsRange", 2500);     // ~ 16 hours
1753   desc.addUntracked<std::string>("dqmPath", "HLT/TimerService");
1754 
1755   edm::ParameterSetDescription highlightModulesDescription;
1756   highlightModulesDescription.addUntracked<std::vector<std::string>>("modules", {});
1757   highlightModulesDescription.addUntracked<std::string>("label", "producers");
1758   desc.addVPSetUntracked("highlightModules", highlightModulesDescription, {});
1759 
1760   // # OBSOLETE - these parameters are ignored, they are left only not to break old configurations
1761   // they will not be printed in the generated cfi.py file
1762   desc.addOptionalNode(edm::ParameterDescription<bool>("useRealTimeClock", true, false), false)
1763       ->setComment("This parameter is obsolete and will be ignored.");
1764   desc.addOptionalNode(edm::ParameterDescription<bool>("enableTimingPaths", true, false), false)
1765       ->setComment("This parameter is obsolete and will be ignored.");
1766   desc.addOptionalNode(edm::ParameterDescription<bool>("enableTimingModules", true, false), false)
1767       ->setComment("This parameter is obsolete and will be ignored.");
1768   desc.addOptionalNode(edm::ParameterDescription<bool>("enableTimingExclusive", false, false), false)
1769       ->setComment("This parameter is obsolete and will be ignored.");
1770   desc.addOptionalNode(edm::ParameterDescription<bool>("enableTimingSummary", false, false), false)
1771       ->setComment("This parameter is obsolete and will be ignored.");
1772   desc.addOptionalNode(edm::ParameterDescription<bool>("skipFirstPath", false, false), false)
1773       ->setComment("This parameter is obsolete and will be ignored.");
1774   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyPathActive", false, false), false)
1775       ->setComment("This parameter is obsolete and will be ignored.");
1776   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyPathTotal", true, false), false)
1777       ->setComment("This parameter is obsolete and will be ignored.");
1778   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyPathOverhead", false, false), false)
1779       ->setComment("This parameter is obsolete and will be ignored.");
1780   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyPathDetails", false, false), false)
1781       ->setComment("This parameter is obsolete and will be ignored.");
1782   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyPathCounters", true, false), false)
1783       ->setComment("This parameter is obsolete and will be ignored.");
1784   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyPathExclusive", false, false), false)
1785       ->setComment("This parameter is obsolete and will be ignored.");
1786   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMbyModuleType", false, false), false)
1787       ->setComment("This parameter is obsolete and will be ignored.");
1788   desc.addOptionalNode(edm::ParameterDescription<bool>("enableDQMSummary", false, false), false)
1789       ->setComment("This parameter is obsolete and will be ignored.");
1790 
1791   descriptions.add("FastTimerService", desc);
1792 }
1793 
1794 // declare FastTimerService as a framework Service
1795 #include "FWCore/ServiceRegistry/interface/ServiceMaker.h"
1796 DEFINE_FWK_SERVICE(FastTimerService);