Timer/plugins/FastTimerService.h

0001 #ifndef FastTimerService_h
0002 #define FastTimerService_h
0003
0004 // system headers
0005 #include <unistd.h>
0006 #include <pthread.h>
0007
0008 // C++ headers
0009 #include <chrono>
0010 #include <cmath>
0011 #include <map>
0012 #include <mutex>
0013 #include <string>
0014 #include <unordered_map>
0015
0016 // boost headers
0017 #include <boost/chrono.hpp>
0018
0019 // tbb headers
0020 #include <oneapi/tbb/concurrent_unordered_set.h>
0021 #include <oneapi/tbb/enumerable_thread_specific.h>
0022 #include <oneapi/tbb/task_scheduler_observer.h>
0023
0024 // JSON headers
0025 #include <nlohmann/json_fwd.hpp>
0026 using json = nlohmann::json;
0027
0028 // CMSSW headers
0029 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0030 #include "FWCore/ServiceRegistry/interface/Service.h"
0031 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0032 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0033 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0034 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0035 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0036 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0037 #include "FWCore/ServiceRegistry/interface/ESModuleCallingContext.h"
0038 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0039 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0040 #include "FWCore/Framework/interface/TriggerNamesService.h"
0041 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0042 #include "DataFormats/Common/interface/HLTPathStatus.h"
0043 #include "DataFormats/Provenance/interface/EventID.h"
0044 #include "DataFormats/Provenance/interface/Timestamp.h"
0045 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0046 #include "DQMServices/Core/interface/DQMStore.h"
0047 #include "HLTrigger/Timer/interface/ProcessCallGraph.h"
0048
0049 /*
0050 procesing time is divided into
0051  - source
0052  - event processing, sum of the time spent in all the modules
0053 */
0054
0055 class FastTimerService : public tbb::task_scheduler_observer {
0056 public:
0057   FastTimerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0058   ~FastTimerService() override = default;
0059
0060 private:
0061   void ignoredSignal(const std::string& signal) const;
0062   void unsupportedSignal(const std::string& signal) const;
0063
0064   // these signal pairs are not guaranteed to happen in the same thread
0065
0066   void preallocate(edm::service::SystemBounds const&);
0067
0068   void lookupInitializationComplete(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0069
0070   void postEndJob();
0071
0072   void preGlobalBeginRun(edm::GlobalContext const&);
0073   void postGlobalBeginRun(edm::GlobalContext const&);
0074
0075   void preGlobalEndRun(edm::GlobalContext const&);
0076   void postGlobalEndRun(edm::GlobalContext const&);
0077
0078   void preStreamBeginRun(edm::StreamContext const&);
0079   void postStreamBeginRun(edm::StreamContext const&);
0080
0081   void preStreamEndRun(edm::StreamContext const&);
0082   void postStreamEndRun(edm::StreamContext const&);
0083
0084   void preGlobalBeginLumi(edm::GlobalContext const&);
0085   void postGlobalBeginLumi(edm::GlobalContext const&);
0086
0087   void preGlobalEndLumi(edm::GlobalContext const&);
0088   void postGlobalEndLumi(edm::GlobalContext const&);
0089
0090   void preStreamBeginLumi(edm::StreamContext const&);
0091   void postStreamBeginLumi(edm::StreamContext const&);
0092
0093   void preStreamEndLumi(edm::StreamContext const&);
0094   void postStreamEndLumi(edm::StreamContext const&);
0095
0096   void preEvent(edm::StreamContext const&);
0097   void postEvent(edm::StreamContext const&);
0098
0099   void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0100   void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0101
0102   void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0103   void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0104
0105   // these signal pairs are guaranteed to be called from the same thread
0106
0107   //void preOpenFile(std::string const&, bool);
0108   //void postOpenFile(std::string const&, bool);
0109
0110   //void preCloseFile(std::string const&, bool);
0111   //void postCloseFile(std::string const&, bool);
0112
0113   void preSourceConstruction(edm::ModuleDescription const&);
0114   //void postSourceConstruction(edm::ModuleDescription const&);
0115
0116   void preSourceRun(edm::RunIndex);
0117   void postSourceRun(edm::RunIndex);
0118
0119   void preSourceLumi(edm::LuminosityBlockIndex);
0120   void postSourceLumi(edm::LuminosityBlockIndex);
0121
0122   void preSourceEvent(edm::StreamID);
0123   void postSourceEvent(edm::StreamID);
0124
0125   //void preModuleConstruction(edm::ModuleDescription const&);
0126   //void postModuleConstruction(edm::ModuleDescription const&);
0127
0128   //void preModuleBeginJob(edm::ModuleDescription const&);
0129   //void postModuleBeginJob(edm::ModuleDescription const&);
0130
0131   //void preModuleEndJob(edm::ModuleDescription const&);
0132   //void postModuleEndJob(edm::ModuleDescription const&);
0133
0134   //void preModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0135   //void postModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0136
0137   //void preModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0138   //void postModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0139
0140   void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0141   void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0142
0143   void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0144   void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0145
0146   void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0147   void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0148
0149   void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0150   void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0151
0152   void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0153   void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0154
0155   void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0156   void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0157
0158   void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0159   void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0160
0161   void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0162   void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0163
0164   void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0165   void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0166
0167   void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0168   void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0169
0170   void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0171   void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0172
0173   void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0174   void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0175
0176   void preESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0177   void postESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0178
0179   // inherited from TBB task_scheduler_observer
0180   void on_scheduler_entry(bool worker) final;
0181   void on_scheduler_exit(bool worker) final;
0182
0183 public:
0184   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0185   static void fixForDQM(std::string& label);
0186
0187 private:
0188   // forward declarations
0189   struct Resources;
0190   struct AtomicResources;
0191
0192   // per-thread measurements
0193   struct Measurement {
0194   public:
0195     Measurement() noexcept;
0196     // take per-thread measurements
0197     void measure() noexcept;
0198     // take per-thread measurements, compute the delta with respect to the previous measurement, and store them in the argument
0199     void measure_and_store(Resources& store) noexcept;
0200     // take per-thread measurements, compute the delta with respect to the previous measurement, and add them to the argument
0201     void measure_and_accumulate(Resources& store) noexcept;
0202     void measure_and_accumulate(AtomicResources& store) noexcept;
0203
0204   public:
0205 #ifdef DEBUG_THREAD_CONCURRENCY
0206     std::thread::id id;
0207 #endif  // DEBUG_THREAD_CONCURRENCY
0208     boost::chrono::thread_clock::time_point time_thread;
0209     boost::chrono::high_resolution_clock::time_point time_real;
0210     uint64_t allocated;
0211     uint64_t deallocated;
0212   };
0213
0214   // highlight a group of modules
0215   struct GroupOfModules {
0216   public:
0217     std::string label;
0218     std::vector<unsigned int> modules;
0219   };
0220
0221   // resources being monitored by the service
0222   struct Resources {
0223   public:
0224     Resources();
0225     void reset();
0226
0227     Resources& operator+=(Resources const& other);
0228     Resources& operator+=(struct AtomicResources const& other);
0229     Resources operator+(Resources const& other) const;
0230     Resources operator+(struct AtomicResources const& other) const;
0231
0232   public:
0233     boost::chrono::nanoseconds time_thread;
0234     boost::chrono::nanoseconds time_real;
0235     uint64_t allocated;
0236     uint64_t deallocated;
0237   };
0238
0239   // atomic version of Resources
0240   // Note: the structure as a whole is *not* atomic, only the individual fields are
0241   struct AtomicResources {
0242   public:
0243     AtomicResources();
0244     AtomicResources(AtomicResources const& other);
0245     void reset();
0246
0247     AtomicResources& operator=(AtomicResources const& other);
0248     AtomicResources& operator+=(AtomicResources const& other);
0249     AtomicResources& operator+=(Resources const& other);
0250     AtomicResources operator+(AtomicResources const& other) const;
0251     Resources operator+(Resources const& other) const;
0252
0253   public:
0254     std::atomic<boost::chrono::nanoseconds::rep> time_thread;
0255     std::atomic<boost::chrono::nanoseconds::rep> time_real;
0256     std::atomic<uint64_t> allocated;
0257     std::atomic<uint64_t> deallocated;
0258   };
0259
0260   // resources associated to each module, path, process and job
0261
0262   struct ResourcesPerModule {
0263   public:
0264     ResourcesPerModule() noexcept;
0265     void reset() noexcept;
0266
0267     ResourcesPerModule& operator+=(ResourcesPerModule const& other);
0268     ResourcesPerModule operator+(ResourcesPerModule const& other) const;
0269
0270   public:
0271     Resources total;
0272     unsigned events;
0273     bool has_acquire;  // whether this module has an acquire() method
0274   };
0275
0276   struct ResourcesPerPath {
0277   public:
0278     void reset();
0279     ResourcesPerPath& operator+=(ResourcesPerPath const& other);
0280     ResourcesPerPath operator+(ResourcesPerPath const& other) const;
0281
0282   public:
0283     Resources active;  // resources used by all modules on this path
0284     Resources total;   // resources used by all modules on this path, and their dependencies
0285     unsigned last;     // one-past-the last module that ran on this path
0286     bool status;       // whether the path accepted or rejected the event
0287   };
0288
0289   struct ResourcesPerProcess {
0290   public:
0291     ResourcesPerProcess(ProcessCallGraph::ProcessType const& process);
0292     void reset();
0293     ResourcesPerProcess& operator+=(ResourcesPerProcess const& other);
0294     ResourcesPerProcess operator+(ResourcesPerProcess const& other) const;
0295
0296   public:
0297     Resources total;
0298     std::vector<ResourcesPerPath> paths;
0299     std::vector<ResourcesPerPath> endpaths;
0300   };
0301
0302   struct ResourcesPerJob {
0303   public:
0304     ResourcesPerJob() = default;
0305     ResourcesPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0306     void reset();
0307     ResourcesPerJob& operator+=(ResourcesPerJob const& other);
0308     ResourcesPerJob operator+(ResourcesPerJob const& other) const;
0309
0310   public:
0311     Resources total;
0312     AtomicResources overhead;
0313     AtomicResources eventsetup;
0314     AtomicResources idle;
0315     Resources event;  // total time etc. spent between preSourceEvent and postEvent
0316     Measurement event_measurement;
0317     std::vector<Resources> highlight;
0318     std::vector<ResourcesPerModule> modules;
0319     std::vector<ResourcesPerProcess> processes;
0320     unsigned events;
0321   };
0322
0323   // plot ranges and resolution
0324   struct PlotRanges {
0325     double time_range;
0326     double time_resolution;
0327     double memory_range;
0328     double memory_resolution;
0329   };
0330
0331   // plots associated to each module or other element (path, process, etc)
0332   class PlotsPerElement {
0333   public:
0334     PlotsPerElement() = default;
0335     void book(dqm::reco::DQMStore::IBooker&,
0336               std::string const& name,
0337               std::string const& title,
0338               PlotRanges const& ranges,
0339               unsigned int lumisections,
0340               bool byls);
0341     void fill(Resources const&, unsigned int lumisection);
0342     void fill(AtomicResources const&, unsigned int lumisection);
0343     void fill_fraction(Resources const&, Resources const&, unsigned int lumisection);
0344
0345   private:
0346     // resources spent in the module
0347     dqm::reco::MonitorElement* time_thread_ = nullptr;       // TH1F
0348     dqm::reco::MonitorElement* time_thread_byls_ = nullptr;  // TProfile
0349     dqm::reco::MonitorElement* time_real_ = nullptr;         // TH1F
0350     dqm::reco::MonitorElement* time_real_byls_ = nullptr;    // TProfile
0351     dqm::reco::MonitorElement* allocated_ = nullptr;         // TH1F
0352     dqm::reco::MonitorElement* allocated_byls_ = nullptr;    // TProfile
0353     dqm::reco::MonitorElement* deallocated_ = nullptr;       // TH1F
0354     dqm::reco::MonitorElement* deallocated_byls_ = nullptr;  // TProfile
0355   };
0356
0357   // plots associated to each path or endpath
0358   class PlotsPerPath {
0359   public:
0360     PlotsPerPath() = default;
0361     void book(dqm::reco::DQMStore::IBooker&,
0362               std::string const&,
0363               ProcessCallGraph const&,
0364               ProcessCallGraph::PathType const&,
0365               PlotRanges const& ranges,
0366               unsigned int lumisections,
0367               bool byls);
0368     void fill(ProcessCallGraph::PathType const&,
0369               ResourcesPerJob const&,
0370               ResourcesPerPath const&,
0371               unsigned int lumisection);
0372
0373   private:
0374     // resources spent in all the modules in the path, including their dependencies
0375     PlotsPerElement total_;
0376
0377     // Note:
0378     //   a TH1F has 7 significant digits, while a 24-hour long run could process
0379     //   order of 10 billion events; a 64-bit long integer would work and might
0380     //   be better suited than a double, but there is no "TH1L" in ROOT.
0381
0382     // how many times each module and their dependencies has run
0383     dqm::reco::MonitorElement* module_counter_ = nullptr;  // TH1D
0384     // resources spent in each module and their dependencies
0385     dqm::reco::MonitorElement* module_time_thread_total_ = nullptr;  // TH1D
0386     dqm::reco::MonitorElement* module_time_real_total_ = nullptr;    // TH1D
0387     dqm::reco::MonitorElement* module_allocated_total_ = nullptr;    // TH1D
0388     dqm::reco::MonitorElement* module_deallocated_total_ = nullptr;  // TH1D
0389   };
0390
0391   class PlotsPerProcess {
0392   public:
0393     PlotsPerProcess(ProcessCallGraph::ProcessType const&);
0394     void book(dqm::reco::DQMStore::IBooker&,
0395               ProcessCallGraph const&,
0396               ProcessCallGraph::ProcessType const&,
0397               PlotRanges const& event_ranges,
0398               PlotRanges const& path_ranges,
0399               unsigned int lumisections,
0400               bool bypath,
0401               bool byls);
0402     void fill(ProcessCallGraph::ProcessType const&, ResourcesPerJob const&, ResourcesPerProcess const&, unsigned int ls);
0403
0404   private:
0405     // resources spent in all the modules of the (sub)process
0406     PlotsPerElement event_;
0407     // resources spent in each path and endpath
0408     std::vector<PlotsPerPath> paths_;
0409     std::vector<PlotsPerPath> endpaths_;
0410   };
0411
0412   class PlotsPerJob {
0413   public:
0414     PlotsPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0415     void book(dqm::reco::DQMStore::IBooker&,
0416               ProcessCallGraph const&,
0417               std::vector<GroupOfModules> const&,
0418               PlotRanges const& event_ranges,
0419               PlotRanges const& path_ranges,
0420               PlotRanges const& module_ranges,
0421               unsigned int lumisections,
0422               bool bymodule,
0423               bool bypath,
0424               bool byls,
0425               bool transitions);
0426     void fill(ProcessCallGraph const&, ResourcesPerJob const&, unsigned int ls);
0427     void fill_run(AtomicResources const&);
0428     void fill_lumi(AtomicResources const&, unsigned int lumisection);
0429
0430   private:
0431     // resources spent in all the modules of the job
0432     PlotsPerElement event_;
0433     PlotsPerElement event_ex_;
0434     PlotsPerElement overhead_;
0435     PlotsPerElement idle_;
0436     // resources spent in the modules' lumi and run transitions
0437     PlotsPerElement lumi_;
0438     PlotsPerElement run_;
0439     // resources spent in the highlighted modules
0440     std::vector<PlotsPerElement> highlight_;
0441     // resources spent in each module
0442     std::vector<PlotsPerElement> modules_;
0443     // resources spent in each (sub)process
0444     std::vector<PlotsPerProcess> processes_;
0445   };
0446
0447   // keep track of the dependencies among modules
0448   ProcessCallGraph callgraph_;
0449
0450   // per-stream information
0451   std::vector<ResourcesPerJob> streams_;
0452
0453   // concurrent histograms and profiles
0454   std::unique_ptr<PlotsPerJob> plots_;
0455
0456   // per-lumi and per-run information
0457   std::vector<AtomicResources> lumi_transition_;  // resources spent in the modules' global and stream lumi transitions
0458   std::vector<AtomicResources> run_transition_;   // resources spent in the modules' global and stream run transitions
0459
0460   // summary data
0461   ResourcesPerJob job_summary_;               // whole event time accounting per-job
0462   std::vector<ResourcesPerJob> run_summary_;  // whole event time accounting per-run
0463   std::mutex summary_mutex_;                  // synchronise access to the summary objects across different threads
0464
0465   //
0466   struct ThreadGuard {
0467     struct specific_t {
0468       specific_t(AtomicResources& r) : resource_(r), live_(true) {}
0469       ~specific_t() = default;
0470
0471       Measurement measurement_;
0472       AtomicResources& resource_;
0473       std::atomic<bool> live_;
0474     };
0475
0476     ThreadGuard();
0477     ~ThreadGuard() = default;
0478
0479     static void retire_thread(void* t);
0480     static std::shared_ptr<specific_t>* ptr(void* p);
0481
0482     bool register_thread(FastTimerService::AtomicResources& r);
0483     Measurement& thread();
0484     void finalize();
0485
0486     tbb::concurrent_vector<std::shared_ptr<specific_t>> thread_resources_;
0487     pthread_key_t key_;
0488   };
0489
0490   //
0491   ThreadGuard guard_;
0492
0493   // atomic variables to keep track of the completion of each step, process by process
0494   std::unique_ptr<std::atomic<unsigned int>[]> subprocess_event_check_;
0495   std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_lumi_check_;
0496   std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_run_check_;
0497
0498   // retrieve the current thread's per-thread quantities
0499   Measurement& thread();
0500
0501   // job configuration
0502   unsigned int concurrent_lumis_;
0503   unsigned int concurrent_runs_;
0504   unsigned int concurrent_streams_;
0505   unsigned int concurrent_threads_;
0506
0507   // logging configuration
0508   const bool print_event_summary_;  // print the time spent in each process, path and module after every event
0509   const bool print_run_summary_;    // print the time spent in each process, path and module for each run
0510   const bool print_job_summary_;    // print the time spent in each process, path and module for the whole job
0511
0512   // JSON configuration
0513   //const bool write_json_per_event_;
0514   //const bool write_json_per_ls_;
0515   //const bool write_json_per_run_;
0516   const bool write_json_summary_;
0517   const std::string json_filename_;
0518
0519   // dqm configuration
0520   bool enable_dqm_;  // non const, depends on the availability of the DQMStore
0521   const bool enable_dqm_bymodule_;
0522   const bool enable_dqm_bypath_;
0523   const bool enable_dqm_byls_;
0524   const bool enable_dqm_bynproc_;
0525   const bool enable_dqm_transitions_;
0526
0527   const PlotRanges dqm_event_ranges_;
0528   const PlotRanges dqm_path_ranges_;
0529   const PlotRanges dqm_module_ranges_;
0530   const unsigned int dqm_lumisections_range_;
0531   std::string dqm_path_;
0532
0533   std::vector<edm::ParameterSet> highlight_module_psets_;  // non-const, cleared in postBeginJob()
0534   std::vector<GroupOfModules> highlight_modules_;          // non-const, filled in postBeginJob()
0535
0536   // log unsupported signals
0537   mutable tbb::concurrent_unordered_set<std::string> unsupported_signals_;  // keep track of unsupported signals received
0538
0539   // print the resource usage summary for en event, a run, or the while job
0540   template <typename T>
0541   void printHeader(T& out, std::string const& label) const;
0542
0543   template <typename T>
0544   void printEventHeader(T& out, std::string const& label) const;
0545
0546   template <typename T>
0547   void printEventLine(T& out, Resources const& data, std::string const& label) const;
0548
0549   template <typename T>
0550   void printEventLine(T& out, AtomicResources const& data, std::string const& label) const;
0551
0552   template <typename T>
0553   void printEvent(T& out, ResourcesPerJob const&) const;
0554
0555   template <typename T>
0556   void printSummaryHeader(T& out, std::string const& label, bool detailed) const;
0557
0558   template <typename T>
0559   void printPathSummaryHeader(T& out, std::string const& label) const;
0560
0561   template <typename T>
0562   void printSummaryLine(T& out, Resources const& data, uint64_t events, std::string const& label) const;
0563
0564   template <typename T>
0565   void printSummaryLine(T& out, Resources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0566
0567   template <typename T>
0568   void printSummaryLine(T& out, AtomicResources const& data, uint64_t events, std::string const& label) const;
0569
0570   template <typename T>
0571   void printSummaryLine(
0572       T& out, AtomicResources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0573
0574   template <typename T>
0575   void printPathSummaryLine(
0576       T& out, Resources const& data, Resources const& total, uint64_t events, std::string const& label) const;
0577
0578   template <typename T>
0579   void printSummary(T& out, ResourcesPerJob const& data, std::string const& label) const;
0580
0581   template <typename T>
0582   void printTransition(T& out, AtomicResources const& data, std::string const& label) const;
0583
0584   template <typename T>
0585   json encodeToJSON(std::string const& type, std::string const& label, unsigned int events, T const& data) const;
0586
0587   json encodeToJSON(edm::ModuleDescription const& module, ResourcesPerModule const& data) const;
0588
0589   void writeSummaryJSON(ResourcesPerJob const& data, std::string const& filename) const;
0590
0591   // check if this is the first process being signalled
0592   bool isFirstSubprocess(edm::StreamContext const&);
0593   bool isFirstSubprocess(edm::GlobalContext const&);
0594
0595   // check if this is the lest process being signalled
0596   bool isLastSubprocess(std::atomic<unsigned int>& check);
0597 };
0598
0599 #endif  // ! FastTimerService_h