Timer/plugins/FastTimerService.h

0001 #ifndef FastTimerService_h
0002 #define FastTimerService_h
0003
0004 // system headers
0005 #include <unistd.h>
0006 #include <pthread.h>
0007
0008 // C++ headers
0009 #include <chrono>
0010 #include <cmath>
0011 #include <map>
0012 #include <mutex>
0013 #include <string>
0014 #include <unordered_map>
0015
0016 // boost headers
0017 #include <boost/chrono.hpp>
0018
0019 // tbb headers
0020 #include <oneapi/tbb/concurrent_unordered_set.h>
0021 #include <oneapi/tbb/enumerable_thread_specific.h>
0022 #include <oneapi/tbb/task_scheduler_observer.h>
0023
0024 // JSON headers
0025 #include <nlohmann/json_fwd.hpp>
0026 using json = nlohmann::json;
0027
0028 // CMSSW headers
0029 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0030 #include "FWCore/ServiceRegistry/interface/Service.h"
0031 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0032 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0033 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0034 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0035 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0036 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0037 #include "FWCore/ServiceRegistry/interface/ESModuleCallingContext.h"
0038 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0039 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0040 #include "FWCore/Framework/interface/TriggerNamesService.h"
0041 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0042 #include "DataFormats/Common/interface/HLTPathStatus.h"
0043 #include "DataFormats/Provenance/interface/EventID.h"
0044 #include "DataFormats/Provenance/interface/Timestamp.h"
0045 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0046 #include "DQMServices/Core/interface/DQMStore.h"
0047 #include "HLTrigger/Timer/interface/ProcessCallGraph.h"
0048
0049 /*
0050 procesing time is divided into
0051  - source
0052  - event processing, sum of the time spent in all the modules
0053 */
0054
0055 class FastTimerService : public tbb::task_scheduler_observer {
0056 public:
0057   FastTimerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0058   ~FastTimerService() override = default;
0059
0060 private:
0061   void ignoredSignal(const std::string& signal) const;
0062   void unsupportedSignal(const std::string& signal) const;
0063
0064   // these signal pairs are not guaranteed to happen in the same thread
0065
0066   void preallocate(edm::service::SystemBounds const&);
0067
0068   void preBeginJob(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0069   void postBeginJob();
0070
0071   void postEndJob();
0072
0073   void preGlobalBeginRun(edm::GlobalContext const&);
0074   void postGlobalBeginRun(edm::GlobalContext const&);
0075
0076   void preGlobalEndRun(edm::GlobalContext const&);
0077   void postGlobalEndRun(edm::GlobalContext const&);
0078
0079   void preStreamBeginRun(edm::StreamContext const&);
0080   void postStreamBeginRun(edm::StreamContext const&);
0081
0082   void preStreamEndRun(edm::StreamContext const&);
0083   void postStreamEndRun(edm::StreamContext const&);
0084
0085   void preGlobalBeginLumi(edm::GlobalContext const&);
0086   void postGlobalBeginLumi(edm::GlobalContext const&);
0087
0088   void preGlobalEndLumi(edm::GlobalContext const&);
0089   void postGlobalEndLumi(edm::GlobalContext const&);
0090
0091   void preStreamBeginLumi(edm::StreamContext const&);
0092   void postStreamBeginLumi(edm::StreamContext const&);
0093
0094   void preStreamEndLumi(edm::StreamContext const&);
0095   void postStreamEndLumi(edm::StreamContext const&);
0096
0097   void preEvent(edm::StreamContext const&);
0098   void postEvent(edm::StreamContext const&);
0099
0100   void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0101   void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0102
0103   void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0104   void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0105
0106   // these signal pairs are guaranteed to be called from the same thread
0107
0108   //void preOpenFile(std::string const&, bool);
0109   //void postOpenFile(std::string const&, bool);
0110
0111   //void preCloseFile(std::string const&, bool);
0112   //void postCloseFile(std::string const&, bool);
0113
0114   void preSourceConstruction(edm::ModuleDescription const&);
0115   //void postSourceConstruction(edm::ModuleDescription const&);
0116
0117   void preSourceRun(edm::RunIndex);
0118   void postSourceRun(edm::RunIndex);
0119
0120   void preSourceLumi(edm::LuminosityBlockIndex);
0121   void postSourceLumi(edm::LuminosityBlockIndex);
0122
0123   void preSourceEvent(edm::StreamID);
0124   void postSourceEvent(edm::StreamID);
0125
0126   //void preModuleConstruction(edm::ModuleDescription const&);
0127   //void postModuleConstruction(edm::ModuleDescription const&);
0128
0129   //void preModuleBeginJob(edm::ModuleDescription const&);
0130   //void postModuleBeginJob(edm::ModuleDescription const&);
0131
0132   //void preModuleEndJob(edm::ModuleDescription const&);
0133   //void postModuleEndJob(edm::ModuleDescription const&);
0134
0135   //void preModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0136   //void postModuleBeginStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0137
0138   //void preModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0139   //void postModuleEndStream(edm::StreamContext const&, edm::ModuleCallingContext const&);
0140
0141   void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0142   void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0143
0144   void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0145   void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0146
0147   void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0148   void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0149
0150   void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0151   void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0152
0153   void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0154   void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0155
0156   void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0157   void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0158
0159   void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0160   void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0161
0162   void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0163   void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0164
0165   void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0166   void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0167
0168   void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0169   void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0170
0171   void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0172   void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0173
0174   void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0175   void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0176
0177   void preESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0178   void postESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0179
0180   // inherited from TBB task_scheduler_observer
0181   void on_scheduler_entry(bool worker) final;
0182   void on_scheduler_exit(bool worker) final;
0183
0184 public:
0185   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0186   static void fixForDQM(std::string& label);
0187
0188 private:
0189   // forward declarations
0190   struct Resources;
0191   struct AtomicResources;
0192
0193   // per-thread measurements
0194   struct Measurement {
0195   public:
0196     Measurement() noexcept;
0197     // take per-thread measurements
0198     void measure() noexcept;
0199     // take per-thread measurements, compute the delta with respect to the previous measurement, and store them in the argument
0200     void measure_and_store(Resources& store) noexcept;
0201     // take per-thread measurements, compute the delta with respect to the previous measurement, and add them to the argument
0202     void measure_and_accumulate(Resources& store) noexcept;
0203     void measure_and_accumulate(AtomicResources& store) noexcept;
0204
0205   public:
0206 #ifdef DEBUG_THREAD_CONCURRENCY
0207     std::thread::id id;
0208 #endif  // DEBUG_THREAD_CONCURRENCY
0209     boost::chrono::thread_clock::time_point time_thread;
0210     boost::chrono::high_resolution_clock::time_point time_real;
0211     uint64_t allocated;
0212     uint64_t deallocated;
0213   };
0214
0215   // highlight a group of modules
0216   struct GroupOfModules {
0217   public:
0218     std::string label;
0219     std::vector<unsigned int> modules;
0220   };
0221
0222   // resources being monitored by the service
0223   struct Resources {
0224   public:
0225     Resources();
0226     void reset();
0227
0228     Resources& operator+=(Resources const& other);
0229     Resources& operator+=(struct AtomicResources const& other);
0230     Resources operator+(Resources const& other) const;
0231     Resources operator+(struct AtomicResources const& other) const;
0232
0233   public:
0234     boost::chrono::nanoseconds time_thread;
0235     boost::chrono::nanoseconds time_real;
0236     uint64_t allocated;
0237     uint64_t deallocated;
0238   };
0239
0240   // atomic version of Resources
0241   // Note: the structure as a whole is *not* atomic, only the individual fields are
0242   struct AtomicResources {
0243   public:
0244     AtomicResources();
0245     AtomicResources(AtomicResources const& other);
0246     void reset();
0247
0248     AtomicResources& operator=(AtomicResources const& other);
0249     AtomicResources& operator+=(AtomicResources const& other);
0250     AtomicResources& operator+=(Resources const& other);
0251     AtomicResources operator+(AtomicResources const& other) const;
0252     Resources operator+(Resources const& other) const;
0253
0254   public:
0255     std::atomic<boost::chrono::nanoseconds::rep> time_thread;
0256     std::atomic<boost::chrono::nanoseconds::rep> time_real;
0257     std::atomic<uint64_t> allocated;
0258     std::atomic<uint64_t> deallocated;
0259   };
0260
0261   // resources associated to each module, path, process and job
0262
0263   struct ResourcesPerModule {
0264   public:
0265     ResourcesPerModule() noexcept;
0266     void reset() noexcept;
0267
0268     ResourcesPerModule& operator+=(ResourcesPerModule const& other);
0269     ResourcesPerModule operator+(ResourcesPerModule const& other) const;
0270
0271   public:
0272     Resources total;
0273     unsigned events;
0274     bool has_acquire;  // whether this module has an acquire() method
0275   };
0276
0277   struct ResourcesPerPath {
0278   public:
0279     void reset();
0280     ResourcesPerPath& operator+=(ResourcesPerPath const& other);
0281     ResourcesPerPath operator+(ResourcesPerPath const& other) const;
0282
0283   public:
0284     Resources active;  // resources used by all modules on this path
0285     Resources total;   // resources used by all modules on this path, and their dependencies
0286     unsigned last;     // one-past-the last module that ran on this path
0287     bool status;       // whether the path accepted or rejected the event
0288   };
0289
0290   struct ResourcesPerProcess {
0291   public:
0292     ResourcesPerProcess(ProcessCallGraph::ProcessType const& process);
0293     void reset();
0294     ResourcesPerProcess& operator+=(ResourcesPerProcess const& other);
0295     ResourcesPerProcess operator+(ResourcesPerProcess const& other) const;
0296
0297   public:
0298     Resources total;
0299     std::vector<ResourcesPerPath> paths;
0300     std::vector<ResourcesPerPath> endpaths;
0301   };
0302
0303   struct ResourcesPerJob {
0304   public:
0305     ResourcesPerJob() = default;
0306     ResourcesPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0307     void reset();
0308     ResourcesPerJob& operator+=(ResourcesPerJob const& other);
0309     ResourcesPerJob operator+(ResourcesPerJob const& other) const;
0310
0311   public:
0312     Resources total;
0313     AtomicResources overhead;
0314     AtomicResources eventsetup;
0315     AtomicResources idle;
0316     Resources event;  // total time etc. spent between preSourceEvent and postEvent
0317     Measurement event_measurement;
0318     std::vector<Resources> highlight;
0319     std::vector<ResourcesPerModule> modules;
0320     std::vector<ResourcesPerProcess> processes;
0321     unsigned events;
0322   };
0323
0324   // plot ranges and resolution
0325   struct PlotRanges {
0326     double time_range;
0327     double time_resolution;
0328     double memory_range;
0329     double memory_resolution;
0330   };
0331
0332   // plots associated to each module or other element (path, process, etc)
0333   class PlotsPerElement {
0334   public:
0335     PlotsPerElement() = default;
0336     void book(dqm::reco::DQMStore::IBooker&,
0337               std::string const& name,
0338               std::string const& title,
0339               PlotRanges const& ranges,
0340               unsigned int lumisections,
0341               bool byls);
0342     void fill(Resources const&, unsigned int lumisection);
0343     void fill(AtomicResources const&, unsigned int lumisection);
0344     void fill_fraction(Resources const&, Resources const&, unsigned int lumisection);
0345
0346   private:
0347     // resources spent in the module
0348     dqm::reco::MonitorElement* time_thread_ = nullptr;       // TH1F
0349     dqm::reco::MonitorElement* time_thread_byls_ = nullptr;  // TProfile
0350     dqm::reco::MonitorElement* time_real_ = nullptr;         // TH1F
0351     dqm::reco::MonitorElement* time_real_byls_ = nullptr;    // TProfile
0352     dqm::reco::MonitorElement* allocated_ = nullptr;         // TH1F
0353     dqm::reco::MonitorElement* allocated_byls_ = nullptr;    // TProfile
0354     dqm::reco::MonitorElement* deallocated_ = nullptr;       // TH1F
0355     dqm::reco::MonitorElement* deallocated_byls_ = nullptr;  // TProfile
0356   };
0357
0358   // plots associated to each path or endpath
0359   class PlotsPerPath {
0360   public:
0361     PlotsPerPath() = default;
0362     void book(dqm::reco::DQMStore::IBooker&,
0363               std::string const&,
0364               ProcessCallGraph const&,
0365               ProcessCallGraph::PathType const&,
0366               PlotRanges const& ranges,
0367               unsigned int lumisections,
0368               bool byls);
0369     void fill(ProcessCallGraph::PathType const&,
0370               ResourcesPerJob const&,
0371               ResourcesPerPath const&,
0372               unsigned int lumisection);
0373
0374   private:
0375     // resources spent in all the modules in the path, including their dependencies
0376     PlotsPerElement total_;
0377
0378     // Note:
0379     //   a TH1F has 7 significant digits, while a 24-hour long run could process
0380     //   order of 10 billion events; a 64-bit long integer would work and might
0381     //   be better suited than a double, but there is no "TH1L" in ROOT.
0382
0383     // how many times each module and their dependencies has run
0384     dqm::reco::MonitorElement* module_counter_ = nullptr;  // TH1D
0385     // resources spent in each module and their dependencies
0386     dqm::reco::MonitorElement* module_time_thread_total_ = nullptr;  // TH1D
0387     dqm::reco::MonitorElement* module_time_real_total_ = nullptr;    // TH1D
0388     dqm::reco::MonitorElement* module_allocated_total_ = nullptr;    // TH1D
0389     dqm::reco::MonitorElement* module_deallocated_total_ = nullptr;  // TH1D
0390   };
0391
0392   class PlotsPerProcess {
0393   public:
0394     PlotsPerProcess(ProcessCallGraph::ProcessType const&);
0395     void book(dqm::reco::DQMStore::IBooker&,
0396               ProcessCallGraph const&,
0397               ProcessCallGraph::ProcessType const&,
0398               PlotRanges const& event_ranges,
0399               PlotRanges const& path_ranges,
0400               unsigned int lumisections,
0401               bool bypath,
0402               bool byls);
0403     void fill(ProcessCallGraph::ProcessType const&, ResourcesPerJob const&, ResourcesPerProcess const&, unsigned int ls);
0404
0405   private:
0406     // resources spent in all the modules of the (sub)process
0407     PlotsPerElement event_;
0408     // resources spent in each path and endpath
0409     std::vector<PlotsPerPath> paths_;
0410     std::vector<PlotsPerPath> endpaths_;
0411   };
0412
0413   class PlotsPerJob {
0414   public:
0415     PlotsPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0416     void book(dqm::reco::DQMStore::IBooker&,
0417               ProcessCallGraph const&,
0418               std::vector<GroupOfModules> const&,
0419               PlotRanges const& event_ranges,
0420               PlotRanges const& path_ranges,
0421               PlotRanges const& module_ranges,
0422               unsigned int lumisections,
0423               bool bymodule,
0424               bool bypath,
0425               bool byls,
0426               bool transitions);
0427     void fill(ProcessCallGraph const&, ResourcesPerJob const&, unsigned int ls);
0428     void fill_run(AtomicResources const&);
0429     void fill_lumi(AtomicResources const&, unsigned int lumisection);
0430
0431   private:
0432     // resources spent in all the modules of the job
0433     PlotsPerElement event_;
0434     PlotsPerElement event_ex_;
0435     PlotsPerElement overhead_;
0436     PlotsPerElement idle_;
0437     // resources spent in the modules' lumi and run transitions
0438     PlotsPerElement lumi_;
0439     PlotsPerElement run_;
0440     // resources spent in the highlighted modules
0441     std::vector<PlotsPerElement> highlight_;
0442     // resources spent in each module
0443     std::vector<PlotsPerElement> modules_;
0444     // resources spent in each (sub)process
0445     std::vector<PlotsPerProcess> processes_;
0446   };
0447
0448   // keep track of the dependencies among modules
0449   ProcessCallGraph callgraph_;
0450
0451   // per-stream information
0452   std::vector<ResourcesPerJob> streams_;
0453
0454   // concurrent histograms and profiles
0455   std::unique_ptr<PlotsPerJob> plots_;
0456
0457   // per-lumi and per-run information
0458   std::vector<AtomicResources> lumi_transition_;  // resources spent in the modules' global and stream lumi transitions
0459   std::vector<AtomicResources> run_transition_;   // resources spent in the modules' global and stream run transitions
0460
0461   // summary data
0462   ResourcesPerJob job_summary_;               // whole event time accounting per-job
0463   std::vector<ResourcesPerJob> run_summary_;  // whole event time accounting per-run
0464   std::mutex summary_mutex_;                  // synchronise access to the summary objects across different threads
0465
0466   //
0467   struct ThreadGuard {
0468     struct specific_t {
0469       specific_t(AtomicResources& r) : resource_(r), live_(true) {}
0470       ~specific_t() = default;
0471
0472       Measurement measurement_;
0473       AtomicResources& resource_;
0474       std::atomic<bool> live_;
0475     };
0476
0477     ThreadGuard();
0478     ~ThreadGuard() = default;
0479
0480     static void retire_thread(void* t);
0481     static std::shared_ptr<specific_t>* ptr(void* p);
0482
0483     bool register_thread(FastTimerService::AtomicResources& r);
0484     Measurement& thread();
0485     void finalize();
0486
0487     tbb::concurrent_vector<std::shared_ptr<specific_t>> thread_resources_;
0488     pthread_key_t key_;
0489   };
0490
0491   //
0492   ThreadGuard guard_;
0493
0494   // atomic variables to keep track of the completion of each step, process by process
0495   std::unique_ptr<std::atomic<unsigned int>[]> subprocess_event_check_;
0496   std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_lumi_check_;
0497   std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_run_check_;
0498
0499   // retrieve the current thread's per-thread quantities
0500   Measurement& thread();
0501
0502   // job configuration
0503   unsigned int concurrent_lumis_;
0504   unsigned int concurrent_runs_;
0505   unsigned int concurrent_streams_;
0506   unsigned int concurrent_threads_;
0507
0508   // logging configuration
0509   const bool print_event_summary_;  // print the time spent in each process, path and module after every event
0510   const bool print_run_summary_;    // print the time spent in each process, path and module for each run
0511   const bool print_job_summary_;    // print the time spent in each process, path and module for the whole job
0512
0513   // JSON configuration
0514   //const bool write_json_per_event_;
0515   //const bool write_json_per_ls_;
0516   //const bool write_json_per_run_;
0517   const bool write_json_summary_;
0518   const std::string json_filename_;
0519
0520   // dqm configuration
0521   bool enable_dqm_;  // non const, depends on the availability of the DQMStore
0522   const bool enable_dqm_bymodule_;
0523   const bool enable_dqm_bypath_;
0524   const bool enable_dqm_byls_;
0525   const bool enable_dqm_bynproc_;
0526   const bool enable_dqm_transitions_;
0527
0528   const PlotRanges dqm_event_ranges_;
0529   const PlotRanges dqm_path_ranges_;
0530   const PlotRanges dqm_module_ranges_;
0531   const unsigned int dqm_lumisections_range_;
0532   std::string dqm_path_;
0533
0534   std::vector<edm::ParameterSet> highlight_module_psets_;  // non-const, cleared in postBeginJob()
0535   std::vector<GroupOfModules> highlight_modules_;          // non-const, filled in postBeginJob()
0536
0537   // log unsupported signals
0538   mutable tbb::concurrent_unordered_set<std::string> unsupported_signals_;  // keep track of unsupported signals received
0539
0540   // print the resource usage summary for en event, a run, or the while job
0541   template <typename T>
0542   void printHeader(T& out, std::string const& label) const;
0543
0544   template <typename T>
0545   void printEventHeader(T& out, std::string const& label) const;
0546
0547   template <typename T>
0548   void printEventLine(T& out, Resources const& data, std::string const& label) const;
0549
0550   template <typename T>
0551   void printEventLine(T& out, AtomicResources const& data, std::string const& label) const;
0552
0553   template <typename T>
0554   void printEvent(T& out, ResourcesPerJob const&) const;
0555
0556   template <typename T>
0557   void printSummaryHeader(T& out, std::string const& label, bool detailed) const;
0558
0559   template <typename T>
0560   void printPathSummaryHeader(T& out, std::string const& label) const;
0561
0562   template <typename T>
0563   void printSummaryLine(T& out, Resources const& data, uint64_t events, std::string const& label) const;
0564
0565   template <typename T>
0566   void printSummaryLine(T& out, Resources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0567
0568   template <typename T>
0569   void printSummaryLine(T& out, AtomicResources const& data, uint64_t events, std::string const& label) const;
0570
0571   template <typename T>
0572   void printSummaryLine(
0573       T& out, AtomicResources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0574
0575   template <typename T>
0576   void printPathSummaryLine(
0577       T& out, Resources const& data, Resources const& total, uint64_t events, std::string const& label) const;
0578
0579   template <typename T>
0580   void printSummary(T& out, ResourcesPerJob const& data, std::string const& label) const;
0581
0582   template <typename T>
0583   void printTransition(T& out, AtomicResources const& data, std::string const& label) const;
0584
0585   template <typename T>
0586   json encodeToJSON(std::string const& type, std::string const& label, unsigned int events, T const& data) const;
0587
0588   json encodeToJSON(edm::ModuleDescription const& module, ResourcesPerModule const& data) const;
0589
0590   void writeSummaryJSON(ResourcesPerJob const& data, std::string const& filename) const;
0591
0592   // check if this is the first process being signalled
0593   bool isFirstSubprocess(edm::StreamContext const&);
0594   bool isFirstSubprocess(edm::GlobalContext const&);
0595
0596   // check if this is the lest process being signalled
0597   bool isLastSubprocess(std::atomic<unsigned int>& check);
0598 };
0599
0600 #endif  // ! FastTimerService_h