File indexing completed on 2025-06-04 22:36:17
0001 #ifndef FastTimerService_h
0002 #define FastTimerService_h
0003
0004
0005 #include <unistd.h>
0006 #include <pthread.h>
0007
0008
0009 #include <atomic>
0010 #include <chrono>
0011 #include <cmath>
0012 #include <memory>
0013 #include <mutex>
0014 #include <string>
0015 #include <vector>
0016
0017
0018 #include <boost/chrono.hpp>
0019
0020
0021 #include <oneapi/tbb/concurrent_unordered_set.h>
0022 #include <oneapi/tbb/enumerable_thread_specific.h>
0023 #include <oneapi/tbb/task_scheduler_observer.h>
0024
0025
0026 #include <nlohmann/json_fwd.hpp>
0027 using json = nlohmann::json;
0028
0029
0030 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0031 #include "FWCore/ServiceRegistry/interface/Service.h"
0032 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0033 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0034 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0035 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0036 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0037 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0038 #include "FWCore/ServiceRegistry/interface/ESModuleCallingContext.h"
0039 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0040 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0041 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0042 #include "DataFormats/Common/interface/HLTPathStatus.h"
0043 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0044 #include "DQMServices/Core/interface/DQMStore.h"
0045 #include "HLTrigger/Timer/interface/ProcessCallGraph.h"
0046
0047
0048
0049
0050
0051
0052
0053 class FastTimerService : public tbb::task_scheduler_observer {
0054 public:
0055 FastTimerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0056 ~FastTimerService() override = default;
0057
0058 private:
0059 void ignoredSignal(const std::string& signal) const;
0060 void unsupportedSignal(const std::string& signal) const;
0061
0062
0063
0064 void preallocate(edm::service::SystemBounds const&);
0065
0066 void lookupInitializationComplete(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0067
0068 void postEndJob();
0069
0070 void preGlobalBeginRun(edm::GlobalContext const&);
0071 void postGlobalBeginRun(edm::GlobalContext const&);
0072
0073 void preGlobalEndRun(edm::GlobalContext const&);
0074 void postGlobalEndRun(edm::GlobalContext const&);
0075
0076 void preStreamBeginRun(edm::StreamContext const&);
0077 void postStreamBeginRun(edm::StreamContext const&);
0078
0079 void preStreamEndRun(edm::StreamContext const&);
0080 void postStreamEndRun(edm::StreamContext const&);
0081
0082 void preGlobalBeginLumi(edm::GlobalContext const&);
0083 void postGlobalBeginLumi(edm::GlobalContext const&);
0084
0085 void preGlobalEndLumi(edm::GlobalContext const&);
0086 void postGlobalEndLumi(edm::GlobalContext const&);
0087
0088 void preStreamBeginLumi(edm::StreamContext const&);
0089 void postStreamBeginLumi(edm::StreamContext const&);
0090
0091 void preStreamEndLumi(edm::StreamContext const&);
0092 void postStreamEndLumi(edm::StreamContext const&);
0093
0094 void preEvent(edm::StreamContext const&);
0095 void postEvent(edm::StreamContext const&);
0096
0097 void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0098 void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0099
0100 void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0101 void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0102
0103
0104
0105
0106
0107
0108
0109
0110
0111 void preSourceConstruction(edm::ModuleDescription const&);
0112
0113
0114 void preSourceRun(edm::RunIndex);
0115 void postSourceRun(edm::RunIndex);
0116
0117 void preSourceLumi(edm::LuminosityBlockIndex);
0118 void postSourceLumi(edm::LuminosityBlockIndex);
0119
0120 void preSourceEvent(edm::StreamID);
0121 void postSourceEvent(edm::StreamID);
0122
0123
0124
0125
0126
0127
0128
0129
0130
0131
0132
0133
0134
0135
0136
0137
0138 void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0139 void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0140
0141 void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0142 void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0143
0144 void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0145 void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0146
0147 void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0148 void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0149
0150 void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0151 void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0152
0153 void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0154 void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0155
0156 void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0157 void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0158
0159 void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0160 void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0161
0162 void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0163 void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0164
0165 void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0166 void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0167
0168 void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0169 void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0170
0171 void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0172 void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0173
0174 void preESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0175 void postESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0176
0177
0178 void on_scheduler_entry(bool worker) final;
0179 void on_scheduler_exit(bool worker) final;
0180
0181 public:
0182 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0183 static void fixForDQM(std::string& label);
0184
0185 private:
0186
0187 struct Resources;
0188 struct AtomicResources;
0189
0190
0191 struct Measurement {
0192 public:
0193 Measurement() noexcept;
0194
0195 void measure() noexcept;
0196
0197 void measure_and_store(Resources& store) noexcept;
0198
0199 void measure_and_accumulate(Resources& store) noexcept;
0200 void measure_and_accumulate(AtomicResources& store) noexcept;
0201
0202 public:
0203 #ifdef DEBUG_THREAD_CONCURRENCY
0204 std::thread::id id;
0205 #endif
0206 boost::chrono::thread_clock::time_point time_thread;
0207 boost::chrono::high_resolution_clock::time_point time_real;
0208 uint64_t allocated;
0209 uint64_t deallocated;
0210 };
0211
0212
0213 struct GroupOfModules {
0214 public:
0215 std::string label;
0216 std::vector<unsigned int> modules;
0217 };
0218
0219
0220 struct Resources {
0221 public:
0222 Resources();
0223 void reset();
0224
0225 Resources& operator+=(Resources const& other);
0226 Resources& operator+=(struct AtomicResources const& other);
0227 Resources operator+(Resources const& other) const;
0228 Resources operator+(struct AtomicResources const& other) const;
0229
0230 public:
0231 boost::chrono::nanoseconds time_thread;
0232 boost::chrono::nanoseconds time_real;
0233 uint64_t allocated;
0234 uint64_t deallocated;
0235 };
0236
0237
0238
0239 struct AtomicResources {
0240 public:
0241 AtomicResources();
0242 AtomicResources(AtomicResources const& other);
0243 void reset();
0244
0245 AtomicResources& operator=(AtomicResources const& other);
0246 AtomicResources& operator+=(AtomicResources const& other);
0247 AtomicResources& operator+=(Resources const& other);
0248 AtomicResources operator+(AtomicResources const& other) const;
0249 Resources operator+(Resources const& other) const;
0250
0251 public:
0252 std::atomic<boost::chrono::nanoseconds::rep> time_thread;
0253 std::atomic<boost::chrono::nanoseconds::rep> time_real;
0254 std::atomic<uint64_t> allocated;
0255 std::atomic<uint64_t> deallocated;
0256 };
0257
0258
0259
0260 struct ResourcesPerModule {
0261 public:
0262 ResourcesPerModule() noexcept;
0263 void reset() noexcept;
0264
0265 ResourcesPerModule& operator+=(ResourcesPerModule const& other);
0266 ResourcesPerModule operator+(ResourcesPerModule const& other) const;
0267
0268 public:
0269 Resources total;
0270 unsigned events;
0271 bool has_acquire;
0272 };
0273
0274 struct ResourcesPerPath {
0275 public:
0276 void reset();
0277 ResourcesPerPath& operator+=(ResourcesPerPath const& other);
0278 ResourcesPerPath operator+(ResourcesPerPath const& other) const;
0279
0280 public:
0281 Resources active;
0282 Resources total;
0283 unsigned last;
0284 bool status;
0285 };
0286
0287 struct ResourcesPerProcess {
0288 public:
0289 ResourcesPerProcess() = default;
0290 ResourcesPerProcess(ProcessCallGraph::ProcessType const& process);
0291 void reset();
0292 ResourcesPerProcess& operator+=(ResourcesPerProcess const& other);
0293 ResourcesPerProcess operator+(ResourcesPerProcess const& other) const;
0294
0295 public:
0296 Resources total;
0297 std::vector<ResourcesPerPath> paths;
0298 std::vector<ResourcesPerPath> endpaths;
0299 };
0300
0301 struct ResourcesPerJob {
0302 public:
0303 ResourcesPerJob() = default;
0304 ResourcesPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0305 void reset();
0306 ResourcesPerJob& operator+=(ResourcesPerJob const& other);
0307 ResourcesPerJob operator+(ResourcesPerJob const& other) const;
0308
0309 public:
0310 Resources total;
0311 AtomicResources overhead;
0312 AtomicResources eventsetup;
0313 AtomicResources idle;
0314 Resources event;
0315 Measurement event_measurement;
0316 std::vector<Resources> highlight;
0317 std::vector<ResourcesPerModule> modules;
0318
0319
0320
0321
0322
0323
0324
0325 ResourcesPerProcess process;
0326 unsigned events;
0327 };
0328
0329
0330 struct PlotRanges {
0331 double time_range;
0332 double time_resolution;
0333 double memory_range;
0334 double memory_resolution;
0335 };
0336
0337
0338 class PlotsPerElement {
0339 public:
0340 PlotsPerElement() = default;
0341 void book(dqm::reco::DQMStore::IBooker&,
0342 std::string const& name,
0343 std::string const& title,
0344 PlotRanges const& ranges,
0345 unsigned int lumisections,
0346 bool byls);
0347 void fill(Resources const&, unsigned int lumisection);
0348 void fill(AtomicResources const&, unsigned int lumisection);
0349 void fill_fraction(Resources const&, Resources const&, unsigned int lumisection);
0350
0351 private:
0352
0353 dqm::reco::MonitorElement* time_thread_ = nullptr;
0354 dqm::reco::MonitorElement* time_thread_byls_ = nullptr;
0355 dqm::reco::MonitorElement* time_real_ = nullptr;
0356 dqm::reco::MonitorElement* time_real_byls_ = nullptr;
0357 dqm::reco::MonitorElement* allocated_ = nullptr;
0358 dqm::reco::MonitorElement* allocated_byls_ = nullptr;
0359 dqm::reco::MonitorElement* deallocated_ = nullptr;
0360 dqm::reco::MonitorElement* deallocated_byls_ = nullptr;
0361 };
0362
0363
0364 class PlotsPerPath {
0365 public:
0366 PlotsPerPath() = default;
0367 void book(dqm::reco::DQMStore::IBooker&,
0368 std::string const&,
0369 ProcessCallGraph const&,
0370 ProcessCallGraph::PathType const&,
0371 PlotRanges const& ranges,
0372 unsigned int lumisections,
0373 bool byls);
0374 void fill(ProcessCallGraph::PathType const&,
0375 ResourcesPerJob const&,
0376 ResourcesPerPath const&,
0377 unsigned int lumisection);
0378
0379 private:
0380
0381 PlotsPerElement total_;
0382
0383
0384
0385
0386
0387
0388
0389 dqm::reco::MonitorElement* module_counter_ = nullptr;
0390
0391 dqm::reco::MonitorElement* module_time_thread_total_ = nullptr;
0392 dqm::reco::MonitorElement* module_time_real_total_ = nullptr;
0393 dqm::reco::MonitorElement* module_allocated_total_ = nullptr;
0394 dqm::reco::MonitorElement* module_deallocated_total_ = nullptr;
0395 };
0396
0397 class PlotsPerProcess {
0398 public:
0399 PlotsPerProcess(ProcessCallGraph::ProcessType const&);
0400 void book(dqm::reco::DQMStore::IBooker&,
0401 ProcessCallGraph const&,
0402 ProcessCallGraph::ProcessType const&,
0403 PlotRanges const& event_ranges,
0404 PlotRanges const& path_ranges,
0405 unsigned int lumisections,
0406 bool bypath,
0407 bool byls);
0408 void fill(ProcessCallGraph::ProcessType const&, ResourcesPerJob const&, ResourcesPerProcess const&, unsigned int ls);
0409
0410 private:
0411
0412 PlotsPerElement event_;
0413
0414 std::vector<PlotsPerPath> paths_;
0415 std::vector<PlotsPerPath> endpaths_;
0416 };
0417
0418 class PlotsPerJob {
0419 public:
0420 PlotsPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0421 void book(dqm::reco::DQMStore::IBooker&,
0422 ProcessCallGraph const&,
0423 std::vector<GroupOfModules> const&,
0424 PlotRanges const& event_ranges,
0425 PlotRanges const& path_ranges,
0426 PlotRanges const& module_ranges,
0427 unsigned int lumisections,
0428 bool bymodule,
0429 bool bypath,
0430 bool byls,
0431 bool transitions);
0432 void fill(ProcessCallGraph const&, ResourcesPerJob const&, unsigned int ls);
0433 void fill_run(AtomicResources const&);
0434 void fill_lumi(AtomicResources const&, unsigned int lumisection);
0435
0436 private:
0437
0438 PlotsPerElement event_;
0439 PlotsPerElement event_ex_;
0440 PlotsPerElement overhead_;
0441 PlotsPerElement idle_;
0442
0443 PlotsPerElement lumi_;
0444 PlotsPerElement run_;
0445
0446 std::vector<PlotsPerElement> highlight_;
0447
0448 std::vector<PlotsPerElement> modules_;
0449
0450 PlotsPerProcess process_;
0451 };
0452
0453
0454 ProcessCallGraph callgraph_;
0455
0456
0457 std::vector<ResourcesPerJob> streams_;
0458
0459
0460 std::unique_ptr<PlotsPerJob> plots_;
0461
0462
0463 std::vector<AtomicResources> lumi_transition_;
0464 std::vector<AtomicResources> run_transition_;
0465
0466
0467 ResourcesPerJob job_summary_;
0468 std::vector<ResourcesPerJob> run_summary_;
0469 std::mutex summary_mutex_;
0470
0471
0472 struct ThreadGuard {
0473 struct specific_t {
0474 specific_t(AtomicResources& r) : resource_(r), live_(true) {}
0475 ~specific_t() = default;
0476
0477 Measurement measurement_;
0478 AtomicResources& resource_;
0479 std::atomic<bool> live_;
0480 };
0481
0482 ThreadGuard();
0483 ~ThreadGuard() = default;
0484
0485 static void retire_thread(void* t);
0486 static std::shared_ptr<specific_t>* ptr(void* p);
0487
0488 bool register_thread(FastTimerService::AtomicResources& r);
0489 Measurement& thread();
0490 void finalize();
0491
0492 tbb::concurrent_vector<std::shared_ptr<specific_t>> thread_resources_;
0493 pthread_key_t key_;
0494 };
0495
0496
0497 ThreadGuard guard_;
0498
0499
0500 Measurement& thread();
0501
0502
0503 unsigned int concurrent_lumis_;
0504 unsigned int concurrent_runs_;
0505 unsigned int concurrent_streams_;
0506 unsigned int concurrent_threads_;
0507
0508
0509 const bool print_event_summary_;
0510 const bool print_run_summary_;
0511 const bool print_job_summary_;
0512
0513
0514
0515
0516
0517 const bool write_json_summary_;
0518 const std::string json_filename_;
0519
0520
0521 bool enable_dqm_;
0522 const bool enable_dqm_bymodule_;
0523 const bool enable_dqm_bypath_;
0524 const bool enable_dqm_byls_;
0525 const bool enable_dqm_bynproc_;
0526 const bool enable_dqm_transitions_;
0527
0528 const PlotRanges dqm_event_ranges_;
0529 const PlotRanges dqm_path_ranges_;
0530 const PlotRanges dqm_module_ranges_;
0531 const unsigned int dqm_lumisections_range_;
0532 std::string dqm_path_;
0533
0534 std::vector<edm::ParameterSet> highlight_module_psets_;
0535 std::vector<GroupOfModules> highlight_modules_;
0536
0537
0538 mutable tbb::concurrent_unordered_set<std::string> unsupported_signals_;
0539
0540
0541 template <typename T>
0542 void printHeader(T& out, std::string const& label) const;
0543
0544 template <typename T>
0545 void printEventHeader(T& out, std::string const& label) const;
0546
0547 template <typename T>
0548 void printEventLine(T& out, Resources const& data, std::string const& label) const;
0549
0550 template <typename T>
0551 void printEventLine(T& out, AtomicResources const& data, std::string const& label) const;
0552
0553 template <typename T>
0554 void printEvent(T& out, ResourcesPerJob const&) const;
0555
0556 template <typename T>
0557 void printSummaryHeader(T& out, std::string const& label, bool detailed) const;
0558
0559 template <typename T>
0560 void printPathSummaryHeader(T& out, std::string const& label) const;
0561
0562 template <typename T>
0563 void printSummaryLine(T& out, Resources const& data, uint64_t events, std::string const& label) const;
0564
0565 template <typename T>
0566 void printSummaryLine(T& out, Resources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0567
0568 template <typename T>
0569 void printSummaryLine(T& out, AtomicResources const& data, uint64_t events, std::string const& label) const;
0570
0571 template <typename T>
0572 void printSummaryLine(
0573 T& out, AtomicResources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0574
0575 template <typename T>
0576 void printPathSummaryLine(
0577 T& out, Resources const& data, Resources const& total, uint64_t events, std::string const& label) const;
0578
0579 template <typename T>
0580 void printSummary(T& out, ResourcesPerJob const& data, std::string const& label) const;
0581
0582 template <typename T>
0583 void printTransition(T& out, AtomicResources const& data, std::string const& label) const;
0584
0585 template <typename T>
0586 json encodeToJSON(std::string const& type, std::string const& label, unsigned int events, T const& data) const;
0587
0588 json encodeToJSON(edm::ModuleDescription const& module, ResourcesPerModule const& data) const;
0589
0590 void writeSummaryJSON(ResourcesPerJob const& data, std::string const& filename) const;
0591 };
0592
0593 #endif