File indexing completed on 2025-04-13 22:50:02
0001 #ifndef FastTimerService_h
0002 #define FastTimerService_h
0003
0004
0005 #include <unistd.h>
0006 #include <pthread.h>
0007
0008
0009 #include <chrono>
0010 #include <cmath>
0011 #include <map>
0012 #include <mutex>
0013 #include <string>
0014 #include <unordered_map>
0015
0016
0017 #include <boost/chrono.hpp>
0018
0019
0020 #include <oneapi/tbb/concurrent_unordered_set.h>
0021 #include <oneapi/tbb/enumerable_thread_specific.h>
0022 #include <oneapi/tbb/task_scheduler_observer.h>
0023
0024
0025 #include <nlohmann/json_fwd.hpp>
0026 using json = nlohmann::json;
0027
0028
0029 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0030 #include "FWCore/ServiceRegistry/interface/Service.h"
0031 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0032 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0033 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0034 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0035 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0036 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0037 #include "FWCore/ServiceRegistry/interface/ESModuleCallingContext.h"
0038 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0039 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0040 #include "FWCore/Framework/interface/TriggerNamesService.h"
0041 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0042 #include "DataFormats/Common/interface/HLTPathStatus.h"
0043 #include "DataFormats/Provenance/interface/EventID.h"
0044 #include "DataFormats/Provenance/interface/Timestamp.h"
0045 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0046 #include "DQMServices/Core/interface/DQMStore.h"
0047 #include "HLTrigger/Timer/interface/ProcessCallGraph.h"
0048
0049
0050
0051
0052
0053
0054
0055 class FastTimerService : public tbb::task_scheduler_observer {
0056 public:
0057 FastTimerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0058 ~FastTimerService() override = default;
0059
0060 private:
0061 void ignoredSignal(const std::string& signal) const;
0062 void unsupportedSignal(const std::string& signal) const;
0063
0064
0065
0066 void preallocate(edm::service::SystemBounds const&);
0067
0068 void lookupInitializationComplete(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0069
0070 void postEndJob();
0071
0072 void preGlobalBeginRun(edm::GlobalContext const&);
0073 void postGlobalBeginRun(edm::GlobalContext const&);
0074
0075 void preGlobalEndRun(edm::GlobalContext const&);
0076 void postGlobalEndRun(edm::GlobalContext const&);
0077
0078 void preStreamBeginRun(edm::StreamContext const&);
0079 void postStreamBeginRun(edm::StreamContext const&);
0080
0081 void preStreamEndRun(edm::StreamContext const&);
0082 void postStreamEndRun(edm::StreamContext const&);
0083
0084 void preGlobalBeginLumi(edm::GlobalContext const&);
0085 void postGlobalBeginLumi(edm::GlobalContext const&);
0086
0087 void preGlobalEndLumi(edm::GlobalContext const&);
0088 void postGlobalEndLumi(edm::GlobalContext const&);
0089
0090 void preStreamBeginLumi(edm::StreamContext const&);
0091 void postStreamBeginLumi(edm::StreamContext const&);
0092
0093 void preStreamEndLumi(edm::StreamContext const&);
0094 void postStreamEndLumi(edm::StreamContext const&);
0095
0096 void preEvent(edm::StreamContext const&);
0097 void postEvent(edm::StreamContext const&);
0098
0099 void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0100 void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0101
0102 void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0103 void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0104
0105
0106
0107
0108
0109
0110
0111
0112
0113 void preSourceConstruction(edm::ModuleDescription const&);
0114
0115
0116 void preSourceRun(edm::RunIndex);
0117 void postSourceRun(edm::RunIndex);
0118
0119 void preSourceLumi(edm::LuminosityBlockIndex);
0120 void postSourceLumi(edm::LuminosityBlockIndex);
0121
0122 void preSourceEvent(edm::StreamID);
0123 void postSourceEvent(edm::StreamID);
0124
0125
0126
0127
0128
0129
0130
0131
0132
0133
0134
0135
0136
0137
0138
0139
0140 void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0141 void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0142
0143 void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0144 void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0145
0146 void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0147 void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0148
0149 void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0150 void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0151
0152 void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0153 void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0154
0155 void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0156 void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0157
0158 void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0159 void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0160
0161 void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0162 void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0163
0164 void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0165 void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0166
0167 void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0168 void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0169
0170 void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0171 void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0172
0173 void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0174 void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0175
0176 void preESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0177 void postESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0178
0179
0180 void on_scheduler_entry(bool worker) final;
0181 void on_scheduler_exit(bool worker) final;
0182
0183 public:
0184 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0185 static void fixForDQM(std::string& label);
0186
0187 private:
0188
0189 struct Resources;
0190 struct AtomicResources;
0191
0192
0193 struct Measurement {
0194 public:
0195 Measurement() noexcept;
0196
0197 void measure() noexcept;
0198
0199 void measure_and_store(Resources& store) noexcept;
0200
0201 void measure_and_accumulate(Resources& store) noexcept;
0202 void measure_and_accumulate(AtomicResources& store) noexcept;
0203
0204 public:
0205 #ifdef DEBUG_THREAD_CONCURRENCY
0206 std::thread::id id;
0207 #endif
0208 boost::chrono::thread_clock::time_point time_thread;
0209 boost::chrono::high_resolution_clock::time_point time_real;
0210 uint64_t allocated;
0211 uint64_t deallocated;
0212 };
0213
0214
0215 struct GroupOfModules {
0216 public:
0217 std::string label;
0218 std::vector<unsigned int> modules;
0219 };
0220
0221
0222 struct Resources {
0223 public:
0224 Resources();
0225 void reset();
0226
0227 Resources& operator+=(Resources const& other);
0228 Resources& operator+=(struct AtomicResources const& other);
0229 Resources operator+(Resources const& other) const;
0230 Resources operator+(struct AtomicResources const& other) const;
0231
0232 public:
0233 boost::chrono::nanoseconds time_thread;
0234 boost::chrono::nanoseconds time_real;
0235 uint64_t allocated;
0236 uint64_t deallocated;
0237 };
0238
0239
0240
0241 struct AtomicResources {
0242 public:
0243 AtomicResources();
0244 AtomicResources(AtomicResources const& other);
0245 void reset();
0246
0247 AtomicResources& operator=(AtomicResources const& other);
0248 AtomicResources& operator+=(AtomicResources const& other);
0249 AtomicResources& operator+=(Resources const& other);
0250 AtomicResources operator+(AtomicResources const& other) const;
0251 Resources operator+(Resources const& other) const;
0252
0253 public:
0254 std::atomic<boost::chrono::nanoseconds::rep> time_thread;
0255 std::atomic<boost::chrono::nanoseconds::rep> time_real;
0256 std::atomic<uint64_t> allocated;
0257 std::atomic<uint64_t> deallocated;
0258 };
0259
0260
0261
0262 struct ResourcesPerModule {
0263 public:
0264 ResourcesPerModule() noexcept;
0265 void reset() noexcept;
0266
0267 ResourcesPerModule& operator+=(ResourcesPerModule const& other);
0268 ResourcesPerModule operator+(ResourcesPerModule const& other) const;
0269
0270 public:
0271 Resources total;
0272 unsigned events;
0273 bool has_acquire;
0274 };
0275
0276 struct ResourcesPerPath {
0277 public:
0278 void reset();
0279 ResourcesPerPath& operator+=(ResourcesPerPath const& other);
0280 ResourcesPerPath operator+(ResourcesPerPath const& other) const;
0281
0282 public:
0283 Resources active;
0284 Resources total;
0285 unsigned last;
0286 bool status;
0287 };
0288
0289 struct ResourcesPerProcess {
0290 public:
0291 ResourcesPerProcess(ProcessCallGraph::ProcessType const& process);
0292 void reset();
0293 ResourcesPerProcess& operator+=(ResourcesPerProcess const& other);
0294 ResourcesPerProcess operator+(ResourcesPerProcess const& other) const;
0295
0296 public:
0297 Resources total;
0298 std::vector<ResourcesPerPath> paths;
0299 std::vector<ResourcesPerPath> endpaths;
0300 };
0301
0302 struct ResourcesPerJob {
0303 public:
0304 ResourcesPerJob() = default;
0305 ResourcesPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0306 void reset();
0307 ResourcesPerJob& operator+=(ResourcesPerJob const& other);
0308 ResourcesPerJob operator+(ResourcesPerJob const& other) const;
0309
0310 public:
0311 Resources total;
0312 AtomicResources overhead;
0313 AtomicResources eventsetup;
0314 AtomicResources idle;
0315 Resources event;
0316 Measurement event_measurement;
0317 std::vector<Resources> highlight;
0318 std::vector<ResourcesPerModule> modules;
0319 std::vector<ResourcesPerProcess> processes;
0320 unsigned events;
0321 };
0322
0323
0324 struct PlotRanges {
0325 double time_range;
0326 double time_resolution;
0327 double memory_range;
0328 double memory_resolution;
0329 };
0330
0331
0332 class PlotsPerElement {
0333 public:
0334 PlotsPerElement() = default;
0335 void book(dqm::reco::DQMStore::IBooker&,
0336 std::string const& name,
0337 std::string const& title,
0338 PlotRanges const& ranges,
0339 unsigned int lumisections,
0340 bool byls);
0341 void fill(Resources const&, unsigned int lumisection);
0342 void fill(AtomicResources const&, unsigned int lumisection);
0343 void fill_fraction(Resources const&, Resources const&, unsigned int lumisection);
0344
0345 private:
0346
0347 dqm::reco::MonitorElement* time_thread_ = nullptr;
0348 dqm::reco::MonitorElement* time_thread_byls_ = nullptr;
0349 dqm::reco::MonitorElement* time_real_ = nullptr;
0350 dqm::reco::MonitorElement* time_real_byls_ = nullptr;
0351 dqm::reco::MonitorElement* allocated_ = nullptr;
0352 dqm::reco::MonitorElement* allocated_byls_ = nullptr;
0353 dqm::reco::MonitorElement* deallocated_ = nullptr;
0354 dqm::reco::MonitorElement* deallocated_byls_ = nullptr;
0355 };
0356
0357
0358 class PlotsPerPath {
0359 public:
0360 PlotsPerPath() = default;
0361 void book(dqm::reco::DQMStore::IBooker&,
0362 std::string const&,
0363 ProcessCallGraph const&,
0364 ProcessCallGraph::PathType const&,
0365 PlotRanges const& ranges,
0366 unsigned int lumisections,
0367 bool byls);
0368 void fill(ProcessCallGraph::PathType const&,
0369 ResourcesPerJob const&,
0370 ResourcesPerPath const&,
0371 unsigned int lumisection);
0372
0373 private:
0374
0375 PlotsPerElement total_;
0376
0377
0378
0379
0380
0381
0382
0383 dqm::reco::MonitorElement* module_counter_ = nullptr;
0384
0385 dqm::reco::MonitorElement* module_time_thread_total_ = nullptr;
0386 dqm::reco::MonitorElement* module_time_real_total_ = nullptr;
0387 dqm::reco::MonitorElement* module_allocated_total_ = nullptr;
0388 dqm::reco::MonitorElement* module_deallocated_total_ = nullptr;
0389 };
0390
0391 class PlotsPerProcess {
0392 public:
0393 PlotsPerProcess(ProcessCallGraph::ProcessType const&);
0394 void book(dqm::reco::DQMStore::IBooker&,
0395 ProcessCallGraph const&,
0396 ProcessCallGraph::ProcessType const&,
0397 PlotRanges const& event_ranges,
0398 PlotRanges const& path_ranges,
0399 unsigned int lumisections,
0400 bool bypath,
0401 bool byls);
0402 void fill(ProcessCallGraph::ProcessType const&, ResourcesPerJob const&, ResourcesPerProcess const&, unsigned int ls);
0403
0404 private:
0405
0406 PlotsPerElement event_;
0407
0408 std::vector<PlotsPerPath> paths_;
0409 std::vector<PlotsPerPath> endpaths_;
0410 };
0411
0412 class PlotsPerJob {
0413 public:
0414 PlotsPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0415 void book(dqm::reco::DQMStore::IBooker&,
0416 ProcessCallGraph const&,
0417 std::vector<GroupOfModules> const&,
0418 PlotRanges const& event_ranges,
0419 PlotRanges const& path_ranges,
0420 PlotRanges const& module_ranges,
0421 unsigned int lumisections,
0422 bool bymodule,
0423 bool bypath,
0424 bool byls,
0425 bool transitions);
0426 void fill(ProcessCallGraph const&, ResourcesPerJob const&, unsigned int ls);
0427 void fill_run(AtomicResources const&);
0428 void fill_lumi(AtomicResources const&, unsigned int lumisection);
0429
0430 private:
0431
0432 PlotsPerElement event_;
0433 PlotsPerElement event_ex_;
0434 PlotsPerElement overhead_;
0435 PlotsPerElement idle_;
0436
0437 PlotsPerElement lumi_;
0438 PlotsPerElement run_;
0439
0440 std::vector<PlotsPerElement> highlight_;
0441
0442 std::vector<PlotsPerElement> modules_;
0443
0444 std::vector<PlotsPerProcess> processes_;
0445 };
0446
0447
0448 ProcessCallGraph callgraph_;
0449
0450
0451 std::vector<ResourcesPerJob> streams_;
0452
0453
0454 std::unique_ptr<PlotsPerJob> plots_;
0455
0456
0457 std::vector<AtomicResources> lumi_transition_;
0458 std::vector<AtomicResources> run_transition_;
0459
0460
0461 ResourcesPerJob job_summary_;
0462 std::vector<ResourcesPerJob> run_summary_;
0463 std::mutex summary_mutex_;
0464
0465
0466 struct ThreadGuard {
0467 struct specific_t {
0468 specific_t(AtomicResources& r) : resource_(r), live_(true) {}
0469 ~specific_t() = default;
0470
0471 Measurement measurement_;
0472 AtomicResources& resource_;
0473 std::atomic<bool> live_;
0474 };
0475
0476 ThreadGuard();
0477 ~ThreadGuard() = default;
0478
0479 static void retire_thread(void* t);
0480 static std::shared_ptr<specific_t>* ptr(void* p);
0481
0482 bool register_thread(FastTimerService::AtomicResources& r);
0483 Measurement& thread();
0484 void finalize();
0485
0486 tbb::concurrent_vector<std::shared_ptr<specific_t>> thread_resources_;
0487 pthread_key_t key_;
0488 };
0489
0490
0491 ThreadGuard guard_;
0492
0493
0494 std::unique_ptr<std::atomic<unsigned int>[]> subprocess_event_check_;
0495 std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_lumi_check_;
0496 std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_run_check_;
0497
0498
0499 Measurement& thread();
0500
0501
0502 unsigned int concurrent_lumis_;
0503 unsigned int concurrent_runs_;
0504 unsigned int concurrent_streams_;
0505 unsigned int concurrent_threads_;
0506
0507
0508 const bool print_event_summary_;
0509 const bool print_run_summary_;
0510 const bool print_job_summary_;
0511
0512
0513
0514
0515
0516 const bool write_json_summary_;
0517 const std::string json_filename_;
0518
0519
0520 bool enable_dqm_;
0521 const bool enable_dqm_bymodule_;
0522 const bool enable_dqm_bypath_;
0523 const bool enable_dqm_byls_;
0524 const bool enable_dqm_bynproc_;
0525 const bool enable_dqm_transitions_;
0526
0527 const PlotRanges dqm_event_ranges_;
0528 const PlotRanges dqm_path_ranges_;
0529 const PlotRanges dqm_module_ranges_;
0530 const unsigned int dqm_lumisections_range_;
0531 std::string dqm_path_;
0532
0533 std::vector<edm::ParameterSet> highlight_module_psets_;
0534 std::vector<GroupOfModules> highlight_modules_;
0535
0536
0537 mutable tbb::concurrent_unordered_set<std::string> unsupported_signals_;
0538
0539
0540 template <typename T>
0541 void printHeader(T& out, std::string const& label) const;
0542
0543 template <typename T>
0544 void printEventHeader(T& out, std::string const& label) const;
0545
0546 template <typename T>
0547 void printEventLine(T& out, Resources const& data, std::string const& label) const;
0548
0549 template <typename T>
0550 void printEventLine(T& out, AtomicResources const& data, std::string const& label) const;
0551
0552 template <typename T>
0553 void printEvent(T& out, ResourcesPerJob const&) const;
0554
0555 template <typename T>
0556 void printSummaryHeader(T& out, std::string const& label, bool detailed) const;
0557
0558 template <typename T>
0559 void printPathSummaryHeader(T& out, std::string const& label) const;
0560
0561 template <typename T>
0562 void printSummaryLine(T& out, Resources const& data, uint64_t events, std::string const& label) const;
0563
0564 template <typename T>
0565 void printSummaryLine(T& out, Resources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0566
0567 template <typename T>
0568 void printSummaryLine(T& out, AtomicResources const& data, uint64_t events, std::string const& label) const;
0569
0570 template <typename T>
0571 void printSummaryLine(
0572 T& out, AtomicResources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0573
0574 template <typename T>
0575 void printPathSummaryLine(
0576 T& out, Resources const& data, Resources const& total, uint64_t events, std::string const& label) const;
0577
0578 template <typename T>
0579 void printSummary(T& out, ResourcesPerJob const& data, std::string const& label) const;
0580
0581 template <typename T>
0582 void printTransition(T& out, AtomicResources const& data, std::string const& label) const;
0583
0584 template <typename T>
0585 json encodeToJSON(std::string const& type, std::string const& label, unsigned int events, T const& data) const;
0586
0587 json encodeToJSON(edm::ModuleDescription const& module, ResourcesPerModule const& data) const;
0588
0589 void writeSummaryJSON(ResourcesPerJob const& data, std::string const& filename) const;
0590
0591
0592 bool isFirstSubprocess(edm::StreamContext const&);
0593 bool isFirstSubprocess(edm::GlobalContext const&);
0594
0595
0596 bool isLastSubprocess(std::atomic<unsigned int>& check);
0597 };
0598
0599 #endif