File indexing completed on 2022-05-07 00:41:25
0001 #ifndef FastTimerService_h
0002 #define FastTimerService_h
0003
0004
0005 #include <unistd.h>
0006 #include <pthread.h>
0007
0008
0009 #include <chrono>
0010 #include <cmath>
0011 #include <map>
0012 #include <mutex>
0013 #include <string>
0014 #include <unordered_map>
0015
0016
0017 #include <boost/chrono.hpp>
0018
0019
0020 #include <oneapi/tbb/concurrent_unordered_set.h>
0021 #include <oneapi/tbb/enumerable_thread_specific.h>
0022 #include <oneapi/tbb/task_scheduler_observer.h>
0023
0024
0025 #include <nlohmann/json_fwd.hpp>
0026 using json = nlohmann::json;
0027
0028
0029 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
0030 #include "FWCore/ServiceRegistry/interface/Service.h"
0031 #include "FWCore/ServiceRegistry/interface/SystemBounds.h"
0032 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0033 #include "FWCore/ServiceRegistry/interface/PathContext.h"
0034 #include "FWCore/ServiceRegistry/interface/StreamContext.h"
0035 #include "FWCore/ServiceRegistry/interface/ProcessContext.h"
0036 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0037 #include "FWCore/ServiceRegistry/interface/ESModuleCallingContext.h"
0038 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0039 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0040 #include "FWCore/Framework/interface/TriggerNamesService.h"
0041 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0042 #include "DataFormats/Common/interface/HLTPathStatus.h"
0043 #include "DataFormats/Provenance/interface/EventID.h"
0044 #include "DataFormats/Provenance/interface/Timestamp.h"
0045 #include "DataFormats/Provenance/interface/ModuleDescription.h"
0046 #include "DQMServices/Core/interface/DQMStore.h"
0047 #include "HLTrigger/Timer/interface/ProcessCallGraph.h"
0048
0049
0050
0051
0052
0053
0054
0055 class FastTimerService : public tbb::task_scheduler_observer {
0056 public:
0057 FastTimerService(const edm::ParameterSet&, edm::ActivityRegistry&);
0058 ~FastTimerService() override = default;
0059
0060 private:
0061 void ignoredSignal(const std::string& signal) const;
0062 void unsupportedSignal(const std::string& signal) const;
0063
0064
0065
0066 void preallocate(edm::service::SystemBounds const&);
0067
0068 void preBeginJob(edm::PathsAndConsumesOfModulesBase const&, edm::ProcessContext const&);
0069 void postBeginJob();
0070
0071 void postEndJob();
0072
0073 void preGlobalBeginRun(edm::GlobalContext const&);
0074 void postGlobalBeginRun(edm::GlobalContext const&);
0075
0076 void preGlobalEndRun(edm::GlobalContext const&);
0077 void postGlobalEndRun(edm::GlobalContext const&);
0078
0079 void preStreamBeginRun(edm::StreamContext const&);
0080 void postStreamBeginRun(edm::StreamContext const&);
0081
0082 void preStreamEndRun(edm::StreamContext const&);
0083 void postStreamEndRun(edm::StreamContext const&);
0084
0085 void preGlobalBeginLumi(edm::GlobalContext const&);
0086 void postGlobalBeginLumi(edm::GlobalContext const&);
0087
0088 void preGlobalEndLumi(edm::GlobalContext const&);
0089 void postGlobalEndLumi(edm::GlobalContext const&);
0090
0091 void preStreamBeginLumi(edm::StreamContext const&);
0092 void postStreamBeginLumi(edm::StreamContext const&);
0093
0094 void preStreamEndLumi(edm::StreamContext const&);
0095 void postStreamEndLumi(edm::StreamContext const&);
0096
0097 void preEvent(edm::StreamContext const&);
0098 void postEvent(edm::StreamContext const&);
0099
0100 void prePathEvent(edm::StreamContext const&, edm::PathContext const&);
0101 void postPathEvent(edm::StreamContext const&, edm::PathContext const&, edm::HLTPathStatus const&);
0102
0103 void preModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0104 void postModuleEventPrefetching(edm::StreamContext const&, edm::ModuleCallingContext const&);
0105
0106
0107
0108
0109
0110
0111
0112
0113
0114 void preSourceConstruction(edm::ModuleDescription const&);
0115
0116
0117 void preSourceRun(edm::RunIndex);
0118 void postSourceRun(edm::RunIndex);
0119
0120 void preSourceLumi(edm::LuminosityBlockIndex);
0121 void postSourceLumi(edm::LuminosityBlockIndex);
0122
0123 void preSourceEvent(edm::StreamID);
0124 void postSourceEvent(edm::StreamID);
0125
0126
0127
0128
0129
0130
0131
0132
0133
0134
0135
0136
0137
0138
0139
0140
0141 void preModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0142 void postModuleGlobalBeginRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0143
0144 void preModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0145 void postModuleGlobalEndRun(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0146
0147 void preModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0148 void postModuleGlobalBeginLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0149
0150 void preModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0151 void postModuleGlobalEndLumi(edm::GlobalContext const&, edm::ModuleCallingContext const&);
0152
0153 void preModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0154 void postModuleStreamBeginRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0155
0156 void preModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0157 void postModuleStreamEndRun(edm::StreamContext const&, edm::ModuleCallingContext const&);
0158
0159 void preModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0160 void postModuleStreamBeginLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0161
0162 void preModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0163 void postModuleStreamEndLumi(edm::StreamContext const&, edm::ModuleCallingContext const&);
0164
0165 void preModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0166 void postModuleEventAcquire(edm::StreamContext const&, edm::ModuleCallingContext const&);
0167
0168 void preModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0169 void postModuleEvent(edm::StreamContext const&, edm::ModuleCallingContext const&);
0170
0171 void preModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0172 void postModuleEventDelayedGet(edm::StreamContext const&, edm::ModuleCallingContext const&);
0173
0174 void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0175 void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
0176
0177 void preESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0178 void postESModule(edm::eventsetup::EventSetupRecordKey const&, edm::ESModuleCallingContext const&);
0179
0180
0181 void on_scheduler_entry(bool worker) final;
0182 void on_scheduler_exit(bool worker) final;
0183
0184 public:
0185 static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0186
0187 private:
0188
0189 struct Resources;
0190 struct AtomicResources;
0191
0192
0193 struct Measurement {
0194 public:
0195 Measurement() noexcept;
0196
0197 void measure() noexcept;
0198
0199 void measure_and_store(Resources& store) noexcept;
0200
0201 void measure_and_accumulate(Resources& store) noexcept;
0202 void measure_and_accumulate(AtomicResources& store) noexcept;
0203
0204 public:
0205 #ifdef DEBUG_THREAD_CONCURRENCY
0206 std::thread::id id;
0207 #endif
0208 boost::chrono::thread_clock::time_point time_thread;
0209 boost::chrono::high_resolution_clock::time_point time_real;
0210 uint64_t allocated;
0211 uint64_t deallocated;
0212 };
0213
0214
0215 struct GroupOfModules {
0216 public:
0217 std::string label;
0218 std::vector<unsigned int> modules;
0219 };
0220
0221
0222 struct Resources {
0223 public:
0224 Resources();
0225 void reset();
0226
0227 Resources& operator+=(Resources const& other);
0228 Resources& operator+=(struct AtomicResources const& other);
0229 Resources operator+(Resources const& other) const;
0230 Resources operator+(struct AtomicResources const& other) const;
0231
0232 public:
0233 boost::chrono::nanoseconds time_thread;
0234 boost::chrono::nanoseconds time_real;
0235 uint64_t allocated;
0236 uint64_t deallocated;
0237 };
0238
0239
0240
0241 struct AtomicResources {
0242 public:
0243 AtomicResources();
0244 AtomicResources(AtomicResources const& other);
0245 void reset();
0246
0247 AtomicResources& operator=(AtomicResources const& other);
0248 AtomicResources& operator+=(AtomicResources const& other);
0249 AtomicResources& operator+=(Resources const& other);
0250 AtomicResources operator+(AtomicResources const& other) const;
0251 Resources operator+(Resources const& other) const;
0252
0253 public:
0254 std::atomic<boost::chrono::nanoseconds::rep> time_thread;
0255 std::atomic<boost::chrono::nanoseconds::rep> time_real;
0256 std::atomic<uint64_t> allocated;
0257 std::atomic<uint64_t> deallocated;
0258 };
0259
0260
0261
0262 struct ResourcesPerModule {
0263 public:
0264 ResourcesPerModule() noexcept;
0265 void reset() noexcept;
0266
0267 ResourcesPerModule& operator+=(ResourcesPerModule const& other);
0268 ResourcesPerModule operator+(ResourcesPerModule const& other) const;
0269
0270 public:
0271 Resources total;
0272 unsigned events;
0273 bool has_acquire;
0274 };
0275
0276 struct ResourcesPerPath {
0277 public:
0278 void reset();
0279 ResourcesPerPath& operator+=(ResourcesPerPath const& other);
0280 ResourcesPerPath operator+(ResourcesPerPath const& other) const;
0281
0282 public:
0283 Resources active;
0284 Resources total;
0285 unsigned last;
0286 bool status;
0287 };
0288
0289 struct ResourcesPerProcess {
0290 public:
0291 ResourcesPerProcess(ProcessCallGraph::ProcessType const& process);
0292 void reset();
0293 ResourcesPerProcess& operator+=(ResourcesPerProcess const& other);
0294 ResourcesPerProcess operator+(ResourcesPerProcess const& other) const;
0295
0296 public:
0297 Resources total;
0298 std::vector<ResourcesPerPath> paths;
0299 std::vector<ResourcesPerPath> endpaths;
0300 };
0301
0302 struct ResourcesPerJob {
0303 public:
0304 ResourcesPerJob() = default;
0305 ResourcesPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0306 void reset();
0307 ResourcesPerJob& operator+=(ResourcesPerJob const& other);
0308 ResourcesPerJob operator+(ResourcesPerJob const& other) const;
0309
0310 public:
0311 Resources total;
0312 AtomicResources overhead;
0313 AtomicResources eventsetup;
0314 Resources event;
0315 Measurement event_measurement;
0316 std::vector<Resources> highlight;
0317 std::vector<ResourcesPerModule> modules;
0318 std::vector<ResourcesPerProcess> processes;
0319 unsigned events;
0320 };
0321
0322
0323 struct PlotRanges {
0324 double time_range;
0325 double time_resolution;
0326 double memory_range;
0327 double memory_resolution;
0328 };
0329
0330
0331 class PlotsPerElement {
0332 public:
0333 PlotsPerElement() = default;
0334 void book(dqm::reco::DQMStore::IBooker&,
0335 std::string const& name,
0336 std::string const& title,
0337 PlotRanges const& ranges,
0338 unsigned int lumisections,
0339 bool byls);
0340 void fill(Resources const&, unsigned int lumisection);
0341 void fill(AtomicResources const&, unsigned int lumisection);
0342 void fill_fraction(Resources const&, Resources const&, unsigned int lumisection);
0343
0344 private:
0345
0346 dqm::reco::MonitorElement* time_thread_;
0347 dqm::reco::MonitorElement* time_thread_byls_;
0348 dqm::reco::MonitorElement* time_real_;
0349 dqm::reco::MonitorElement* time_real_byls_;
0350 dqm::reco::MonitorElement* allocated_;
0351 dqm::reco::MonitorElement* allocated_byls_;
0352 dqm::reco::MonitorElement* deallocated_;
0353 dqm::reco::MonitorElement* deallocated_byls_;
0354 };
0355
0356
0357 class PlotsPerPath {
0358 public:
0359 PlotsPerPath() = default;
0360 void book(dqm::reco::DQMStore::IBooker&,
0361 std::string const&,
0362 ProcessCallGraph const&,
0363 ProcessCallGraph::PathType const&,
0364 PlotRanges const& ranges,
0365 unsigned int lumisections,
0366 bool byls);
0367 void fill(ProcessCallGraph::PathType const&,
0368 ResourcesPerJob const&,
0369 ResourcesPerPath const&,
0370 unsigned int lumisection);
0371
0372 private:
0373
0374 PlotsPerElement total_;
0375
0376
0377
0378
0379
0380
0381
0382 dqm::reco::MonitorElement* module_counter_;
0383
0384 dqm::reco::MonitorElement* module_time_thread_total_;
0385 dqm::reco::MonitorElement* module_time_real_total_;
0386 dqm::reco::MonitorElement* module_allocated_total_;
0387 dqm::reco::MonitorElement* module_deallocated_total_;
0388 };
0389
0390 class PlotsPerProcess {
0391 public:
0392 PlotsPerProcess(ProcessCallGraph::ProcessType const&);
0393 void book(dqm::reco::DQMStore::IBooker&,
0394 ProcessCallGraph const&,
0395 ProcessCallGraph::ProcessType const&,
0396 PlotRanges const& event_ranges,
0397 PlotRanges const& path_ranges,
0398 unsigned int lumisections,
0399 bool bypath,
0400 bool byls);
0401 void fill(ProcessCallGraph::ProcessType const&, ResourcesPerJob const&, ResourcesPerProcess const&, unsigned int ls);
0402
0403 private:
0404
0405 PlotsPerElement event_;
0406
0407 std::vector<PlotsPerPath> paths_;
0408 std::vector<PlotsPerPath> endpaths_;
0409 };
0410
0411 class PlotsPerJob {
0412 public:
0413 PlotsPerJob(ProcessCallGraph const& job, std::vector<GroupOfModules> const& groups);
0414 void book(dqm::reco::DQMStore::IBooker&,
0415 ProcessCallGraph const&,
0416 std::vector<GroupOfModules> const&,
0417 PlotRanges const& event_ranges,
0418 PlotRanges const& path_ranges,
0419 PlotRanges const& module_ranges,
0420 unsigned int lumisections,
0421 bool bymodule,
0422 bool bypath,
0423 bool byls,
0424 bool transitions);
0425 void fill(ProcessCallGraph const&, ResourcesPerJob const&, unsigned int ls);
0426 void fill_run(AtomicResources const&);
0427 void fill_lumi(AtomicResources const&, unsigned int lumisection);
0428
0429 private:
0430
0431 PlotsPerElement event_;
0432 PlotsPerElement event_ex_;
0433 PlotsPerElement overhead_;
0434
0435 PlotsPerElement lumi_;
0436 PlotsPerElement run_;
0437
0438 std::vector<PlotsPerElement> highlight_;
0439
0440 std::vector<PlotsPerElement> modules_;
0441
0442 std::vector<PlotsPerProcess> processes_;
0443 };
0444
0445
0446 ProcessCallGraph callgraph_;
0447
0448
0449 std::vector<ResourcesPerJob> streams_;
0450
0451
0452 std::unique_ptr<PlotsPerJob> plots_;
0453
0454
0455 std::vector<AtomicResources> lumi_transition_;
0456 std::vector<AtomicResources> run_transition_;
0457 AtomicResources overhead_;
0458
0459
0460 ResourcesPerJob job_summary_;
0461 std::vector<ResourcesPerJob> run_summary_;
0462 std::mutex summary_mutex_;
0463
0464
0465 struct ThreadGuard {
0466 struct specific_t {
0467 specific_t(AtomicResources& r) : resource_(r), live_(true) {}
0468 ~specific_t() = default;
0469
0470 Measurement measurement_;
0471 AtomicResources& resource_;
0472 std::atomic<bool> live_;
0473 };
0474
0475 ThreadGuard();
0476 ~ThreadGuard() = default;
0477
0478 static void retire_thread(void* t);
0479 static std::shared_ptr<specific_t>* ptr(void* p);
0480
0481 bool register_thread(FastTimerService::AtomicResources& r);
0482 Measurement& thread();
0483 void finalize();
0484
0485 tbb::concurrent_vector<std::shared_ptr<specific_t>> thread_resources_;
0486 pthread_key_t key_;
0487 };
0488
0489
0490 ThreadGuard guard_;
0491
0492
0493 std::unique_ptr<std::atomic<unsigned int>[]> subprocess_event_check_;
0494 std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_lumi_check_;
0495 std::unique_ptr<std::atomic<unsigned int>[]> subprocess_global_run_check_;
0496
0497
0498 Measurement& thread();
0499
0500
0501 unsigned int concurrent_lumis_;
0502 unsigned int concurrent_runs_;
0503 unsigned int concurrent_streams_;
0504 unsigned int concurrent_threads_;
0505
0506
0507 const bool print_event_summary_;
0508 const bool print_run_summary_;
0509 const bool print_job_summary_;
0510
0511
0512
0513
0514
0515 const bool write_json_summary_;
0516 const std::string json_filename_;
0517
0518
0519 bool enable_dqm_;
0520 const bool enable_dqm_bymodule_;
0521 const bool enable_dqm_bypath_;
0522 const bool enable_dqm_byls_;
0523 const bool enable_dqm_bynproc_;
0524 const bool enable_dqm_transitions_;
0525
0526 const PlotRanges dqm_event_ranges_;
0527 const PlotRanges dqm_path_ranges_;
0528 const PlotRanges dqm_module_ranges_;
0529 const unsigned int dqm_lumisections_range_;
0530 std::string dqm_path_;
0531
0532 std::vector<edm::ParameterSet> highlight_module_psets_;
0533 std::vector<GroupOfModules> highlight_modules_;
0534
0535
0536 mutable tbb::concurrent_unordered_set<std::string> unsupported_signals_;
0537
0538
0539 template <typename T>
0540 void printHeader(T& out, std::string const& label) const;
0541
0542 template <typename T>
0543 void printEventHeader(T& out, std::string const& label) const;
0544
0545 template <typename T>
0546 void printEventLine(T& out, Resources const& data, std::string const& label) const;
0547
0548 template <typename T>
0549 void printEventLine(T& out, AtomicResources const& data, std::string const& label) const;
0550
0551 template <typename T>
0552 void printEvent(T& out, ResourcesPerJob const&) const;
0553
0554 template <typename T>
0555 void printSummaryHeader(T& out, std::string const& label, bool detailed) const;
0556
0557 template <typename T>
0558 void printPathSummaryHeader(T& out, std::string const& label) const;
0559
0560 template <typename T>
0561 void printSummaryLine(T& out, Resources const& data, uint64_t events, std::string const& label) const;
0562
0563 template <typename T>
0564 void printSummaryLine(T& out, Resources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0565
0566 template <typename T>
0567 void printSummaryLine(T& out, AtomicResources const& data, uint64_t events, std::string const& label) const;
0568
0569 template <typename T>
0570 void printSummaryLine(
0571 T& out, AtomicResources const& data, uint64_t events, uint64_t active, std::string const& label) const;
0572
0573 template <typename T>
0574 void printPathSummaryLine(
0575 T& out, Resources const& data, Resources const& total, uint64_t events, std::string const& label) const;
0576
0577 template <typename T>
0578 void printSummary(T& out, ResourcesPerJob const& data, std::string const& label) const;
0579
0580 template <typename T>
0581 void printTransition(T& out, AtomicResources const& data, std::string const& label) const;
0582
0583 template <typename T>
0584 json encodeToJSON(std::string const& type, std::string const& label, unsigned int events, T const& data) const;
0585
0586 json encodeToJSON(edm::ModuleDescription const& module, ResourcesPerModule const& data) const;
0587
0588 void writeSummaryJSON(ResourcesPerJob const& data, std::string const& filename) const;
0589
0590
0591 bool isFirstSubprocess(edm::StreamContext const&);
0592 bool isFirstSubprocess(edm::GlobalContext const&);
0593
0594
0595 bool isLastSubprocess(std::atomic<unsigned int>& check);
0596 };
0597
0598 #endif