File indexing completed on 2025-03-13 02:31:45
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 #include <string>
0019 #include <sstream>
0020 #include <utility>
0021 #include <vector>
0022 #include <iostream>
0023 #include <iomanip>
0024 #include <utility>
0025 #include <fstream>
0026 #include <sstream>
0027
0028 #include "TFile.h"
0029 #include "TTree.h"
0030
0031 #include "DQMServices/Core/interface/DQMStore.h"
0032 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
0033 #include "FWCore/MessageLogger/interface/JobReport.h"
0034 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0035 #include "FWCore/ServiceRegistry/interface/Service.h"
0036
0037
0038
0039
0040
0041 using dqm::legacy::DQMStore;
0042 using dqm::legacy::MonitorElement;
0043
0044
0045
0046
0047
0048 class DQMStoreStatsSubfolder {
0049 public:
0050 DQMStoreStatsSubfolder() {
0051 totalHistos_ = 0;
0052 totalBins_ = 0;
0053 totalMemory_ = 0;
0054 totalEmptyBins_ = 0;
0055 }
0056 std::string subfolderName_;
0057 unsigned int totalHistos_;
0058 unsigned int totalBins_;
0059 unsigned int totalEmptyBins_;
0060 unsigned int totalMemory_;
0061 void AddBinsF(unsigned int nBins, unsigned int nEmptyBins) {
0062 ++totalHistos_;
0063 totalBins_ += nBins;
0064 totalEmptyBins_ += nEmptyBins;
0065 totalMemory_ += (nBins * sizeof(float));
0066 }
0067 void AddBinsS(unsigned int nBins, unsigned int nEmptyBins) {
0068 ++totalHistos_;
0069 totalBins_ += nBins;
0070 totalEmptyBins_ += nEmptyBins;
0071 totalMemory_ += (nBins * sizeof(short));
0072 }
0073 void AddBinsD(unsigned int nBins, unsigned int nEmptyBins) {
0074 ++totalHistos_;
0075 totalBins_ += nBins;
0076 totalEmptyBins_ += nEmptyBins;
0077 totalMemory_ += (nBins * sizeof(double));
0078 }
0079 void AddBinsI(unsigned int nBins, unsigned int nEmptyBins) {
0080 ++totalHistos_;
0081 totalBins_ += nBins;
0082 totalEmptyBins_ += nEmptyBins;
0083 totalMemory_ += (nBins * sizeof(int));
0084 }
0085 };
0086
0087
0088
0089
0090
0091 class DQMStoreStatsSubsystem : public std::vector<DQMStoreStatsSubfolder> {
0092 public:
0093 DQMStoreStatsSubsystem() = default;
0094 std::string subsystemName_;
0095 };
0096
0097
0098
0099
0100
0101 class DQMStoreStatsTopLevel : public std::vector<DQMStoreStatsSubsystem> {
0102 public:
0103 DQMStoreStatsTopLevel() = default;
0104 };
0105
0106 template <class Item>
0107 class Iterator {
0108 public:
0109 virtual ~Iterator() = default;
0110 virtual void First() = 0;
0111 virtual void Next() = 0;
0112 virtual bool IsDone() const = 0;
0113 virtual Item CurrentItem() const = 0;
0114
0115 protected:
0116 Iterator() = default;
0117 };
0118
0119 template <class Item>
0120 class VIterator : public Iterator<Item> {
0121 public:
0122 VIterator(const std::vector<Item>* aVector) : vector_(aVector), index(0) {}
0123 ~VIterator() override = default;
0124 void First() override { index = 0; }
0125 void Next() override { ++index; }
0126 virtual int size() { return vector_->size(); }
0127 virtual int getIndex() { return (int)index; }
0128
0129 bool IsDone() const override {
0130 if (index < (unsigned int)vector_->size())
0131 return false;
0132 return true;
0133 }
0134
0135 Item CurrentItem() const override { return vector_->operator[](index); }
0136
0137 private:
0138 const std::vector<Item>* vector_;
0139 unsigned int index;
0140 };
0141
0142 static unsigned int getId() {
0143 static unsigned int id = 10;
0144 return ++id;
0145 }
0146
0147 class Folder {
0148 public:
0149 Folder(std::string name)
0150 : totalHistos_(0),
0151 totalBins_(0),
0152 totalEmptyBins_(0),
0153 totalMemory_(0),
0154 id_(10),
0155 level_(0),
0156 folderName_(std::move(name)),
0157 father_(nullptr) {}
0158
0159 ~Folder() {
0160 for (auto& subfolder : subfolders_)
0161 delete subfolder;
0162 }
0163
0164 void setFather(Folder* e) { father_ = e; }
0165 Folder* getFather() { return father_; }
0166 const std::string& name() { return folderName_; }
0167
0168 Folder* cd(const std::string& name) {
0169 for (auto& subfolder : subfolders_)
0170 if (subfolder->name() == name)
0171 return subfolder;
0172 auto* tmp = new Folder(name);
0173 this->add(tmp);
0174 return tmp;
0175 }
0176
0177 void setId(unsigned int id) { id_ = id; }
0178 unsigned int id() { return id_; }
0179 void setLevel(unsigned int value) { level_ = value; }
0180 unsigned int level() { return level_; }
0181
0182 void add(Folder* f) {
0183 f->setFather(this);
0184 subfolders_.push_back(f);
0185 f->setLevel(level_ + 1);
0186 f->setId(getId());
0187 }
0188
0189 unsigned int getHistos() {
0190 unsigned int result = totalHistos_;
0191 for (auto& subfolder : subfolders_)
0192 result += subfolder->getHistos();
0193 return result;
0194 }
0195 unsigned int getBins() {
0196 unsigned int result = totalBins_;
0197 for (auto& subfolder : subfolders_)
0198 result += subfolder->getBins();
0199 return result;
0200 }
0201 unsigned int getEmptyBins() {
0202 unsigned int result = totalEmptyBins_;
0203 for (auto& subfolder : subfolders_)
0204 result += subfolder->getEmptyBins();
0205 return result;
0206 }
0207 unsigned int getMemory() {
0208 unsigned int result = totalMemory_;
0209 for (auto& subfolder : subfolders_)
0210 result += subfolder->getMemory();
0211 return result;
0212 }
0213 void update(unsigned int bins, unsigned int empty, unsigned int memory) {
0214 totalHistos_ += 1;
0215 totalBins_ += bins;
0216 totalEmptyBins_ += empty;
0217 totalMemory_ += memory;
0218 }
0219 void dump(std::string indent) {
0220 indent.append(" ");
0221 std::cout << indent << "I'm a " << name() << " whose father is " << getFather() << " with ID: " << id_
0222 << " Histo: " << getHistos() << " Bins: " << getBins() << " EmptyBins: " << getEmptyBins()
0223 << " Memory: " << getMemory() << " and my children are: " << std::endl;
0224 for (auto& subfolder : subfolders_)
0225 subfolder->dump(indent);
0226 }
0227 VIterator<Folder*> CreateIterator() { return VIterator<Folder*>(&subfolders_); }
0228
0229 void mainrows(std::string& sql_statement) {
0230 std::stringstream s("");
0231 s << "INSERT INTO mainrows(id, symbol_id, self_count, cumulative_count, kids, self_calls, total_calls, self_paths, "
0232 "total_paths, pct)"
0233 " VALUES("
0234 << id_ << ", " << id_ << ", " << getMemory() << ", " << getMemory() << ", " << subfolders_.size() << ", "
0235 << getBins() - getEmptyBins() << ", " << getBins() << ", " << getHistos() << ", " << getHistos() << ", 0.0);\n";
0236 sql_statement.append(s.str());
0237 for (auto& subfolder : subfolders_)
0238 subfolder->mainrows(sql_statement);
0239 }
0240
0241 void symbols(std::string& sql_statement) {
0242 unsigned int parentid = this->getFather() ? this->getFather()->id() : id_;
0243 std::stringstream s("");
0244 s << "INSERT INTO symbols(id, name, filename_id) VALUES (" << id_ << ",\"" << folderName_ << "\", " << parentid
0245 << ");\n";
0246 sql_statement.append(s.str());
0247 for (auto& subfolder : subfolders_)
0248 subfolder->symbols(sql_statement);
0249 }
0250
0251 void parents(std::string& sql_statement) {
0252 unsigned int parentid = this->getFather() ? this->getFather()->id() : id_;
0253 std::stringstream s("");
0254 s << "INSERT INTO parents(self_id, child_id, to_child_count, to_child_calls, to_child_paths, pct) VALUES("
0255 << parentid << "," << id_ << "," << totalMemory_ << "," << totalBins_ << "," << totalHistos_ << ",0"
0256 << ");\n";
0257 sql_statement.append(s.str());
0258 for (auto& subfolder : subfolders_)
0259 subfolder->parents(sql_statement);
0260 }
0261
0262 void children(std::string& sql_statement) {
0263 unsigned int parentid = this->getFather() ? this->getFather()->id() : id_;
0264 std::stringstream s("");
0265 s << "INSERT INTO children(self_id, parent_id, from_parent_count, from_parent_calls, from_parent_paths, pct) "
0266 "VALUES("
0267 << id_ << "," << parentid << "," << getMemory() << "," << getBins() - getEmptyBins() << "," << totalHistos_
0268 << ",0"
0269 << ");\n";
0270 sql_statement.append(s.str());
0271 for (auto& subfolder : subfolders_)
0272 subfolder->children(sql_statement);
0273 }
0274
0275 void mainrows_cumulative(std::string& sql_statement) {
0276 std::stringstream s("");
0277 s << "INSERT INTO mainrows(id, symbol_id, self_count, cumulative_count, kids, self_calls, total_calls, self_paths, "
0278 "total_paths, pct)"
0279 << " VALUES(" << id_ << "," << id_ << "," << 0 << "," << getMemory() << ", 0," << getBins() - getEmptyBins()
0280 << "," << getBins() << ", 0, " << getHistos() << ", 0);\n";
0281 sql_statement.append(s.str());
0282 }
0283
0284 void summary(std::string& sql_statement) {
0285 std::stringstream s("");
0286 s << "INSERT INTO summary(counter, total_count, total_freq, tick_period) VALUES (\"BINS_LIVE\"," << getMemory()
0287 << "," << getBins() << ", 1);\n";
0288 sql_statement.append(s.str());
0289 }
0290
0291 void files(std::string& sql_statement) {
0292 std::stringstream s("");
0293 s << "INSERT INTO files(id, name) VALUES(" << id_ << ",\"" << folderName_ << "\");\n";
0294 sql_statement.append(s.str());
0295 }
0296
0297 private:
0298 unsigned int totalHistos_;
0299 unsigned int totalBins_;
0300 unsigned int totalEmptyBins_;
0301 unsigned int totalMemory_;
0302 unsigned int id_;
0303 unsigned int level_;
0304 std::string folderName_;
0305 Folder* father_;
0306 std::vector<Folder*> subfolders_;
0307 };
0308
0309
0310
0311
0312 class DQMStoreStats : public edm::one::EDAnalyzer<edm::one::WatchRuns, edm::one::WatchLuminosityBlocks> {
0313 public:
0314 DQMStoreStats(const edm::ParameterSet&);
0315 ~DQMStoreStats() override = default;
0316
0317 enum statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 };
0318
0319 protected:
0320
0321 void beginJob() override;
0322
0323
0324 void beginRun(const edm::Run& r, const edm::EventSetup& c) override;
0325
0326
0327 void analyze(const edm::Event& e, const edm::EventSetup& c) override;
0328
0329
0330 void beginLuminosityBlock(const edm::LuminosityBlock& lumiSeg, const edm::EventSetup& c) override;
0331 void endLuminosityBlock(const edm::LuminosityBlock& lumiSeg, const edm::EventSetup& c) override;
0332
0333
0334 void endRun(const edm::Run& r, const edm::EventSetup& c) override;
0335
0336
0337 void endJob() override;
0338
0339 private:
0340 int calcstats(int);
0341 std::vector<double> GetTH2PolyArray(TH2Poly* poly);
0342 void calcIgProfDump(Folder&);
0343 void dumpMemoryProfile();
0344 std::pair<unsigned int, unsigned int> readMemoryEntry() const;
0345 void print();
0346
0347 DQMStore* dbe_;
0348 edm::ParameterSet parameters_;
0349
0350 std::string subsystem_;
0351 std::string subfolder_;
0352 int nbinsglobal_;
0353 int nbinssubsys_;
0354 int nmeglobal_;
0355 int nmesubsys_;
0356 int maxbinsglobal_;
0357 int maxbinssubsys_;
0358 std::string maxbinsmeglobal_;
0359 std::string maxbinsmesubsys_;
0360
0361 int statsdepth_;
0362 std::string pathnamematch_;
0363 int verbose_;
0364
0365 std::vector<std::pair<time_t, unsigned int> > memoryHistoryVector_;
0366 time_t startingTime_;
0367 bool isOpenProcFileSuccessful_;
0368 std::stringstream procFileName_;
0369
0370 bool runonendrun_;
0371 bool runonendjob_;
0372 bool runonendlumi_;
0373 bool runineventloop_;
0374 bool dumpMemHistory_;
0375 bool dumpToFWJR_;
0376
0377
0378 };
0379
0380 using namespace std;
0381 using namespace edm;
0382
0383 template <class T>
0384 static unsigned int getEmptyMetric(T* array, int lenx, int leny, int lenz) {
0385
0386 unsigned int len = lenx + leny + lenz;
0387 unsigned int result = 0;
0388
0389
0390
0391 for (unsigned int i = 1; i < len; ++i) {
0392
0393 if (i % (lenx - 1) == 0)
0394 continue;
0395 if (i % lenx == 0)
0396 continue;
0397 if (i % (lenx + leny - 1) == 0)
0398 continue;
0399 if (i % (lenx + leny) == 0)
0400 continue;
0401 if (i % (lenx + leny + lenz - 1) == 0)
0402 continue;
0403
0404 if (array[i] == 0)
0405 result += 1;
0406 }
0407
0408 return result;
0409 }
0410
0411
0412
0413
0414 DQMStoreStats::DQMStoreStats(const edm::ParameterSet& ps)
0415 : subsystem_(""),
0416 subfolder_(""),
0417 nbinsglobal_(0),
0418 nbinssubsys_(0),
0419 nmeglobal_(0),
0420 nmesubsys_(0),
0421 maxbinsglobal_(0),
0422 maxbinssubsys_(0),
0423 maxbinsmeglobal_(""),
0424 maxbinsmesubsys_(""),
0425 statsdepth_(1),
0426 pathnamematch_(""),
0427 verbose_(0) {
0428 parameters_ = ps;
0429 pathnamematch_ = ps.getUntrackedParameter<std::string>("pathNameMatch", pathnamematch_);
0430 statsdepth_ = ps.getUntrackedParameter<int>("statsDepth", statsdepth_);
0431 verbose_ = ps.getUntrackedParameter<int>("verbose", verbose_);
0432 dumpMemHistory_ = ps.getUntrackedParameter<bool>("dumpMemoryHistory", false);
0433 runonendrun_ = ps.getUntrackedParameter<bool>("runOnEndRun", true);
0434 runonendjob_ = ps.getUntrackedParameter<bool>("runOnEndJob", false);
0435 runonendlumi_ = ps.getUntrackedParameter<bool>("runOnEndLumi", false);
0436 runineventloop_ = ps.getUntrackedParameter<bool>("runInEventLoop", false);
0437 dumpToFWJR_ = ps.getUntrackedParameter<bool>("dumpToFWJR", false);
0438
0439 startingTime_ = time(nullptr);
0440 }
0441
0442 void DQMStoreStats::calcIgProfDump(Folder& root) {
0443 std::ofstream stream("dqm-bin-stats.sql");
0444 stream << ""
0445 " PRAGMA journal_mode=OFF;"
0446 " PRAGMA count_changes=OFF;"
0447 " DROP TABLE IF EXISTS files;"
0448 " DROP TABLE IF EXISTS symbols;"
0449 " DROP TABLE IF EXISTS mainrows;"
0450 " DROP TABLE IF EXISTS children;"
0451 " DROP TABLE IF EXISTS parents;"
0452 " DROP TABLE IF EXISTS summary;"
0453 " CREATE TABLE children ("
0454 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
0455 " parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
0456 " from_parent_count INTEGER,"
0457 " from_parent_calls INTEGER,"
0458 " from_parent_paths INTEGER,"
0459 " pct REAL"
0460 " );"
0461 " CREATE TABLE files ("
0462 " id,"
0463 " name TEXT"
0464 " );"
0465 " CREATE TABLE mainrows ("
0466 " id INTEGER PRIMARY KEY,"
0467 " symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
0468 " self_count INTEGER,"
0469 " cumulative_count INTEGER,"
0470 " kids INTEGER,"
0471 " self_calls INTEGER,"
0472 " total_calls INTEGER,"
0473 " self_paths INTEGER,"
0474 " total_paths INTEGER,"
0475 " pct REAL"
0476 " );"
0477 " CREATE TABLE parents ("
0478 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
0479 " child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
0480 " to_child_count INTEGER,"
0481 " to_child_calls INTEGER,"
0482 " to_child_paths INTEGER,"
0483 " pct REAL"
0484 " );"
0485 " CREATE TABLE summary ("
0486 " counter TEXT,"
0487 " total_count INTEGER,"
0488 " total_freq INTEGER,"
0489 " tick_period REAL"
0490 " );"
0491 " CREATE TABLE symbols ("
0492 " id,"
0493 " name TEXT,"
0494 " filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
0495 " );"
0496 " CREATE UNIQUE INDEX fileIndex ON files (id);"
0497 " CREATE INDEX selfCountIndex ON mainrows(self_count);"
0498 " CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
0499 " CREATE INDEX totalCountIndex ON mainrows(cumulative_count);"
0500 << std::endl;
0501
0502 std::string sql_statement("");
0503
0504 root.files(sql_statement);
0505 root.symbols(sql_statement);
0506 root.mainrows_cumulative(sql_statement);
0507 root.summary(sql_statement);
0508 VIterator<Folder*> subsystems = root.CreateIterator();
0509 size_t ii = 0;
0510 for (subsystems.First(); !subsystems.IsDone(); subsystems.Next(), ++ii) {
0511 subsystems.CurrentItem()->mainrows(sql_statement);
0512 subsystems.CurrentItem()->parents(sql_statement);
0513 subsystems.CurrentItem()->children(sql_statement);
0514 }
0515 stream << sql_statement << std::endl;
0516 }
0517
0518 std::vector<double> DQMStoreStats::GetTH2PolyArray(TH2Poly* poly) {
0519 int nBins = poly->GetNumberOfBins();
0520 std::vector<double> array(nBins + 1, 0.0);
0521 for (int i = 1; i <= nBins; i++) {
0522 array[i] = poly->GetBinContent(i);
0523 }
0524 return array;
0525 }
0526
0527
0528
0529
0530
0531
0532
0533 int DQMStoreStats::calcstats(int mode = DQMStoreStats::considerAllME) {
0534
0535 nbinsglobal_ = 0;
0536 nbinssubsys_ = 0;
0537 maxbinsglobal_ = 0;
0538 maxbinssubsys_ = 0;
0539 std::string path = "";
0540 std::string subsystemname = "";
0541 std::string subfoldername = "";
0542 size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
0543
0544 std::vector<MonitorElement*> melist;
0545 melist = dbe_->getAllContents(pathnamematch_);
0546
0547 Folder dbeFolder("root");
0548 DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
0549
0550
0551 for (auto& it : melist) {
0552
0553 if (mode == DQMStoreStats::considerOnlyLumiProductME && !(it->getLumiFlag()))
0554 continue;
0555
0556
0557 const std::string& path = it->getPathname();
0558
0559 subfolderStringBegin = 0;
0560 Folder* curr = &dbeFolder;
0561 while (true) {
0562 subfolderStringEnd = path.find('/', subfolderStringBegin);
0563 if (std::string::npos == subfolderStringEnd) {
0564 curr = curr->cd(path.substr(subfolderStringBegin, path.size() - subfolderStringBegin));
0565 break;
0566 }
0567 curr = curr->cd(path.substr(subfolderStringBegin, subfolderStringEnd - subfolderStringBegin));
0568 subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
0569 }
0570
0571
0572 if (path.empty())
0573 continue;
0574
0575 subsysStringEnd = path.find('/', 0);
0576 if (std::string::npos == subsysStringEnd)
0577 subsysStringEnd = path.size();
0578
0579
0580 if (path.substr(0, subsysStringEnd) != subsystemname) {
0581 DQMStoreStatsSubsystem aSubsystem;
0582 subsystemname = path.substr(0, subsysStringEnd);
0583 aSubsystem.subsystemName_ = subsystemname;
0584 dqmStoreStatsTopLevel.push_back(aSubsystem);
0585 subfoldername = "";
0586 }
0587
0588
0589 if (path.size() == subsysStringEnd) {
0590
0591 DQMStoreStatsSubfolder aSubfolder;
0592 aSubfolder.subfolderName_ = subsystemname;
0593 dqmStoreStatsTopLevel.back().push_back(aSubfolder);
0594 }
0595
0596 else {
0597
0598 subfolderStringEnd = path.find('/', subsysStringEnd + 1);
0599 if (std::string::npos == subfolderStringEnd)
0600 subfolderStringEnd = path.size();
0601
0602
0603 if (path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1) != subfoldername) {
0604 subfoldername = path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1);
0605 DQMStoreStatsSubfolder aSubfolder;
0606 aSubfolder.subfolderName_ = subfoldername;
0607 dqmStoreStatsTopLevel.back().push_back(aSubfolder);
0608 }
0609 }
0610
0611
0612 DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
0613
0614 switch (it->kind()) {
0615
0616 case MonitorElement::Kind::TH1F:
0617 currentSubfolder.AddBinsF(it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0));
0618 curr->update(it->getNbinsX(),
0619 getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),
0620 it->getNbinsX() * sizeof(float));
0621 break;
0622 case MonitorElement::Kind::TH1S:
0623 currentSubfolder.AddBinsS(it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0));
0624 curr->update(it->getNbinsX(),
0625 getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),
0626 it->getNbinsX() * sizeof(short));
0627 break;
0628 case MonitorElement::Kind::TH1D:
0629 currentSubfolder.AddBinsD(it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0));
0630 curr->update(it->getNbinsX(),
0631 getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),
0632 it->getNbinsX() * sizeof(double));
0633 break;
0634 case MonitorElement::Kind::TH1I:
0635 currentSubfolder.AddBinsI(it->getNbinsX(), getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0));
0636 curr->update(it->getNbinsX(),
0637 getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0),
0638 it->getNbinsX() * sizeof(int));
0639 break;
0640 case MonitorElement::Kind::TPROFILE:
0641 currentSubfolder.AddBinsD(it->getNbinsX(),
0642 getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0));
0643 curr->update(it->getNbinsX(),
0644 getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),
0645 it->getNbinsX() * sizeof(double));
0646 break;
0647
0648
0649 case MonitorElement::Kind::TH2F:
0650 currentSubfolder.AddBinsF(
0651 it->getNbinsX() * it->getNbinsY(),
0652 getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0653 curr->update(it->getNbinsX() * it->getNbinsY(),
0654 getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0655 it->getNbinsX() * it->getNbinsY() * sizeof(float));
0656 break;
0657 case MonitorElement::Kind::TH2S:
0658 currentSubfolder.AddBinsS(
0659 it->getNbinsX() * it->getNbinsY(),
0660 getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0661 curr->update(it->getNbinsX() * it->getNbinsY(),
0662 getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0663 it->getNbinsX() * it->getNbinsY() * sizeof(short));
0664 break;
0665 case MonitorElement::Kind::TH2D:
0666 currentSubfolder.AddBinsD(
0667 it->getNbinsX() * it->getNbinsY(),
0668 getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0669 curr->update(it->getNbinsX() * it->getNbinsY(),
0670 getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0671 it->getNbinsX() * it->getNbinsY() * sizeof(double));
0672 break;
0673 case MonitorElement::Kind::TH2I:
0674 currentSubfolder.AddBinsI(
0675 it->getNbinsX() * it->getNbinsY(),
0676 getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0677 curr->update(it->getNbinsX() * it->getNbinsY(),
0678 getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0679 it->getNbinsX() * it->getNbinsY() * sizeof(int));
0680 break;
0681 case MonitorElement::Kind::TH2Poly: {
0682 std::vector<double> polyArray = GetTH2PolyArray(it->getTH2Poly());
0683 int nBins = polyArray.size() - 1;
0684 currentSubfolder.AddBinsD(nBins, getEmptyMetric(polyArray.data(), nBins + 1, 1, 0));
0685 curr->update(nBins, getEmptyMetric(polyArray.data(), nBins + 1, 1, 0), nBins * sizeof(double));
0686 break;
0687 }
0688 case MonitorElement::Kind::TPROFILE2D:
0689 currentSubfolder.AddBinsD(
0690 it->getNbinsX() * it->getNbinsY(),
0691 getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0692 curr->update(it->getNbinsX() * it->getNbinsY(),
0693 getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0694 it->getNbinsX() * it->getNbinsY() * sizeof(double));
0695 break;
0696
0697
0698 case MonitorElement::Kind::TH3F:
0699 currentSubfolder.AddBinsF(
0700 it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
0701 getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2));
0702 curr->update(
0703 it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
0704 getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2),
0705 it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float));
0706 break;
0707
0708 default: {
0709 }
0710
0711
0712
0713
0714
0715
0716 }
0717 }
0718
0719 if (mode == DQMStoreStats::considerAllME)
0720 calcIgProfDump(dbeFolder);
0721
0722
0723
0724 std::cout << endl;
0725 std::cout << "==========================================================================================="
0726 << std::endl;
0727 std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
0728 if (mode == DQMStoreStats::considerAllME)
0729 std::cout << "FOR ALL ME" << std::endl;
0730 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0731 std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
0732 std::cout << "==========================================================================================="
0733 << std::endl;
0734 std::cout << endl;
0735
0736 std::cout << "------------------------------------------------------------------------------------------"
0737 << std::endl;
0738 std::cout << "Configuration:" << std::endl;
0739 std::cout << "------------------------------------------------------------------------------------------"
0740 << std::endl;
0741 std::cout << " > running ";
0742 if (runonendrun_)
0743 std::cout << "on run end." << std::endl;
0744 if (runonendlumi_)
0745 std::cout << "on lumi end." << std::endl;
0746 if (runonendjob_)
0747 std::cout << "on job end." << std::endl;
0748 if (runineventloop_)
0749 std::cout << "in event loop." << std::endl;
0750 std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
0751 std::cout << std::endl;
0752
0753
0754 std::cout << "------------------------------------------------------------------------------------------"
0755 << std::endl;
0756 std::cout << "Top level folder tree:" << std::endl;
0757 std::cout << "------------------------------------------------------------------------------------------"
0758 << std::endl;
0759 for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
0760 std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
0761
0762 for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
0763 std::cout << " |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
0764 }
0765 }
0766
0767
0768
0769 unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
0770
0771 std::cout << std::endl;
0772 std::cout << "------------------------------------------------------------------------------------------"
0773 << std::endl;
0774 std::cout << "Detailed ressource usage information ";
0775 if (mode == DQMStoreStats::considerAllME)
0776 std::cout << "FOR ALL ME" << std::endl;
0777 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0778 std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
0779 std::cout << "------------------------------------------------------------------------------------------"
0780 << std::endl;
0781 std::cout << "subsystem/folder histograms bins Empty bins Empty/Total "
0782 "bins per MB kB per"
0783 << std::endl;
0784 std::cout << " (total) (total) (total) "
0785 "histogram (total) histogram "
0786 << std::endl;
0787 std::cout << "------------------------------------------------------------------------------------------"
0788 << std::endl;
0789 for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
0790 std::cout << it0->subsystemName_ << std::endl;
0791
0792 unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
0793
0794 for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
0795
0796 std::string thisSubfolderName(it1->subfolderName_);
0797 if (thisSubfolderName.size() > 30) {
0798 thisSubfolderName.resize(30);
0799 thisSubfolderName.replace(thisSubfolderName.size() - 3, 3, 3, '.');
0800 }
0801
0802 std::cout << " -> " << std::setw(30) << std::left << thisSubfolderName;
0803 std::cout << std::setw(14) << std::right << it1->totalHistos_;
0804 std::cout << std::setw(14) << std::right << it1->totalBins_;
0805 std::cout << std::setw(14) << std::right << it1->totalEmptyBins_;
0806 std::cout << std::setw(14) << std::right << std::setprecision(3)
0807 << (float)it1->totalEmptyBins_ / (float)it1->totalBins_;
0808
0809
0810 if (it1->totalHistos_) {
0811 std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalBins_ / float(it1->totalHistos_);
0812 } else
0813 std::cout << std::setw(14) << std::right << "-";
0814
0815 std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalMemory_ / 1024. / 1024.;
0816
0817
0818 if (it1->totalHistos_) {
0819 std::cout << std::setw(14) << std::right << std::setprecision(3)
0820 << it1->totalMemory_ / 1024. / it1->totalHistos_;
0821 } else
0822 std::cout << std::setw(14) << std::right << "-";
0823
0824 std::cout << std::endl;
0825
0826
0827 nHistograms += it1->totalHistos_;
0828 nBins += it1->totalBins_;
0829 nEmptyBins += it1->totalEmptyBins_;
0830 nBytes += it1->totalMemory_;
0831 }
0832
0833 overallNHistograms += nHistograms;
0834 overallNBins += nBins;
0835 overallNBytes += nBytes;
0836
0837
0838 std::cout << " " << std::setw(30) << std::left << "SUBSYSTEM TOTAL";
0839 std::cout << std::setw(14) << std::right << nHistograms;
0840 std::cout << std::setw(14) << std::right << nBins;
0841 std::cout << std::setw(14) << std::right << nEmptyBins;
0842 std::cout << std::setw(14) << std::right << (float)nEmptyBins / (float)nBins;
0843 std::cout << std::setw(14) << std::right << std::setprecision(3) << nBins / float(nHistograms);
0844 std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / 1000.;
0845 std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / nHistograms;
0846 std::cout << std::endl;
0847
0848 std::cout << ".........................................................................................."
0849 << std::endl;
0850 }
0851
0852
0853 std::cout << std::endl;
0854 std::cout << "------------------------------------------------------------------------------------------"
0855 << std::endl;
0856 std::cout << "Grand total ";
0857 if (mode == DQMStoreStats::considerAllME)
0858 std::cout << "FOR ALL ME:" << std::endl;
0859 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0860 std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
0861 std::cout << "------------------------------------------------------------------------------------------"
0862 << std::endl;
0863 std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
0864 std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether"
0865 << std::endl;
0866 std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB"
0867 << std::endl;
0868
0869 std::cout << endl;
0870 std::cout << "==========================================================================================="
0871 << std::endl;
0872 std::cout << "[DQMStoreStats::calcstats] -- End of output ";
0873 if (mode == DQMStoreStats::considerAllME)
0874 std::cout << "FOR ALL ME." << std::endl;
0875 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0876 std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
0877 std::cout << "==========================================================================================="
0878 << std::endl;
0879 std::cout << endl;
0880
0881
0882
0883
0884 if (dumpToFWJR_) {
0885 edm::Service<edm::JobReport> jr;
0886
0887 if (!jr.isAvailable())
0888 return 0;
0889
0890 std::map<std::string, std::string> jrInfo;
0891
0892 jrInfo["Source"] = "DQMServices/Components";
0893 jrInfo["FileClass"] = "DQMStoreStats";
0894 if (runonendrun_)
0895 jrInfo["DumpType"] = "EndRun";
0896 if (runonendlumi_)
0897 jrInfo["DumpType"] = "EndLumi";
0898 if (runonendjob_)
0899 jrInfo["DumpType"] = "EndJob";
0900 if (runineventloop_)
0901 jrInfo["DumpType"] = "EventLoop";
0902 if (mode == DQMStoreStats::considerAllME)
0903 jrInfo["Type"] = "RunProduct";
0904 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0905 jrInfo["Type"] = "LumiProduct";
0906
0907 jrInfo["pathNameMatch"] = pathnamematch_;
0908
0909 for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
0910 unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
0911 for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
0912
0913 nHistograms += it1->totalHistos_;
0914 nBins += it1->totalBins_;
0915 nEmptyBins += it1->totalEmptyBins_;
0916 nBytes += it1->totalMemory_;
0917 }
0918 std::stringstream iss("");
0919 iss << nHistograms;
0920 jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
0921 iss.str("");
0922 iss << nBins;
0923 jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
0924 iss.str("");
0925 iss << nEmptyBins;
0926 jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
0927 iss.str("");
0928 iss << ((float)nEmptyBins / (float)nBins);
0929 jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
0930 iss.str("");
0931 iss << ((float)nBins / (float)nHistograms);
0932 jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
0933 iss.str("");
0934 iss << nBytes / 1024. / 1024.;
0935 jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
0936 iss.str("");
0937 iss << nBytes / 1024. / nHistograms;
0938 jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
0939 }
0940 jr->reportAnalysisFile("DQMStatsReport", jrInfo);
0941 }
0942
0943 return 0;
0944 }
0945
0946
0947
0948
0949 void DQMStoreStats::dumpMemoryProfile() {
0950 std::cout << std::endl;
0951 std::cout << "------------------------------------------------------------------------------------------"
0952 << std::endl;
0953 std::cout << "Memory profile:" << std::endl;
0954 std::cout << "------------------------------------------------------------------------------------------"
0955 << std::endl;
0956
0957
0958 std::pair<time_t, unsigned int> maxItem(0, 0);
0959 for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
0960 if (it->second > maxItem.second) {
0961 maxItem = *it;
0962 }
0963 }
0964
0965 std::stringstream rootOutputFileName;
0966 rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
0967
0968
0969 if (dumpMemHistory_ && isOpenProcFileSuccessful_) {
0970 TFile outputFile(rootOutputFileName.str().c_str(), "RECREATE");
0971
0972 int aTime;
0973 float aMb;
0974
0975 TTree memHistoryTree("dqmstorestats_memhistory", "memory history");
0976 memHistoryTree.Branch("seconds", &aTime, "seconds/I");
0977 memHistoryTree.Branch("megabytes", &aMb, "megabytes/F");
0978 for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
0979 aTime = it->first - startingTime_;
0980 aMb = it->second / 1000.;
0981 memHistoryTree.Fill();
0982 }
0983
0984 outputFile.Write();
0985 outputFile.Close();
0986 }
0987
0988 std::cout << "Approx. maximum total virtual memory size of job: ";
0989 if (isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty()) {
0990 std::cout << maxItem.second / 1000. << " MB (reached " << maxItem.first - startingTime_
0991 << " sec. after constructor called)," << std::endl;
0992 std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size()
0993 << " samples)" << std::endl;
0994 } else {
0995 std::cout << "(could not be determined)" << std::endl;
0996 }
0997
0998 std::cout << std::endl << std::endl;
0999 }
1000
1001
1002
1003
1004 void DQMStoreStats::print() {
1005
1006 std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
1007 std::cout << " " << subfolder_ << ": ";
1008 std::cout << nmesubsys_ << " histograms with " << nbinssubsys_ << " bins. ";
1009 if (nmesubsys_ > 0)
1010 std::cout << nbinssubsys_ / nmesubsys_ << " bins/histogram ";
1011 std::cout << std::endl;
1012 std::cout << " Largest histogram: " << maxbinsmesubsys_ << " with " << maxbinssubsys_ << " bins." << std::endl;
1013 }
1014
1015
1016
1017
1018 std::pair<unsigned int, unsigned int> DQMStoreStats::readMemoryEntry() const {
1019
1020 if (isOpenProcFileSuccessful_) {
1021 std::ifstream procFile(procFileName_.str().c_str(), ios::in);
1022
1023 std::string readBuffer("");
1024 unsigned int memSize = 0;
1025
1026
1027 while (!procFile.eof()) {
1028 procFile >> readBuffer;
1029 if (std::string("VmSize:") == readBuffer) {
1030 procFile >> memSize;
1031 break;
1032 }
1033 }
1034
1035 procFile.close();
1036 return std::pair<time_t, unsigned int>(time(nullptr), memSize);
1037 }
1038
1039 return std::pair<time_t, unsigned int>(0, 0);
1040 }
1041
1042
1043
1044
1045 void DQMStoreStats::beginJob() {
1046
1047 dbe_ = Service<DQMStore>().operator->();
1048
1049
1050 procFileName_ << "/proc/" << getpid() << "/status";
1051
1052
1053 std::ifstream procFile(procFileName_.str().c_str(), ios::in);
1054
1055 if (procFile.good()) {
1056 isOpenProcFileSuccessful_ = true;
1057 } else {
1058 std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
1059 std::cerr << " Total memory profile will not be available." << std::endl;
1060 isOpenProcFileSuccessful_ = false;
1061 }
1062
1063 procFile.close();
1064 }
1065
1066
1067
1068
1069 void DQMStoreStats::beginRun(const edm::Run& r, const EventSetup& context) {}
1070
1071
1072
1073
1074 void DQMStoreStats::analyze(const Event& iEvent, const EventSetup& iSetup) {
1075
1076 memoryHistoryVector_.emplace_back(readMemoryEntry());
1077
1078 if (runineventloop_) {
1079 calcstats(DQMStoreStats::considerAllME);
1080 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1081 dumpMemoryProfile();
1082 }
1083 }
1084
1085
1086
1087
1088 void DQMStoreStats::endLuminosityBlock(const LuminosityBlock& lumiSeg, const EventSetup& context) {
1089 if (runonendlumi_) {
1090 calcstats(DQMStoreStats::considerAllME);
1091 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1092 dumpMemoryProfile();
1093 }
1094 }
1095
1096
1097
1098
1099 void DQMStoreStats::beginLuminosityBlock(const edm::LuminosityBlock& lumiSeg, const edm::EventSetup& context) {}
1100
1101
1102
1103
1104 void DQMStoreStats::endRun(const Run& r, const EventSetup& context) {
1105 if (runonendrun_) {
1106 calcstats(DQMStoreStats::considerAllME);
1107 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1108 dumpMemoryProfile();
1109 }
1110 }
1111
1112
1113
1114
1115 void DQMStoreStats::endJob() {
1116 if (runonendjob_) {
1117 calcstats(DQMStoreStats::considerAllME);
1118 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1119 dumpMemoryProfile();
1120 }
1121 }
1122
1123 #include "FWCore/PluginManager/interface/ModuleDef.h"
1124 #include "FWCore/Framework/interface/MakerMacros.h"
1125 DEFINE_FWK_MODULE(DQMStoreStats);