File indexing completed on 2023-03-17 10:58:55
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014
0015
0016
0017
0018 #include <string>
0019 #include <sstream>
0020 #include <utility>
0021 #include <vector>
0022 #include <iostream>
0023 #include <iomanip>
0024 #include <utility>
0025 #include <fstream>
0026 #include <sstream>
0027
0028 #include "TFile.h"
0029 #include "TTree.h"
0030
0031 #include "DQMServices/Core/interface/DQMStore.h"
0032 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
0033 #include "FWCore/MessageLogger/interface/JobReport.h"
0034 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0035 #include "FWCore/ServiceRegistry/interface/Service.h"
0036
0037
0038
0039
0040
0041 using dqm::legacy::DQMStore;
0042 using dqm::legacy::MonitorElement;
0043
0044
0045
0046
0047
0048 class DQMStoreStatsSubfolder {
0049 public:
0050 DQMStoreStatsSubfolder() {
0051 totalHistos_ = 0;
0052 totalBins_ = 0;
0053 totalMemory_ = 0;
0054 totalEmptyBins_ = 0;
0055 }
0056 std::string subfolderName_;
0057 unsigned int totalHistos_;
0058 unsigned int totalBins_;
0059 unsigned int totalEmptyBins_;
0060 unsigned int totalMemory_;
0061 void AddBinsF(unsigned int nBins, unsigned int nEmptyBins) {
0062 ++totalHistos_;
0063 totalBins_ += nBins;
0064 totalEmptyBins_ += nEmptyBins;
0065 totalMemory_ += (nBins *= sizeof(float));
0066 }
0067 void AddBinsS(unsigned int nBins, unsigned int nEmptyBins) {
0068 ++totalHistos_;
0069 totalBins_ += nBins;
0070 totalEmptyBins_ += nEmptyBins;
0071 totalMemory_ += (nBins *= sizeof(short));
0072 }
0073 void AddBinsD(unsigned int nBins, unsigned int nEmptyBins) {
0074 ++totalHistos_;
0075 totalBins_ += nBins;
0076 totalEmptyBins_ += nEmptyBins;
0077 totalMemory_ += (nBins *= sizeof(double));
0078 }
0079 void AddBinsI(unsigned int nBins, unsigned int nEmptyBins) {
0080 ++totalHistos_;
0081 totalBins_ += nBins;
0082 totalEmptyBins_ += nEmptyBins;
0083 totalMemory_ += (nBins *= sizeof(int));
0084 }
0085 };
0086
0087
0088
0089
0090
0091 class DQMStoreStatsSubsystem : public std::vector<DQMStoreStatsSubfolder> {
0092 public:
0093 DQMStoreStatsSubsystem() = default;
0094 std::string subsystemName_;
0095 };
0096
0097
0098
0099
0100
0101 class DQMStoreStatsTopLevel : public std::vector<DQMStoreStatsSubsystem> {
0102 public:
0103 DQMStoreStatsTopLevel() = default;
0104 };
0105
0106 template <class Item>
0107 class Iterator {
0108 public:
0109 virtual ~Iterator() = default;
0110 virtual void First() = 0;
0111 virtual void Next() = 0;
0112 virtual bool IsDone() const = 0;
0113 virtual Item CurrentItem() const = 0;
0114
0115 protected:
0116 Iterator() = default;
0117 };
0118
0119 template <class Item>
0120 class VIterator : public Iterator<Item> {
0121 public:
0122 VIterator(const std::vector<Item>* aVector) : vector_(aVector), index(0) {}
0123 ~VIterator() override = default;
0124 void First() override { index = 0; }
0125 void Next() override { ++index; }
0126 virtual int size() { return vector_->size(); }
0127 virtual int getIndex() { return (int)index; }
0128
0129 bool IsDone() const override {
0130 if (index < (unsigned int)vector_->size())
0131 return false;
0132 return true;
0133 }
0134
0135 Item CurrentItem() const override { return vector_->operator[](index); }
0136
0137 private:
0138 const std::vector<Item>* vector_;
0139 unsigned int index;
0140 };
0141
0142 static unsigned int getId() {
0143 static unsigned int id = 10;
0144 return ++id;
0145 }
0146
0147 class Folder {
0148 public:
0149 Folder(std::string name)
0150 : totalHistos_(0),
0151 totalBins_(0),
0152 totalEmptyBins_(0),
0153 totalMemory_(0),
0154 id_(10),
0155 level_(0),
0156 folderName_(std::move(name)),
0157 father_(nullptr) {}
0158
0159 ~Folder() {
0160 for (auto& subfolder : subfolders_)
0161 delete subfolder;
0162 }
0163
0164 void setFather(Folder* e) { father_ = e; }
0165 Folder* getFather() { return father_; }
0166 const std::string& name() { return folderName_; }
0167
0168 Folder* cd(const std::string& name) {
0169 for (auto& subfolder : subfolders_)
0170 if (subfolder->name() == name)
0171 return subfolder;
0172 auto* tmp = new Folder(name);
0173 this->add(tmp);
0174 return tmp;
0175 }
0176
0177 void setId(unsigned int id) { id_ = id; }
0178 unsigned int id() { return id_; }
0179 void setLevel(unsigned int value) { level_ = value; }
0180 unsigned int level() { return level_; }
0181
0182 void add(Folder* f) {
0183 f->setFather(this);
0184 subfolders_.push_back(f);
0185 f->setLevel(level_ + 1);
0186 f->setId(getId());
0187 }
0188
0189 unsigned int getHistos() {
0190 unsigned int result = totalHistos_;
0191 for (auto& subfolder : subfolders_)
0192 result += subfolder->getHistos();
0193 return result;
0194 }
0195 unsigned int getBins() {
0196 unsigned int result = totalBins_;
0197 for (auto& subfolder : subfolders_)
0198 result += subfolder->getBins();
0199 return result;
0200 }
0201 unsigned int getEmptyBins() {
0202 unsigned int result = totalEmptyBins_;
0203 for (auto& subfolder : subfolders_)
0204 result += subfolder->getEmptyBins();
0205 return result;
0206 }
0207 unsigned int getMemory() {
0208 unsigned int result = totalMemory_;
0209 for (auto& subfolder : subfolders_)
0210 result += subfolder->getMemory();
0211 return result;
0212 }
0213 void update(unsigned int bins, unsigned int empty, unsigned int memory) {
0214 totalHistos_ += 1;
0215 totalBins_ += bins;
0216 totalEmptyBins_ += empty;
0217 totalMemory_ += memory;
0218 }
0219 void dump(std::string indent) {
0220 indent.append(" ");
0221 std::cout << indent << "I'm a " << name() << " whose father is " << getFather() << " with ID: " << id_
0222 << " Histo: " << getHistos() << " Bins: " << getBins() << " EmptyBins: " << getEmptyBins()
0223 << " Memory: " << getMemory() << " and my children are: " << std::endl;
0224 for (auto& subfolder : subfolders_)
0225 subfolder->dump(indent);
0226 }
0227 VIterator<Folder*> CreateIterator() { return VIterator<Folder*>(&subfolders_); }
0228
0229 void mainrows(std::string& sql_statement) {
0230 std::stringstream s("");
0231 s << "INSERT INTO mainrows(id, symbol_id, self_count, cumulative_count, kids, self_calls, total_calls, self_paths, "
0232 "total_paths, pct)"
0233 " VALUES("
0234 << id_ << ", " << id_ << ", " << getMemory() << ", " << getMemory() << ", " << subfolders_.size() << ", "
0235 << getBins() - getEmptyBins() << ", " << getBins() << ", " << getHistos() << ", " << getHistos() << ", 0.0);\n";
0236 sql_statement.append(s.str());
0237 for (auto& subfolder : subfolders_)
0238 subfolder->mainrows(sql_statement);
0239 }
0240
0241 void symbols(std::string& sql_statement) {
0242 unsigned int parentid = this->getFather() ? this->getFather()->id() : id_;
0243 std::stringstream s("");
0244 s << "INSERT INTO symbols(id, name, filename_id) VALUES (" << id_ << ",\"" << folderName_ << "\", " << parentid
0245 << ");\n";
0246 sql_statement.append(s.str());
0247 for (auto& subfolder : subfolders_)
0248 subfolder->symbols(sql_statement);
0249 }
0250
0251 void parents(std::string& sql_statement) {
0252 unsigned int parentid = this->getFather() ? this->getFather()->id() : id_;
0253 std::stringstream s("");
0254 s << "INSERT INTO parents(self_id, child_id, to_child_count, to_child_calls, to_child_paths, pct) VALUES("
0255 << parentid << "," << id_ << "," << totalMemory_ << "," << totalBins_ << "," << totalHistos_ << ",0"
0256 << ");\n";
0257 sql_statement.append(s.str());
0258 for (auto& subfolder : subfolders_)
0259 subfolder->parents(sql_statement);
0260 }
0261
0262 void children(std::string& sql_statement) {
0263 unsigned int parentid = this->getFather() ? this->getFather()->id() : id_;
0264 std::stringstream s("");
0265 s << "INSERT INTO children(self_id, parent_id, from_parent_count, from_parent_calls, from_parent_paths, pct) "
0266 "VALUES("
0267 << id_ << "," << parentid << "," << getMemory() << "," << getBins() - getEmptyBins() << "," << totalHistos_
0268 << ",0"
0269 << ");\n";
0270 sql_statement.append(s.str());
0271 for (auto& subfolder : subfolders_)
0272 subfolder->children(sql_statement);
0273 }
0274
0275 void mainrows_cumulative(std::string& sql_statement) {
0276 std::stringstream s("");
0277 s << "INSERT INTO mainrows(id, symbol_id, self_count, cumulative_count, kids, self_calls, total_calls, self_paths, "
0278 "total_paths, pct)"
0279 << " VALUES(" << id_ << "," << id_ << "," << 0 << "," << getMemory() << ", 0," << getBins() - getEmptyBins()
0280 << "," << getBins() << ", 0, " << getHistos() << ", 0);\n";
0281 sql_statement.append(s.str());
0282 }
0283
0284 void summary(std::string& sql_statement) {
0285 std::stringstream s("");
0286 s << "INSERT INTO summary(counter, total_count, total_freq, tick_period) VALUES (\"BINS_LIVE\"," << getMemory()
0287 << "," << getBins() << ", 1);\n";
0288 sql_statement.append(s.str());
0289 }
0290
0291 void files(std::string& sql_statement) {
0292 std::stringstream s("");
0293 s << "INSERT INTO files(id, name) VALUES(" << id_ << ",\"" << folderName_ << "\");\n";
0294 sql_statement.append(s.str());
0295 }
0296
0297 private:
0298 unsigned int totalHistos_;
0299 unsigned int totalBins_;
0300 unsigned int totalEmptyBins_;
0301 unsigned int totalMemory_;
0302 unsigned int id_;
0303 unsigned int level_;
0304 std::string folderName_;
0305 Folder* father_;
0306 std::vector<Folder*> subfolders_;
0307 };
0308
0309
0310
0311
0312 class DQMStoreStats : public edm::one::EDAnalyzer<edm::one::WatchRuns, edm::one::WatchLuminosityBlocks> {
0313 public:
0314 DQMStoreStats(const edm::ParameterSet&);
0315 ~DQMStoreStats() override = default;
0316
0317 enum statsMode { considerAllME = 0, considerOnlyLumiProductME = 1 };
0318
0319 protected:
0320
0321 void beginJob() override;
0322
0323
0324 void beginRun(const edm::Run& r, const edm::EventSetup& c) override;
0325
0326
0327 void analyze(const edm::Event& e, const edm::EventSetup& c) override;
0328
0329
0330 void beginLuminosityBlock(const edm::LuminosityBlock& lumiSeg, const edm::EventSetup& c) override;
0331 void endLuminosityBlock(const edm::LuminosityBlock& lumiSeg, const edm::EventSetup& c) override;
0332
0333
0334 void endRun(const edm::Run& r, const edm::EventSetup& c) override;
0335
0336
0337 void endJob() override;
0338
0339 private:
0340 int calcstats(int);
0341 void calcIgProfDump(Folder&);
0342 void dumpMemoryProfile();
0343 std::pair<unsigned int, unsigned int> readMemoryEntry() const;
0344 void print();
0345
0346 DQMStore* dbe_;
0347 edm::ParameterSet parameters_;
0348
0349 std::string subsystem_;
0350 std::string subfolder_;
0351 int nbinsglobal_;
0352 int nbinssubsys_;
0353 int nmeglobal_;
0354 int nmesubsys_;
0355 int maxbinsglobal_;
0356 int maxbinssubsys_;
0357 std::string maxbinsmeglobal_;
0358 std::string maxbinsmesubsys_;
0359
0360 int statsdepth_;
0361 std::string pathnamematch_;
0362 int verbose_;
0363
0364 std::vector<std::pair<time_t, unsigned int> > memoryHistoryVector_;
0365 time_t startingTime_;
0366 bool isOpenProcFileSuccessful_;
0367 std::stringstream procFileName_;
0368
0369 bool runonendrun_;
0370 bool runonendjob_;
0371 bool runonendlumi_;
0372 bool runineventloop_;
0373 bool dumpMemHistory_;
0374 bool dumpToFWJR_;
0375
0376
0377 };
0378
0379 using namespace std;
0380 using namespace edm;
0381
0382 template <class T>
0383 static unsigned int getEmptyMetric(T* array, int lenx, int leny, int lenz) {
0384
0385 unsigned int len = lenx + leny + lenz;
0386 unsigned int result = 0;
0387
0388
0389
0390 for (unsigned int i = 1; i < len; ++i) {
0391
0392 if (i % (lenx - 1) == 0)
0393 continue;
0394 if (i % lenx == 0)
0395 continue;
0396 if (i % (lenx + leny - 1) == 0)
0397 continue;
0398 if (i % (lenx + leny) == 0)
0399 continue;
0400 if (i % (lenx + leny + lenz - 1) == 0)
0401 continue;
0402
0403 if (array[i] == 0)
0404 result += 1;
0405 }
0406
0407 return result;
0408 }
0409
0410
0411
0412
0413 DQMStoreStats::DQMStoreStats(const edm::ParameterSet& ps)
0414 : subsystem_(""),
0415 subfolder_(""),
0416 nbinsglobal_(0),
0417 nbinssubsys_(0),
0418 nmeglobal_(0),
0419 nmesubsys_(0),
0420 maxbinsglobal_(0),
0421 maxbinssubsys_(0),
0422 maxbinsmeglobal_(""),
0423 maxbinsmesubsys_(""),
0424 statsdepth_(1),
0425 pathnamematch_(""),
0426 verbose_(0) {
0427 parameters_ = ps;
0428 pathnamematch_ = ps.getUntrackedParameter<std::string>("pathNameMatch", pathnamematch_);
0429 statsdepth_ = ps.getUntrackedParameter<int>("statsDepth", statsdepth_);
0430 verbose_ = ps.getUntrackedParameter<int>("verbose", verbose_);
0431 dumpMemHistory_ = ps.getUntrackedParameter<bool>("dumpMemoryHistory", false);
0432 runonendrun_ = ps.getUntrackedParameter<bool>("runOnEndRun", true);
0433 runonendjob_ = ps.getUntrackedParameter<bool>("runOnEndJob", false);
0434 runonendlumi_ = ps.getUntrackedParameter<bool>("runOnEndLumi", false);
0435 runineventloop_ = ps.getUntrackedParameter<bool>("runInEventLoop", false);
0436 dumpToFWJR_ = ps.getUntrackedParameter<bool>("dumpToFWJR", false);
0437
0438 startingTime_ = time(nullptr);
0439 }
0440
0441 void DQMStoreStats::calcIgProfDump(Folder& root) {
0442 std::ofstream stream("dqm-bin-stats.sql");
0443 stream << ""
0444 " PRAGMA journal_mode=OFF;"
0445 " PRAGMA count_changes=OFF;"
0446 " DROP TABLE IF EXISTS files;"
0447 " DROP TABLE IF EXISTS symbols;"
0448 " DROP TABLE IF EXISTS mainrows;"
0449 " DROP TABLE IF EXISTS children;"
0450 " DROP TABLE IF EXISTS parents;"
0451 " DROP TABLE IF EXISTS summary;"
0452 " CREATE TABLE children ("
0453 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
0454 " parent_id INTEGER CONSTRAINT parent_exists REFERENCES mainrows(id),"
0455 " from_parent_count INTEGER,"
0456 " from_parent_calls INTEGER,"
0457 " from_parent_paths INTEGER,"
0458 " pct REAL"
0459 " );"
0460 " CREATE TABLE files ("
0461 " id,"
0462 " name TEXT"
0463 " );"
0464 " CREATE TABLE mainrows ("
0465 " id INTEGER PRIMARY KEY,"
0466 " symbol_id INTEGER CONSTRAINT symbol_id_exists REFERENCES symbols(id),"
0467 " self_count INTEGER,"
0468 " cumulative_count INTEGER,"
0469 " kids INTEGER,"
0470 " self_calls INTEGER,"
0471 " total_calls INTEGER,"
0472 " self_paths INTEGER,"
0473 " total_paths INTEGER,"
0474 " pct REAL"
0475 " );"
0476 " CREATE TABLE parents ("
0477 " self_id INTEGER CONSTRAINT self_exists REFERENCES mainrows(id),"
0478 " child_id INTEGER CONSTRAINT child_exists REFERENCES mainrows(id),"
0479 " to_child_count INTEGER,"
0480 " to_child_calls INTEGER,"
0481 " to_child_paths INTEGER,"
0482 " pct REAL"
0483 " );"
0484 " CREATE TABLE summary ("
0485 " counter TEXT,"
0486 " total_count INTEGER,"
0487 " total_freq INTEGER,"
0488 " tick_period REAL"
0489 " );"
0490 " CREATE TABLE symbols ("
0491 " id,"
0492 " name TEXT,"
0493 " filename_id INTEGER CONSTRAINT file_id_exists REFERENCES files(id)"
0494 " );"
0495 " CREATE UNIQUE INDEX fileIndex ON files (id);"
0496 " CREATE INDEX selfCountIndex ON mainrows(self_count);"
0497 " CREATE UNIQUE INDEX symbolsIndex ON symbols (id);"
0498 " CREATE INDEX totalCountIndex ON mainrows(cumulative_count);"
0499 << std::endl;
0500
0501 std::string sql_statement("");
0502
0503 root.files(sql_statement);
0504 root.symbols(sql_statement);
0505 root.mainrows_cumulative(sql_statement);
0506 root.summary(sql_statement);
0507 VIterator<Folder*> subsystems = root.CreateIterator();
0508 size_t ii = 0;
0509 for (subsystems.First(); !subsystems.IsDone(); subsystems.Next(), ++ii) {
0510 subsystems.CurrentItem()->mainrows(sql_statement);
0511 subsystems.CurrentItem()->parents(sql_statement);
0512 subsystems.CurrentItem()->children(sql_statement);
0513 }
0514 stream << sql_statement << std::endl;
0515 }
0516
0517
0518
0519
0520
0521
0522
0523 int DQMStoreStats::calcstats(int mode = DQMStoreStats::considerAllME) {
0524
0525 nbinsglobal_ = 0;
0526 nbinssubsys_ = 0;
0527 maxbinsglobal_ = 0;
0528 maxbinssubsys_ = 0;
0529 std::string path = "";
0530 std::string subsystemname = "";
0531 std::string subfoldername = "";
0532 size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0;
0533
0534 std::vector<MonitorElement*> melist;
0535 melist = dbe_->getAllContents(pathnamematch_);
0536
0537 Folder dbeFolder("root");
0538 DQMStoreStatsTopLevel dqmStoreStatsTopLevel;
0539
0540
0541 for (auto& it : melist) {
0542
0543 if (mode == DQMStoreStats::considerOnlyLumiProductME && !(it->getLumiFlag()))
0544 continue;
0545
0546
0547 const std::string& path = it->getPathname();
0548
0549 subfolderStringBegin = 0;
0550 Folder* curr = &dbeFolder;
0551 while (true) {
0552 subfolderStringEnd = path.find('/', subfolderStringBegin);
0553 if (std::string::npos == subfolderStringEnd) {
0554 curr = curr->cd(path.substr(subfolderStringBegin, path.size() - subfolderStringBegin));
0555 break;
0556 }
0557 curr = curr->cd(path.substr(subfolderStringBegin, subfolderStringEnd - subfolderStringBegin));
0558 subfolderStringBegin = ++subfolderStringEnd < path.size() ? subfolderStringEnd : path.size();
0559 }
0560
0561
0562 if (path.empty())
0563 continue;
0564
0565 subsysStringEnd = path.find('/', 0);
0566 if (std::string::npos == subsysStringEnd)
0567 subsysStringEnd = path.size();
0568
0569
0570 if (path.substr(0, subsysStringEnd) != subsystemname) {
0571 DQMStoreStatsSubsystem aSubsystem;
0572 subsystemname = path.substr(0, subsysStringEnd);
0573 aSubsystem.subsystemName_ = subsystemname;
0574 dqmStoreStatsTopLevel.push_back(aSubsystem);
0575 subfoldername = "";
0576 }
0577
0578
0579 if (path.size() == subsysStringEnd) {
0580
0581 DQMStoreStatsSubfolder aSubfolder;
0582 aSubfolder.subfolderName_ = subsystemname;
0583 dqmStoreStatsTopLevel.back().push_back(aSubfolder);
0584 }
0585
0586 else {
0587
0588 subfolderStringEnd = path.find('/', subsysStringEnd + 1);
0589 if (std::string::npos == subfolderStringEnd)
0590 subfolderStringEnd = path.size();
0591
0592
0593 if (path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1) != subfoldername) {
0594 subfoldername = path.substr(subsysStringEnd + 1, subfolderStringEnd - subsysStringEnd - 1);
0595 DQMStoreStatsSubfolder aSubfolder;
0596 aSubfolder.subfolderName_ = subfoldername;
0597 dqmStoreStatsTopLevel.back().push_back(aSubfolder);
0598 }
0599 }
0600
0601
0602 DQMStoreStatsSubfolder& currentSubfolder = dqmStoreStatsTopLevel.back().back();
0603
0604 switch (it->kind()) {
0605
0606 case MonitorElement::Kind::TH1F:
0607 currentSubfolder.AddBinsF(it->getNbinsX(), getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0));
0608 curr->update(it->getNbinsX(),
0609 getEmptyMetric(it->getTH1F()->GetArray(), it->getTH1F()->fN, 0, 0),
0610 it->getNbinsX() * sizeof(float));
0611 break;
0612 case MonitorElement::Kind::TH1S:
0613 currentSubfolder.AddBinsS(it->getNbinsX(), getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0));
0614 curr->update(it->getNbinsX(),
0615 getEmptyMetric(it->getTH1S()->GetArray(), it->getTH1S()->fN, 0, 0),
0616 it->getNbinsX() * sizeof(short));
0617 break;
0618 case MonitorElement::Kind::TH1D:
0619 currentSubfolder.AddBinsD(it->getNbinsX(), getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0));
0620 curr->update(it->getNbinsX(),
0621 getEmptyMetric(it->getTH1D()->GetArray(), it->getTH1D()->fN, 0, 0),
0622 it->getNbinsX() * sizeof(double));
0623 break;
0624 case MonitorElement::Kind::TH1I:
0625 currentSubfolder.AddBinsI(it->getNbinsX(), getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0));
0626 curr->update(it->getNbinsX(),
0627 getEmptyMetric(it->getTH1I()->GetArray(), it->getTH1I()->fN, 0, 0),
0628 it->getNbinsX() * sizeof(int));
0629 break;
0630 case MonitorElement::Kind::TPROFILE:
0631 currentSubfolder.AddBinsD(it->getNbinsX(),
0632 getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0));
0633 curr->update(it->getNbinsX(),
0634 getEmptyMetric(it->getTProfile()->GetArray(), it->getTProfile()->fN, 0, 0),
0635 it->getNbinsX() * sizeof(double));
0636 break;
0637
0638
0639 case MonitorElement::Kind::TH2F:
0640 currentSubfolder.AddBinsF(
0641 it->getNbinsX() * it->getNbinsY(),
0642 getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0643 curr->update(it->getNbinsX() * it->getNbinsY(),
0644 getEmptyMetric(it->getTH2F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0645 it->getNbinsX() * it->getNbinsY() * sizeof(float));
0646 break;
0647 case MonitorElement::Kind::TH2S:
0648 currentSubfolder.AddBinsS(
0649 it->getNbinsX() * it->getNbinsY(),
0650 getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0651 curr->update(it->getNbinsX() * it->getNbinsY(),
0652 getEmptyMetric(it->getTH2S()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0653 it->getNbinsX() * it->getNbinsY() * sizeof(short));
0654 break;
0655 case MonitorElement::Kind::TH2D:
0656 currentSubfolder.AddBinsD(
0657 it->getNbinsX() * it->getNbinsY(),
0658 getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0659 curr->update(it->getNbinsX() * it->getNbinsY(),
0660 getEmptyMetric(it->getTH2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0661 it->getNbinsX() * it->getNbinsY() * sizeof(double));
0662 break;
0663 case MonitorElement::Kind::TH2I:
0664 currentSubfolder.AddBinsI(
0665 it->getNbinsX() * it->getNbinsY(),
0666 getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0667 curr->update(it->getNbinsX() * it->getNbinsY(),
0668 getEmptyMetric(it->getTH2I()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0669 it->getNbinsX() * it->getNbinsY() * sizeof(int));
0670 break;
0671 case MonitorElement::Kind::TPROFILE2D:
0672 currentSubfolder.AddBinsD(
0673 it->getNbinsX() * it->getNbinsY(),
0674 getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0));
0675 curr->update(it->getNbinsX() * it->getNbinsY(),
0676 getEmptyMetric(it->getTProfile2D()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, 0),
0677 it->getNbinsX() * it->getNbinsY() * sizeof(double));
0678 break;
0679
0680
0681 case MonitorElement::Kind::TH3F:
0682 currentSubfolder.AddBinsF(
0683 it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
0684 getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2));
0685 curr->update(
0686 it->getNbinsX() * it->getNbinsY() * it->getNbinsZ(),
0687 getEmptyMetric(it->getTH3F()->GetArray(), it->getNbinsX() + 2, it->getNbinsY() + 2, it->getNbinsZ() + 2),
0688 it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float));
0689 break;
0690
0691 default: {
0692 }
0693
0694
0695
0696
0697
0698
0699 }
0700 }
0701
0702 if (mode == DQMStoreStats::considerAllME)
0703 calcIgProfDump(dbeFolder);
0704
0705
0706
0707 std::cout << endl;
0708 std::cout << "==========================================================================================="
0709 << std::endl;
0710 std::cout << "[DQMStoreStats::calcstats] -- Dumping stats results ";
0711 if (mode == DQMStoreStats::considerAllME)
0712 std::cout << "FOR ALL ME" << std::endl;
0713 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0714 std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
0715 std::cout << "==========================================================================================="
0716 << std::endl;
0717 std::cout << endl;
0718
0719 std::cout << "------------------------------------------------------------------------------------------"
0720 << std::endl;
0721 std::cout << "Configuration:" << std::endl;
0722 std::cout << "------------------------------------------------------------------------------------------"
0723 << std::endl;
0724 std::cout << " > running ";
0725 if (runonendrun_)
0726 std::cout << "on run end." << std::endl;
0727 if (runonendlumi_)
0728 std::cout << "on lumi end." << std::endl;
0729 if (runonendjob_)
0730 std::cout << "on job end." << std::endl;
0731 if (runineventloop_)
0732 std::cout << "in event loop." << std::endl;
0733 std::cout << " > pathNameMatch = \"" << pathnamematch_ << "\"" << std::endl;
0734 std::cout << std::endl;
0735
0736
0737 std::cout << "------------------------------------------------------------------------------------------"
0738 << std::endl;
0739 std::cout << "Top level folder tree:" << std::endl;
0740 std::cout << "------------------------------------------------------------------------------------------"
0741 << std::endl;
0742 for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
0743 std::cout << it0->subsystemName_ << " (subsystem)" << std::endl;
0744
0745 for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
0746 std::cout << " |--> " << it1->subfolderName_ << " (subfolder)" << std::endl;
0747 }
0748 }
0749
0750
0751
0752 unsigned int overallNHistograms = 0, overallNBins = 0, overallNBytes = 0;
0753
0754 std::cout << std::endl;
0755 std::cout << "------------------------------------------------------------------------------------------"
0756 << std::endl;
0757 std::cout << "Detailed ressource usage information ";
0758 if (mode == DQMStoreStats::considerAllME)
0759 std::cout << "FOR ALL ME" << std::endl;
0760 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0761 std::cout << "FOR LUMI PRODUCTS ONLY" << std::endl;
0762 std::cout << "------------------------------------------------------------------------------------------"
0763 << std::endl;
0764 std::cout << "subsystem/folder histograms bins Empty bins Empty/Total "
0765 "bins per MB kB per"
0766 << std::endl;
0767 std::cout << " (total) (total) (total) "
0768 "histogram (total) histogram "
0769 << std::endl;
0770 std::cout << "------------------------------------------------------------------------------------------"
0771 << std::endl;
0772 for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
0773 std::cout << it0->subsystemName_ << std::endl;
0774
0775 unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
0776
0777 for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
0778
0779 std::string thisSubfolderName(it1->subfolderName_);
0780 if (thisSubfolderName.size() > 30) {
0781 thisSubfolderName.resize(30);
0782 thisSubfolderName.replace(thisSubfolderName.size() - 3, 3, 3, '.');
0783 }
0784
0785 std::cout << " -> " << std::setw(30) << std::left << thisSubfolderName;
0786 std::cout << std::setw(14) << std::right << it1->totalHistos_;
0787 std::cout << std::setw(14) << std::right << it1->totalBins_;
0788 std::cout << std::setw(14) << std::right << it1->totalEmptyBins_;
0789 std::cout << std::setw(14) << std::right << std::setprecision(3)
0790 << (float)it1->totalEmptyBins_ / (float)it1->totalBins_;
0791
0792
0793 if (it1->totalHistos_) {
0794 std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalBins_ / float(it1->totalHistos_);
0795 } else
0796 std::cout << std::setw(14) << std::right << "-";
0797
0798 std::cout << std::setw(14) << std::right << std::setprecision(3) << it1->totalMemory_ / 1024. / 1024.;
0799
0800
0801 if (it1->totalHistos_) {
0802 std::cout << std::setw(14) << std::right << std::setprecision(3)
0803 << it1->totalMemory_ / 1024. / it1->totalHistos_;
0804 } else
0805 std::cout << std::setw(14) << std::right << "-";
0806
0807 std::cout << std::endl;
0808
0809
0810 nHistograms += it1->totalHistos_;
0811 nBins += it1->totalBins_;
0812 nEmptyBins += it1->totalEmptyBins_;
0813 nBytes += it1->totalMemory_;
0814 }
0815
0816 overallNHistograms += nHistograms;
0817 overallNBins += nBins;
0818 overallNBytes += nBytes;
0819
0820
0821 std::cout << " " << std::setw(30) << std::left << "SUBSYSTEM TOTAL";
0822 std::cout << std::setw(14) << std::right << nHistograms;
0823 std::cout << std::setw(14) << std::right << nBins;
0824 std::cout << std::setw(14) << std::right << nEmptyBins;
0825 std::cout << std::setw(14) << std::right << (float)nEmptyBins / (float)nBins;
0826 std::cout << std::setw(14) << std::right << std::setprecision(3) << nBins / float(nHistograms);
0827 std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / 1000.;
0828 std::cout << std::setw(14) << std::right << std::setprecision(3) << nBytes / 1024. / nHistograms;
0829 std::cout << std::endl;
0830
0831 std::cout << ".........................................................................................."
0832 << std::endl;
0833 }
0834
0835
0836 std::cout << std::endl;
0837 std::cout << "------------------------------------------------------------------------------------------"
0838 << std::endl;
0839 std::cout << "Grand total ";
0840 if (mode == DQMStoreStats::considerAllME)
0841 std::cout << "FOR ALL ME:" << std::endl;
0842 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0843 std::cout << "FOR LUMI PRODUCTS ONLY:" << std::endl;
0844 std::cout << "------------------------------------------------------------------------------------------"
0845 << std::endl;
0846 std::cout << "Number of subsystems: " << dqmStoreStatsTopLevel.size() << std::endl;
0847 std::cout << "Total number of histograms: " << overallNHistograms << " with: " << overallNBins << " bins alltogether"
0848 << std::endl;
0849 std::cout << "Total memory occupied by histograms (excl. overhead): " << overallNBytes / 1024. / 1000. << " MB"
0850 << std::endl;
0851
0852 std::cout << endl;
0853 std::cout << "==========================================================================================="
0854 << std::endl;
0855 std::cout << "[DQMStoreStats::calcstats] -- End of output ";
0856 if (mode == DQMStoreStats::considerAllME)
0857 std::cout << "FOR ALL ME." << std::endl;
0858 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0859 std::cout << "FOR LUMI PRODUCTS ONLY." << std::endl;
0860 std::cout << "==========================================================================================="
0861 << std::endl;
0862 std::cout << endl;
0863
0864
0865
0866
0867 if (dumpToFWJR_) {
0868 edm::Service<edm::JobReport> jr;
0869
0870 if (!jr.isAvailable())
0871 return 0;
0872
0873 std::map<std::string, std::string> jrInfo;
0874
0875 jrInfo["Source"] = "DQMServices/Components";
0876 jrInfo["FileClass"] = "DQMStoreStats";
0877 if (runonendrun_)
0878 jrInfo["DumpType"] = "EndRun";
0879 if (runonendlumi_)
0880 jrInfo["DumpType"] = "EndLumi";
0881 if (runonendjob_)
0882 jrInfo["DumpType"] = "EndJob";
0883 if (runineventloop_)
0884 jrInfo["DumpType"] = "EventLoop";
0885 if (mode == DQMStoreStats::considerAllME)
0886 jrInfo["Type"] = "RunProduct";
0887 else if (mode == DQMStoreStats::considerOnlyLumiProductME)
0888 jrInfo["Type"] = "LumiProduct";
0889
0890 jrInfo["pathNameMatch"] = pathnamematch_;
0891
0892 for (auto it0 = dqmStoreStatsTopLevel.begin(); it0 < dqmStoreStatsTopLevel.end(); ++it0) {
0893 unsigned int nHistograms = 0, nBins = 0, nEmptyBins = 0, nBytes = 0;
0894 for (auto it1 = it0->begin(); it1 < it0->end(); ++it1) {
0895
0896 nHistograms += it1->totalHistos_;
0897 nBins += it1->totalBins_;
0898 nEmptyBins += it1->totalEmptyBins_;
0899 nBytes += it1->totalMemory_;
0900 }
0901 std::stringstream iss("");
0902 iss << nHistograms;
0903 jrInfo[it0->subsystemName_ + std::string("_h")] = iss.str();
0904 iss.str("");
0905 iss << nBins;
0906 jrInfo[it0->subsystemName_ + std::string("_b")] = iss.str();
0907 iss.str("");
0908 iss << nEmptyBins;
0909 jrInfo[it0->subsystemName_ + std::string("_be")] = iss.str();
0910 iss.str("");
0911 iss << ((float)nEmptyBins / (float)nBins);
0912 jrInfo[it0->subsystemName_ + std::string("_fbe")] = iss.str();
0913 iss.str("");
0914 iss << ((float)nBins / (float)nHistograms);
0915 jrInfo[it0->subsystemName_ + std::string("_b_h")] = iss.str();
0916 iss.str("");
0917 iss << nBytes / 1024. / 1024.;
0918 jrInfo[it0->subsystemName_ + std::string("_MB")] = iss.str();
0919 iss.str("");
0920 iss << nBytes / 1024. / nHistograms;
0921 jrInfo[it0->subsystemName_ + std::string("_Kb_h")] = iss.str();
0922 }
0923 jr->reportAnalysisFile("DQMStatsReport", jrInfo);
0924 }
0925
0926 return 0;
0927 }
0928
0929
0930
0931
0932 void DQMStoreStats::dumpMemoryProfile() {
0933 std::cout << std::endl;
0934 std::cout << "------------------------------------------------------------------------------------------"
0935 << std::endl;
0936 std::cout << "Memory profile:" << std::endl;
0937 std::cout << "------------------------------------------------------------------------------------------"
0938 << std::endl;
0939
0940
0941 std::pair<time_t, unsigned int> maxItem(0, 0);
0942 for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
0943 if (it->second > maxItem.second) {
0944 maxItem = *it;
0945 }
0946 }
0947
0948 std::stringstream rootOutputFileName;
0949 rootOutputFileName << "dqmStoreStats_memProfile_" << getpid() << ".root";
0950
0951
0952 if (dumpMemHistory_ && isOpenProcFileSuccessful_) {
0953 TFile outputFile(rootOutputFileName.str().c_str(), "RECREATE");
0954
0955 int aTime;
0956 float aMb;
0957
0958 TTree memHistoryTree("dqmstorestats_memhistory", "memory history");
0959 memHistoryTree.Branch("seconds", &aTime, "seconds/I");
0960 memHistoryTree.Branch("megabytes", &aMb, "megabytes/F");
0961 for (auto it = memoryHistoryVector_.begin(); it < memoryHistoryVector_.end(); ++it) {
0962 aTime = it->first - startingTime_;
0963 aMb = it->second / 1000.;
0964 memHistoryTree.Fill();
0965 }
0966
0967 outputFile.Write();
0968 outputFile.Close();
0969 }
0970
0971 std::cout << "Approx. maximum total virtual memory size of job: ";
0972 if (isOpenProcFileSuccessful_ && !memoryHistoryVector_.empty()) {
0973 std::cout << maxItem.second / 1000. << " MB (reached " << maxItem.first - startingTime_
0974 << " sec. after constructor called)," << std::endl;
0975 std::cout << " memory history written to: " << rootOutputFileName.str() << " (" << memoryHistoryVector_.size()
0976 << " samples)" << std::endl;
0977 } else {
0978 std::cout << "(could not be determined)" << std::endl;
0979 }
0980
0981 std::cout << std::endl << std::endl;
0982 }
0983
0984
0985
0986
0987 void DQMStoreStats::print() {
0988
0989 std::cout << " ---------- " << subsystem_ << " ---------- " << std::endl;
0990 std::cout << " " << subfolder_ << ": ";
0991 std::cout << nmesubsys_ << " histograms with " << nbinssubsys_ << " bins. ";
0992 if (nmesubsys_ > 0)
0993 std::cout << nbinssubsys_ / nmesubsys_ << " bins/histogram ";
0994 std::cout << std::endl;
0995 std::cout << " Largest histogram: " << maxbinsmesubsys_ << " with " << maxbinssubsys_ << " bins." << std::endl;
0996 }
0997
0998
0999
1000
1001 std::pair<unsigned int, unsigned int> DQMStoreStats::readMemoryEntry() const {
1002
1003 if (isOpenProcFileSuccessful_) {
1004 std::ifstream procFile(procFileName_.str().c_str(), ios::in);
1005
1006 std::string readBuffer("");
1007 unsigned int memSize = 0;
1008
1009
1010 while (!procFile.eof()) {
1011 procFile >> readBuffer;
1012 if (std::string("VmSize:") == readBuffer) {
1013 procFile >> memSize;
1014 break;
1015 }
1016 }
1017
1018 procFile.close();
1019 return std::pair<time_t, unsigned int>(time(nullptr), memSize);
1020 }
1021
1022 return std::pair<time_t, unsigned int>(0, 0);
1023 }
1024
1025
1026
1027
1028 void DQMStoreStats::beginJob() {
1029
1030 dbe_ = Service<DQMStore>().operator->();
1031
1032
1033 procFileName_ << "/proc/" << getpid() << "/status";
1034
1035
1036 std::ifstream procFile(procFileName_.str().c_str(), ios::in);
1037
1038 if (procFile.good()) {
1039 isOpenProcFileSuccessful_ = true;
1040 } else {
1041 std::cerr << " [DQMStoreStats::beginJob] ** WARNING: could not open file: " << procFileName_.str() << std::endl;
1042 std::cerr << " Total memory profile will not be available." << std::endl;
1043 isOpenProcFileSuccessful_ = false;
1044 }
1045
1046 procFile.close();
1047 }
1048
1049
1050
1051
1052 void DQMStoreStats::beginRun(const edm::Run& r, const EventSetup& context) {}
1053
1054
1055
1056
1057 void DQMStoreStats::analyze(const Event& iEvent, const EventSetup& iSetup) {
1058
1059 memoryHistoryVector_.emplace_back(readMemoryEntry());
1060
1061 if (runineventloop_) {
1062 calcstats(DQMStoreStats::considerAllME);
1063 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1064 dumpMemoryProfile();
1065 }
1066 }
1067
1068
1069
1070
1071 void DQMStoreStats::endLuminosityBlock(const LuminosityBlock& lumiSeg, const EventSetup& context) {
1072 if (runonendlumi_) {
1073 calcstats(DQMStoreStats::considerAllME);
1074 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1075 dumpMemoryProfile();
1076 }
1077 }
1078
1079
1080
1081
1082 void DQMStoreStats::beginLuminosityBlock(const edm::LuminosityBlock& lumiSeg, const edm::EventSetup& context) {}
1083
1084
1085
1086
1087 void DQMStoreStats::endRun(const Run& r, const EventSetup& context) {
1088 if (runonendrun_) {
1089 calcstats(DQMStoreStats::considerAllME);
1090 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1091 dumpMemoryProfile();
1092 }
1093 }
1094
1095
1096
1097
1098 void DQMStoreStats::endJob() {
1099 if (runonendjob_) {
1100 calcstats(DQMStoreStats::considerAllME);
1101 calcstats(DQMStoreStats::considerOnlyLumiProductME);
1102 dumpMemoryProfile();
1103 }
1104 }
1105
1106 #include "FWCore/PluginManager/interface/ModuleDef.h"
1107 #include "FWCore/Framework/interface/MakerMacros.h"
1108 DEFINE_FWK_MODULE(DQMStoreStats);
