Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-12-13 23:50:05

0001 // silence deprecation warnings for the DQMStore itself.
0002 #define DQM_DEPRECATED
0003 #include "DQMServices/Core/interface/DQMStore.h"
0004 #include "DQMServices/Core/interface/LegacyIOHelper.h"
0005 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0006 #include "FWCore/ServiceRegistry/interface/GlobalContext.h"
0007 #include <string>
0008 #include <regex>
0009 #include <csignal>
0010 
0011 #include <execinfo.h>
0012 #include <cxxabi.h>
0013 
0014 namespace dqm::implementation {
0015 
0016   // list of acceptable characters for ME path names, in order to be able to upload to the CMS DQM GUI
0017   // See https://github.com/cms-DQM/dqmgui_prod/blob/af0a388e8f57c60e51111585d298aeeea943367f/src/cpp/DQM/DQMStore.cc#L56
0018   static const std::string s_safe = "/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-+=_()# ";
0019 
0020   std::string NavigatorBase::pwd() {
0021     if (cwd_.empty()) {
0022       return "";
0023     } else {
0024       // strip trailing slash.
0025       // This is inefficient and error prone (callers need to do the same
0026       // branching to re-add the "/"!) but some legacy code expects it like
0027       // that and is to complicated to change.
0028       assert(cwd_[cwd_.size() - 1] == '/');
0029       auto pwd = cwd_.substr(0, cwd_.size() - 1);
0030       return pwd;
0031     }
0032   }
0033   void NavigatorBase::cd() { setCurrentFolder(""); }
0034   void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(dir); }
0035   void NavigatorBase::goUp() { cd(cwd_ + ".."); }
0036   void NavigatorBase::setCurrentFolder(std::string const& fullpath) {
0037     MonitorElementData::Path path;
0038     path.set(fullpath, MonitorElementData::Path::Type::DIR);
0039     assert(this);
0040     cwd_ = path.getDirname();
0041   }
0042 
0043   IBooker::IBooker(DQMStore* store) {
0044     store_ = store;
0045     scope_ = MonitorElementData::Scope::JOB;
0046   }
0047 
0048   IBooker::~IBooker() {}
0049 
0050   MonitorElementData::Scope IBooker::setScope(MonitorElementData::Scope newscope) {
0051     auto oldscope = scope_;
0052     scope_ = newscope;
0053     return oldscope;
0054   }
0055   uint64_t IBooker::setModuleID(uint64_t moduleID) {
0056     auto oldid = moduleID_;
0057     moduleID_ = moduleID;
0058     return oldid;
0059   }
0060 
0061   edm::LuminosityBlockID IBooker::setRunLumi(edm::LuminosityBlockID runlumi) {
0062     auto oldrunlumi = runlumi_;
0063     runlumi_ = runlumi;
0064     return oldrunlumi;
0065   }
0066 
0067   MonitorElement* IBooker::bookME(TString const& name,
0068                                   MonitorElementData::Kind kind,
0069                                   std::function<TH1*()> makeobject,
0070                                   bool forceReplace /* = false */) {
0071     MonitorElementData::Path path;
0072     std::string fullpath = cwd_ + std::string(name.View());
0073 
0074     auto pathToCheck{fullpath};
0075     std::string limiter{".root:/"};          // this indicates that all the substring before is a file name
0076     size_t pos = pathToCheck.find(limiter);  //find location of limiter
0077     //delete everything prior to location found as it might contain illegal chars
0078     pathToCheck.erase(0, pos + limiter.size());
0079 
0080     if (pathToCheck.find_first_not_of(s_safe) != std::string::npos) {
0081       throw cms::Exception("BadMonitorElementPathName")
0082           << " Monitor element path name: '" << pathToCheck.c_str() << "' uses unacceptable characters."
0083           << "\n Acceptable characters are: " << s_safe.c_str();
0084     }
0085 
0086     path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME);
0087 
0088     // We should check if there is a local ME for this module and name already.
0089     // However, it is easier to do that in putME().
0090 
0091     MonitorElement* me = store_->findME(path);
0092     store_->printTrace("Booking " + std::string(name) + (me ? " (existing)" : " (new)"));
0093 
0094     if (me == nullptr) {
0095       // no existing global ME found. We need to instantiate one, and put it
0096       // into the DQMStore. This will typically be a prototype, unless run and
0097       // lumi are set and we proces a legacy booking call.
0098       TH1* th1 = makeobject();
0099       MonitorElementData medata;
0100       medata.key_.path_ = path;
0101       medata.key_.kind_ = kind;
0102       medata.key_.scope_ = this->scope_;
0103 
0104       // will be (0,0) ( = prototype) in the common case.
0105       // This branching is for harvesting, where we have run/lumi in the booker.
0106       if (this->scope_ == MonitorElementData::Scope::JOB) {
0107         medata.key_.id_ = edm::LuminosityBlockID();
0108       } else if (this->scope_ == MonitorElementData::Scope::RUN) {
0109         medata.key_.id_ = edm::LuminosityBlockID(this->runlumi_.run(), 0);
0110       } else if (this->scope_ == MonitorElementData::Scope::LUMI) {
0111         // In the messy case of legacy-booking a LUMI ME in beginRun (or
0112         // similar), where we don't have a valid lumi number yet, make sure to
0113         // book a prototype instead.
0114         if (this->runlumi_.run() != 0 && this->runlumi_.luminosityBlock() != 0) {
0115           medata.key_.id_ = this->runlumi_;
0116         } else {
0117           medata.key_.id_ = edm::LuminosityBlockID();
0118         }
0119       } else {
0120         assert(!"Illegal scope");
0121       }
0122 
0123       medata.value_.object_ = std::unique_ptr<TH1>(th1);
0124       MonitorElement* me_ptr = new MonitorElement(std::move(medata));
0125       me = store_->putME(me_ptr);
0126     } else {
0127       if (forceReplace) {
0128         TH1* th1 = makeobject();
0129         assert(th1);
0130         store_->debugTrackME("bookME (forceReplace)", nullptr, me);
0131         // surgically replace Histogram
0132         me->switchObject(std::unique_ptr<TH1>(th1));
0133       }
0134     }
0135 
0136     // me now points to a global ME owned by the DQMStore.
0137     assert(me);
0138 
0139     // each booking call returns a unique "local" ME, which the DQMStore keeps
0140     // in a container associated with the module (and potentially run, for
0141     // DQMGlobalEDAnalyzer). This will later be update to point to different
0142     // MEData (kept in a global ME) as needed.
0143     // putME creates the local ME object as needed.
0144     auto localme = store_->putME(me, this->moduleID_);
0145     // me now points to a local ME owned by the DQMStore.
0146     assert(localme);
0147 
0148     if (this->moduleID_ == 0) {
0149       // this is a legacy/global/harvesting booking. In this case, we return
0150       // the global directly. It is not advisable to hold this pointer, as we
0151       // may delete the global ME later, but we promise to keep it valid for
0152       // the entire job if there are no concurrent runs/lumis. (see
0153       // assertLegacySafe option).
0154       // We still created a local ME, so we can drive the lumi-changing for
0155       // legacy modules in watchPreGlobalBeginLumi.
0156       store_->debugTrackME("bookME (legacy)", localme, me);
0157       return me;
0158     } else {
0159       // the normal case.
0160       store_->debugTrackME("bookME (normal)", localme, me);
0161       return localme;
0162     }
0163   }
0164 
0165   MonitorElement* DQMStore::putME(MonitorElement* me) {
0166     auto lock = std::scoped_lock(this->booking_mutex_);
0167     assert(me);
0168     auto existing_new = globalMEs_[me->getRunLumi()].insert(me);
0169     if (existing_new.second == true) {
0170       // successfully inserted, return new object
0171       debugTrackME("putME (global)", nullptr, me);
0172       return me;
0173     } else {
0174       // already present, return old object
0175       delete me;
0176       assert(!"Currently, this should never happen.");
0177       return *(existing_new.first);
0178     }
0179   }
0180 
0181   MonitorElement* DQMStore::putME(MonitorElement* me, uint64_t moduleID) {
0182     auto lock = std::scoped_lock(this->booking_mutex_);
0183     assert(me);
0184     auto& localmes = localMEs_[moduleID];
0185     auto existing = localmes.find(me);
0186     if (existing == localmes.end()) {
0187       // insert new local ME
0188       MonitorElement* local_me = new MonitorElement(me);
0189       auto existing_new = localmes.insert(local_me);
0190       // successfully inserted, return new object
0191       assert(existing_new.second == true);  // insert successful
0192       debugTrackME("putME (local, new)", local_me, me);
0193       return local_me;
0194     } else {
0195       // already present, return old object
0196       auto local_me = *existing;
0197       edm::LogInfo("DQMStore") << "ME " << me->getFullname() << " booked twice in the same module.";
0198       // the existing local ME might not have data attached (e.g. in 2nd run)
0199       // in that case, we attach the global ME provided by booking above.
0200       // This may be a prototype or of a random run/lumi, but it ensures that
0201       // even LUMI histos are always valid after booking (as we promise for
0202       // legacy modules -- for sequential runs/lumis, there is only ever one
0203       // global ME, and the local one points to it).
0204       if (!local_me->isValid()) {
0205         local_me->switchData(me);
0206       }
0207       debugTrackME("putME (local, existing)", local_me, me);
0208       return local_me;
0209     }
0210   }
0211 
0212   template <typename MELIKE>
0213   MonitorElement* DQMStore::findME(MELIKE const& path) {
0214     auto lock = std::scoped_lock(this->booking_mutex_);
0215     for (auto& [runlumi, meset] : this->globalMEs_) {
0216       auto it = meset.find(path);
0217       if (it != meset.end()) {
0218         debugTrackME("findME (found)", nullptr, *it);
0219         // no guarantee on which ME we return here -- only that clone'ing this
0220         // would give a valid ME for that path.
0221         return *it;
0222       }
0223     }
0224     return nullptr;
0225   }
0226 
0227   void DQMStore::printTrace(std::string const& message) {
0228     if (verbose_ < 3)
0229       return;
0230     edm::LogWarning("DQMStoreBooking").log([&](auto& logger) {
0231       std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"};
0232       std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"};
0233 
0234       void* array[10];
0235       size_t size;
0236       char** strings;
0237       int demangle_status = 0;
0238       std::vector<std::string> clean_trace;
0239 
0240       // glibc/libgcc backtrace functionality, declared in execinfo.h.
0241       size = backtrace(array, 10);
0242       strings = backtrace_symbols(array, size);
0243 
0244       size_t level = 1;
0245       char* demangled = nullptr;
0246       for (; level < size; ++level) {
0247         std::cmatch match;
0248         bool ok = std::regex_match(strings[level], match, s_rxtrace);
0249 
0250         if (!ok) {
0251           edm::LogWarning("DQMStoreBacktrace") << "failed match" << level << strings[level];
0252           continue;
0253         }
0254 
0255         if (match[2].length() == 0) {
0256           // no symbol, ignore.
0257           continue;
0258         }
0259 
0260         // demangle name to human readable form
0261         demangled = abi::__cxa_demangle(std::string(match[2]).c_str(), nullptr, nullptr, &demangle_status);
0262         if (!demangled || demangle_status != 0) {
0263           edm::LogWarning("DQMStoreBacktrace") << "failed demangle! status " << demangle_status << " on " << match[2];
0264           continue;
0265         }
0266 
0267         if (std::regex_match(demangled, s_rxself)) {
0268           // ignore framework/internal methods
0269           free(demangled);
0270           demangled = nullptr;
0271           continue;
0272         } else {
0273           // keep the demangled name and the address.
0274           // The address can be resolved to a line number in gdb attached to
0275           // the process, using `list *0x<addr>`, but it can only be done in
0276           // the running process and we can"t easily do it in this code.
0277           clean_trace.push_back(std::string(demangled) + std::string(match[3]));
0278           free(demangled);
0279           demangled = nullptr;
0280         }
0281       }
0282 
0283       if (!clean_trace.empty()) {
0284         logger << message << " at ";
0285         for (auto const& s : clean_trace) {
0286           logger << s << "; ";
0287         }
0288       } else {
0289         logger << message << " : failed to collect stack trace.";
0290       }
0291 
0292       free(strings);
0293     });
0294   }
0295 
0296   void DQMStore::debugTrackME(const char* message, MonitorElement* me_local, MonitorElement* me_global) const {
0297     const char* scopename[] = {"INVALID", "JOB", "RUN", "LUMI"};
0298     if (!this->trackME_.empty() && (me_local || me_global)) {
0299       std::string name = me_global ? me_global->getFullname() : me_local->getFullname();
0300       if (name.find(this->trackME_) != std::string::npos) {
0301         edm::LogWarning("DQMStoreTrackME").log([&](auto& logger) {
0302           logger << message << " for " << name << "(" << me_local << "," << me_global << ")";
0303           auto writeme = [&](MonitorElement* me) {
0304             if (me->isValid()) {
0305               logger << " " << me->getRunLumi() << " scope " << scopename[me->getScope()];
0306               if (me->kind() >= MonitorElement::Kind::TH1F) {
0307                 logger << " entries " << me->getEntries();
0308               } else if (me->kind() == MonitorElement::Kind::STRING) {
0309                 logger << " value " << me->getStringValue();
0310               } else if (me->kind() == MonitorElement::Kind::REAL) {
0311                 logger << " value " << me->getFloatValue();
0312               } else if (me->kind() == MonitorElement::Kind::INT) {
0313                 logger << " value " << me->getIntValue();
0314               }
0315             } else {
0316               logger << " (invalid)";
0317             }
0318           };
0319           if (me_local) {
0320             logger << "  local:";
0321             writeme(me_local);
0322           }
0323           if (me_global) {
0324             logger << "  global:";
0325             writeme(me_global);
0326           }
0327         });
0328         // A breakpoint can be useful here.
0329         //std::raise(SIGINT);
0330       }
0331     }
0332   }
0333 
0334   MonitorElement* DQMStore::findOrRecycle(MonitorElementData::Key const& key) {
0335     // This is specifically for DQMRootSource, or other input modules. These
0336     // are special in that they use the legacy interface (no moduleID, no local
0337     // MEs) but need to be able to handle concurrent lumisections correctly.
0338     // The logic is very similar to that in enterLumi; this is enterLumi for
0339     // Input Modules.
0340     auto lock = std::scoped_lock(this->booking_mutex_);
0341     auto existing = this->get(key);
0342     if (existing) {
0343       // exactly matching ME found, needs merging with the new data.
0344       debugTrackME("findOrRecycle (found)", nullptr, existing);
0345       return existing;
0346     }  // else
0347 
0348     // this is where we'd expect the ME.
0349     auto& targetset = this->globalMEs_[key.id_];
0350     // this is where we can get MEs to reuse.
0351     auto& prototypes = this->globalMEs_[edm::LuminosityBlockID()];
0352 
0353     auto proto = prototypes.find(key.path_);
0354     if (proto != prototypes.end()) {
0355       MonitorElement* oldme = *proto;
0356       assert(oldme->getScope() == key.scope_);
0357       prototypes.erase(proto);
0358       auto medata = oldme->release();  // destroy the ME, get its data.
0359       // in this situation, nobody should be filling the ME concurrently.
0360       medata->data_.key_.id_ = key.id_;
0361       // We reuse the ME object here, even if we don't have to. This ensures
0362       // that when running single-threaded without concurrent lumis/runs,
0363       // the global MEs will also live forever and allow legacy usages.
0364       oldme->switchData(medata);
0365       auto result = targetset.insert(oldme);
0366       assert(result.second);       // was new insertion
0367       auto newme = *result.first;  // iterator to new ME
0368       assert(oldme == newme);      // recycling!
0369       // newme is reset and ready to accept data.
0370       debugTrackME("findOrRecycle (recycled)", nullptr, newme);
0371       return newme;
0372     }  // else
0373 
0374     return nullptr;
0375   }
0376 
0377   void DQMStore::initLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) {
0378     // Call initLumi for all modules, as a global operation.
0379     auto lock = std::scoped_lock(this->booking_mutex_);
0380     for (auto& kv : this->localMEs_) {
0381       initLumi(run, lumi, kv.first);
0382     }
0383   }
0384 
0385   void DQMStore::initLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) {
0386     // Make sure global MEs for the run/lumi exist (depending on scope)
0387 
0388     auto lock = std::scoped_lock(this->booking_mutex_);
0389 
0390     // these are the MEs we need to update.
0391     auto& localset = this->localMEs_[moduleID];
0392     // this is where they need to point to.
0393     // This could be a per-run or per-lumi set (depending on lumi == 0)
0394     auto& targetset = this->globalMEs_[edm::LuminosityBlockID(run, lumi)];
0395     // this is where we can get MEs to reuse.
0396     auto& prototypes = this->globalMEs_[edm::LuminosityBlockID()];
0397 
0398     auto checkScope = [run, lumi](MonitorElementData::Scope scope) {
0399       if (scope == MonitorElementData::Scope::JOB) {
0400         return (run == 0 && lumi == 0);
0401       } else if (scope == MonitorElementData::Scope::RUN) {
0402         return (run != 0 && lumi == 0);
0403       } else if (scope == MonitorElementData::Scope::LUMI) {
0404         return (lumi != 0);
0405       }
0406       assert(!"Impossible Scope.");
0407       return false;
0408     };
0409 
0410     for (MonitorElement* me : localset) {
0411       auto target = targetset.find(me);  // lookup by path, thanks to MEComparison
0412       if (target != targetset.end()) {
0413         // we already have a ME, just use it!
0414         debugTrackME("initLumi (existing)", nullptr, *target);
0415       } else {
0416         // look for a prototype to reuse.
0417         auto proto = prototypes.find(me);
0418         if (proto != prototypes.end()) {
0419           // first, check if this ME needs updating at all. We can only check
0420           // the scope once we have an actual global ME instance, the local ME
0421           // might not have any data attached!
0422           if (checkScope((*proto)->getScope()) == false) {
0423             continue;
0424           }  // else
0425           // reuse that.
0426           MonitorElement* oldme = *proto;
0427           prototypes.erase(proto);
0428           auto medata = oldme->release();  // destroy the ME, get its data.
0429           // in this situation, nobody should be filling the ME concurrently.
0430           medata->data_.key_.id_ = edm::LuminosityBlockID(run, lumi);
0431           // We reuse the ME object here, even if we don't have to. This ensures
0432           // that when running single-threaded without concurrent lumis/runs,
0433           // the global MEs will also live forever and allow legacy usages.
0434           oldme->switchData(medata);
0435           auto result = targetset.insert(oldme);
0436           assert(result.second);  // was new insertion
0437           target = result.first;  // iterator to new ME
0438           debugTrackME("initLumi (reused)", nullptr, *target);
0439         } else {
0440           // no prototype available. That means we have concurrent Lumis/Runs,
0441           // and need to make a clone now.
0442           auto anyme = this->findME(me);
0443           assert(anyme || !"local ME without any global ME!");
0444           if (checkScope(anyme->getScope()) == false) {
0445             continue;
0446           }  // else
0447 
0448           // whenever we clone global MEs, it is no longer safe to hold
0449           // pointers to them.
0450           assert(!assertLegacySafe_);
0451 
0452           MonitorElementData newdata = anyme->cloneMEData();
0453           newdata.key_.id_ = edm::LuminosityBlockID(run, lumi);
0454           auto newme = new MonitorElement(std::move(newdata));
0455           newme->Reset();  // we cloned a ME in use, not an empty prototype
0456           auto result = targetset.insert(newme);
0457           assert(result.second);  // was new insertion
0458           target = result.first;  // iterator to new ME
0459           debugTrackME("initLumi (allocated)", nullptr, *target);
0460         }
0461       }
0462     }
0463   }
0464 
0465   void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) {
0466     // point the local MEs for this module to these global MEs.
0467 
0468     // This needs to happen before we can use the global MEs for this run/lumi here.
0469     // We could do it lazyly here, or eagerly globally in global begin lumi.
0470     //initLumi(run, lumi, moduleID);
0471 
0472     auto lock = std::scoped_lock(this->booking_mutex_);
0473 
0474     // these are the MEs we need to update.
0475     auto& localset = this->localMEs_[moduleID];
0476     // this is where they need to point to.
0477     auto& targetset = this->globalMEs_[edm::LuminosityBlockID(run, lumi)];
0478 
0479     // only for a sanity check
0480     auto checkScope = [run, lumi](MonitorElementData::Scope scope) {
0481       if (scope == MonitorElementData::Scope::JOB) {
0482         return (run == 0 && lumi == 0);
0483       } else if (scope == MonitorElementData::Scope::RUN) {
0484         return (run != 0 && lumi == 0);
0485       } else if (scope == MonitorElementData::Scope::LUMI) {
0486         return (lumi != 0);
0487       }
0488       assert(!"Impossible Scope.");
0489       return false;
0490     };
0491 
0492     for (MonitorElement* me : localset) {
0493       auto target = targetset.find(me);  // lookup by path, thanks to MEComparison
0494       if (target == targetset.end()) {
0495         auto anyme = this->findME(me);
0496         debugTrackME("enterLumi (nothingtodo)", me, nullptr);
0497         assert(anyme && checkScope(anyme->getScope()) == false);
0498         continue;
0499       }
0500       assert(target != targetset.end());  // initLumi should have taken care of this.
0501       // now we have the proper global ME in the right place, point the local there.
0502       // This is only safe if the name is exactly the same -- else it might corrupt
0503       // the tree structure of the set!
0504       me->switchData(*target);
0505       debugTrackME("enterLumi (switchdata)", me, *target);
0506     }
0507   }
0508 
0509   void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) {
0510     // here, we remove the pointers in the local MEs. No deletion or recycling
0511     // yet -- this has to happen after the output module had a chance to do its
0512     // work. We just leave the global MEs where they are. This is purely an
0513     // accounting step, the cleanup code has to check that nobody is using the
0514     // ME any more, and here we make sure that is the case.
0515 
0516     auto lock = std::scoped_lock(this->booking_mutex_);
0517 
0518     // these are the MEs we need to update.
0519     auto& localset = this->localMEs_[moduleID];
0520 
0521     auto checkScope = [run, lumi](MonitorElementData::Scope scope) {
0522       if (scope == MonitorElementData::Scope::JOB) {
0523         return (run == 0 && lumi == 0);
0524       } else if (scope == MonitorElementData::Scope::RUN) {
0525         return (run != 0 && lumi == 0);
0526       } else if (scope == MonitorElementData::Scope::LUMI) {
0527         return (lumi != 0);
0528       }
0529       assert(!"Impossible Scope.");
0530       return false;
0531     };
0532 
0533     for (MonitorElement* me : localset) {
0534       // we have to be very careful with the ME here, it might not be backed by data at all.
0535       if (me->isValid() && checkScope(me->getScope()) == true) {
0536         // if we left the scope, simply release the data.
0537         debugTrackME("leaveLumi (release)", me, nullptr);
0538         me->release();
0539       }
0540     }
0541   }
0542 
0543   void DQMStore::cleanupLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) {
0544     // now, we are done with the lumi, no modules have any work to do on these
0545     // MEs, and the output modules have saved this lumi/run. Remove/recycle
0546     // the MEs here.
0547 
0548     auto lock = std::scoped_lock(this->booking_mutex_);
0549 
0550     // in case of end-job cleanup we need different logic because of the
0551     // prototype set.
0552     assert(run != 0 || lumi != 0);
0553     auto& prototypes = this->globalMEs_[edm::LuminosityBlockID()];
0554 
0555     // these are the MEs we need to get rid of...
0556     auto meset = std::set<MonitorElement*, MonitorElement::MEComparison>();
0557     // ... we take them out first.
0558     meset.swap(this->globalMEs_[edm::LuminosityBlockID(run, lumi)]);
0559 
0560     // temporary buffer for the MEs to recycle, we must not change the key
0561     // while they are in a set.
0562     auto torecycle = std::vector<MonitorElement*>();
0563 
0564     // here, this is only a sanity check and not functionally needed.
0565     auto checkScope = [run, lumi](MonitorElementData::Scope scope) {
0566       if (scope == MonitorElementData::Scope::JOB) {
0567         assert(run == 0 && lumi == 0);
0568       } else if (scope == MonitorElementData::Scope::RUN) {
0569         assert(run != 0 && lumi == 0);
0570       } else if (scope == MonitorElementData::Scope::LUMI) {
0571         assert(lumi != 0);
0572       } else {
0573         assert(!"Impossible Scope.");
0574       }
0575     };
0576 
0577     for (MonitorElement* me : meset) {
0578       assert(me->isValid());       // global MEs should always be valid.
0579       checkScope(me->getScope());  // we should only see MEs of one scope here.
0580       auto other = this->findME(me);
0581       if (other) {
0582         // we still have a global one, so we can just remove this.
0583         debugTrackME("cleanupLumi (delete)", nullptr, me);
0584         delete me;
0585       } else {
0586         // we will modify the ME, so it needs to be out of the set.
0587         // use a temporary vector to be save.
0588         debugTrackME("cleanupLumi (recycle)", nullptr, me);
0589         torecycle.push_back(me);
0590       }
0591     }
0592 
0593     meset.clear();
0594 
0595     for (MonitorElement* me : torecycle) {
0596       auto medata = me->release();                        // destroy the ME, get its data.
0597       medata->data_.key_.id_ = edm::LuminosityBlockID();  // prototype
0598       // We reuse the ME object here, even if we don't have to. This ensures
0599       // that when running single-threaded without concurrent lumis/runs,
0600       // the global MEs will also live forever and allow legacy usages.
0601       me->switchData(medata);
0602       // reset here (not later) to still catch random legacy fill calls.
0603       me->Reset();
0604       auto result = prototypes.insert(me);
0605       assert(result.second);  // was new insertion, else findME should succeed
0606       debugTrackME("cleanupLumi (reset)", nullptr, me);
0607     }
0608   }
0609 
0610   std::vector<dqm::harvesting::MonitorElement*> IGetter::getContents(std::string const& pathname) const {
0611     auto lock = std::scoped_lock(store_->booking_mutex_);
0612     std::vector<MonitorElement*> out;
0613     MonitorElementData::Path path;
0614     path.set(pathname, MonitorElementData::Path::Type::DIR);
0615     for (auto& [runlumi, meset] : store_->globalMEs_) {
0616       auto it = meset.lower_bound(path);
0617       while (it != meset.end() && (*it)->getPathname() == path.getDirname()) {
0618         store_->debugTrackME("getContents (match)", nullptr, *it);
0619         out.push_back(*it);
0620         ++it;
0621       }
0622     }
0623     return out;
0624   }
0625 
0626   std::vector<dqm::harvesting::MonitorElement*> IGetter::getAllContents(std::string const& pathname) const {
0627     auto lock = std::scoped_lock(store_->booking_mutex_);
0628     std::vector<MonitorElement*> out;
0629     MonitorElementData::Path path;
0630     path.set(pathname, MonitorElementData::Path::Type::DIR);
0631     // make sure this is normalized by getting it from Path object.
0632     auto path_str = path.getFullname();
0633     for (auto& [runlumi, meset] : store_->globalMEs_) {
0634       auto it = meset.lower_bound(path);
0635       // rfind can be used as a prefix match.
0636       while (it != meset.end() && (*it)->getPathname().rfind(path_str, 0) == 0) {
0637         if (runlumi == edm::LuminosityBlockID() && (*it)->getScope() != MonitorElementData::Scope::JOB) {
0638           // skip prototypes
0639         } else {
0640           store_->debugTrackME("getAllContents (match)", nullptr, *it);
0641           out.push_back(*it);
0642         }
0643         ++it;
0644       }
0645     }
0646     return out;
0647   }
0648   std::vector<dqm::harvesting::MonitorElement*> IGetter::getAllContents(std::string const& pathname,
0649                                                                         uint32_t runNumber,
0650                                                                         uint32_t lumi) const {
0651     auto lock = std::scoped_lock(store_->booking_mutex_);
0652     std::vector<MonitorElement*> out;
0653     MonitorElementData::Path path;
0654     path.set(pathname, MonitorElementData::Path::Type::DIR);
0655     // make sure this is normalized by getting it from Path object.
0656     auto path_str = path.getFullname();
0657     auto const& meset = store_->globalMEs_[edm::LuminosityBlockID(runNumber, lumi)];
0658     auto it = meset.lower_bound(path);
0659 
0660     // decide if the ME should be saved in DQMIO and/or nanoDQMIO
0661     // if doSaveByLumi_ is false: store all monitoring elements (needed for harvesting step!)
0662     // if doSaveByLumi_ is true: store only selected monitoring elements (i.e. "nanoDQMIO")
0663     while (it != meset.end() && (*it)->getFullname().rfind(path_str, 0) == 0) {
0664       bool saveIt = true;  // default value if doSaveByLumi_ is false
0665 
0666       if (store_->doSaveByLumi_ && not store_->MEsToSave_.empty()) {
0667         std::string name = (*it)->getFullname();
0668         saveIt = false;  // default value if doSaveByLumi_ is true
0669         for (std::vector<std::string>::const_iterator ipath = store_->MEsToSave_.begin();
0670              ipath != store_->MEsToSave_.end();
0671              ++ipath) {
0672           std::string nameToSave = *ipath;
0673           // option 1 (used in the past): inclusive selection
0674           // (store all MEs that contain any of the requested patterns)
0675           // if (name.find(nameToSave) != std::string::npos) {
0676           // option 2 (current criterion): exact selection
0677           // (store only MEs that exactly match a requested pattern)
0678           if (name == nameToSave) {
0679             saveIt = true;
0680             // std::cout<<name<<" compared to"<<ipath->data()<<std::endl;
0681             break;
0682           }
0683         }
0684       }
0685 
0686       store_->debugTrackME("getAllContents (run/lumi match)", nullptr, *it);
0687       if (saveIt) {
0688         out.push_back(*it);
0689         if (store_->doSaveByLumi_)
0690           store_->debugTrackME("getAllContents (run/lumi saved)", nullptr, *it);
0691       }
0692       ++it;
0693     }
0694     return out;
0695   }
0696 
0697   MonitorElement* IGetter::get(std::string const& fullpath) const {
0698     MonitorElementData::Path path;
0699     path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME);
0700     // this only really makes sense if there is only one instance of this ME,
0701     // but the signature of this method also only makes sense in that case.
0702     return store_->findME(path);
0703   }
0704 
0705   MonitorElement* IGetter::get(MonitorElementData::Key const& key) const {
0706     auto const& meset = store_->globalMEs_[key.id_];
0707     auto it = meset.find(key.path_);
0708     if (it != meset.end()) {
0709       assert((*it)->getScope() == key.scope_);
0710       store_->debugTrackME("get (key found)", nullptr, *it);
0711       return *it;
0712     }
0713     return nullptr;
0714   }
0715 
0716   MonitorElement* IGetter::getElement(std::string const& path) const {
0717     auto result = this->get(path);
0718     if (result == nullptr) {
0719       throw cms::Exception("iGetter Error") << "ME " << path << " was requested but not found.";
0720     }
0721     return result;
0722   }
0723 
0724   std::vector<std::string> IGetter::getSubdirs() const {
0725     // This is terribly inefficient, esp. if this method is then used to
0726     // recursively enumerate whatever getAllContents would return anyways.
0727     // But that is fine, any such code should just use getAllContents instead.
0728     std::set<std::string> subdirs;
0729     for (auto me : this->getAllContents(this->cwd_)) {
0730       const auto& name = me->getPathname();
0731       auto subdirname = name.substr(this->cwd_.length(), std::string::npos);
0732       auto dirname = subdirname.substr(0, subdirname.find('/'));
0733       subdirs.insert(dirname);
0734     }
0735     std::vector<std::string> out;
0736     for (const auto& dir : subdirs) {
0737       if (dir.length() == 0)
0738         continue;
0739       out.push_back(this->cwd_ + dir);
0740     }
0741     return out;
0742   }
0743 
0744   std::vector<std::string> IGetter::getMEs() const {
0745     auto mes = this->getContents(this->cwd_);
0746     std::vector<std::string> out;
0747     out.reserve(mes.size());
0748     for (auto me : mes) {
0749       out.push_back(me->getName());
0750     }
0751     return out;
0752   }
0753 
0754   bool IGetter::dirExists(std::string const& path) const {
0755     // we don't claim this is fast.
0756     return !this->getAllContents(path).empty();
0757   }
0758 
0759   IGetter::IGetter(DQMStore* store) { store_ = store; }
0760 
0761   IGetter::~IGetter() {}
0762 
0763   DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry& ar) : IGetter(this), IBooker(this) {
0764     verbose_ = pset.getUntrackedParameter<int>("verbose", 0);
0765     assertLegacySafe_ = pset.getUntrackedParameter<bool>("assertLegacySafe", false);
0766     doSaveByLumi_ = pset.getUntrackedParameter<bool>("saveByLumi", false);
0767     MEsToSave_ = pset.getUntrackedParameter<std::vector<std::string>>("MEsToSave", std::vector<std::string>());
0768     trackME_ = pset.getUntrackedParameter<std::string>("trackME", "");
0769 
0770     // Set lumi and run for legacy booking.
0771     // This is no more than a guess with concurrent runs/lumis, but should be
0772     // correct for purely sequential legacy stuff.
0773     // Also reset Scope, such that legacy modules can expect it to be JOB.
0774     // initLumi and leaveLumi are needed for all module types: these handle
0775     // creating and deleting global MEs as needed, which has to happen even if
0776     // a module does not see lumi transitions.
0777     ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) {
0778       this->setRunLumi(gc.luminosityBlockID());
0779       this->initLumi(gc.luminosityBlockID().run(), /* lumi */ 0);
0780       this->enterLumi(gc.luminosityBlockID().run(), /* lumi */ 0, /* moduleID */ 0);
0781       this->setScope(MonitorElementData::Scope::JOB);
0782     });
0783     ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) {
0784       this->setRunLumi(gc.luminosityBlockID());
0785       this->initLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock());
0786       this->enterLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock(), /* moduleID */ 0);
0787     });
0788     ar.watchPostGlobalEndRun([this](edm::GlobalContext const& gc) {
0789       this->leaveLumi(gc.luminosityBlockID().run(), /* lumi */ 0, /* moduleID */ 0);
0790     });
0791     ar.watchPostGlobalEndLumi([this](edm::GlobalContext const& gc) {
0792       this->leaveLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock(), /* moduleID */ 0);
0793     });
0794 
0795     // Trigger cleanup after writing. This is needed for all modules; we can
0796     // only run the cleanup after all output modules have run.
0797     ar.watchPostGlobalWriteLumi([this](edm::GlobalContext const& gc) {
0798       this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock());
0799     });
0800     ar.watchPostGlobalWriteRun(
0801         [this](edm::GlobalContext const& gc) { this->cleanupLumi(gc.luminosityBlockID().run(), 0); });
0802 
0803     // no cleanup at end of job, we don't really need it.
0804   }
0805 
0806   DQMStore::~DQMStore() {}
0807 
0808   void DQMStore::save(std::string const& filename, std::string const& path) {
0809     LegacyIOHelper h(this);
0810     // no run number passed, will save a flat ROOT file (rather than 'Run xxxxxx/.../Run Summary/...')
0811     h.save(filename, path);
0812   }
0813 
0814   bool DQMStore::open(std::string const& filename,
0815                       bool overwrite,
0816                       std::string const& path,
0817                       std::string const& prepend,
0818                       OpenRunDirs stripdirs,
0819                       bool fileMustExist) {
0820     assert(!"NIY");
0821   }
0822 
0823 }  // namespace dqm::implementation