Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-03-19 06:07:20

0001 #include <filesystem>
0002 #include <map>
0003 #include <vector>
0004 #include <sys/stat.h>
0005 
0006 #include <fmt/printf.h>
0007 #include <boost/algorithm/string/predicate.hpp>
0008 #include <boost/range.hpp>
0009 #include <boost/regex.hpp>
0010 
0011 #include "DQMServices/Core/interface/DQMOneEDAnalyzer.h"
0012 #include "DQMServices/Core/interface/DQMStore.h"
0013 #include "DQMServices/StreamerIO/plugins/DQMFileIterator.h"
0014 #include "FWCore/Framework/interface/Event.h"
0015 #include "FWCore/Framework/interface/EventSetup.h"
0016 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0017 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0018 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0019 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0020 
0021 namespace dqm {
0022   namespace rdm {
0023     struct Empty {};
0024   }  // namespace rdm
0025   class RamdiskMonitor : public DQMOneEDAnalyzer<edm::LuminosityBlockCache<rdm::Empty>> {
0026   public:
0027     RamdiskMonitor(const edm::ParameterSet &ps);
0028     ~RamdiskMonitor() override;
0029     static void fillDescriptions(edm::ConfigurationDescriptions &descriptions);
0030 
0031   protected:
0032     void bookHistograms(DQMStore::IBooker &, edm::Run const &, edm::EventSetup const &) override;
0033     std::shared_ptr<rdm::Empty> globalBeginLuminosityBlock(edm::LuminosityBlock const &lumi,
0034                                                            edm::EventSetup const &eSetup) const override;
0035     void globalEndLuminosityBlock(edm::LuminosityBlock const &lumi, edm::EventSetup const &eSetup) final {}
0036     void analyze(edm::Event const &e, edm::EventSetup const &eSetup) override{};
0037 
0038     void analyzeFile(std::string fn, unsigned int run, unsigned int lumi, std::string label) const;
0039     double getRunTimestamp() const;
0040 
0041     const unsigned int runNumber_;
0042     const std::string runInputDir_;
0043     const std::vector<std::string> streamLabels_;
0044     const std::string runPath_;
0045 
0046     struct StreamME {
0047       MonitorElement *eventsAccepted;
0048       MonitorElement *eventsProcessed;
0049 
0050       MonitorElement *deliveryDelayMTime;
0051       MonitorElement *deliveryDelayCTime;
0052     };
0053 
0054     std::map<std::string, StreamME> streams_;
0055     mutable std::set<std::string> filesSeen_;
0056     mutable double global_start_ = 0.;
0057 
0058     static constexpr double LUMI = 23.310893056;
0059   };
0060 
0061   RamdiskMonitor::RamdiskMonitor(const edm::ParameterSet &ps)
0062       : runNumber_{ps.getUntrackedParameter<unsigned int>("runNumber")},
0063         runInputDir_{ps.getUntrackedParameter<std::string>("runInputDir")},
0064         streamLabels_{ps.getUntrackedParameter<std::vector<std::string>>("streamLabels")},
0065         runPath_{fmt::sprintf("%s/run%06d", runInputDir_, runNumber_)}
0066 
0067   {}
0068 
0069   RamdiskMonitor::~RamdiskMonitor(){};
0070 
0071   void RamdiskMonitor::bookHistograms(DQMStore::IBooker &ib, edm::Run const &, edm::EventSetup const &) {
0072     for (const auto &stream : streamLabels_) {
0073       edm::LogInfo("RamdiskMonitor") << "Booking: " << stream;
0074 
0075       ib.cd();
0076       ib.setCurrentFolder(std::string("Info/RamdiskMonitor/") + stream + "/");
0077 
0078       StreamME m;
0079 
0080       m.eventsAccepted = ib.book1D("EventAccepted", "# of accepted events per lumi", 4, 0., 4.);
0081       m.eventsProcessed = ib.book1D("EventProcessed", "# of processed events per lumi", 4, 0., 4.);
0082       m.deliveryDelayMTime =
0083           ib.book1D("DeliveryDelayMTime", "Observed delivery delay for the data file (mtime).", 4, 0., 4.);
0084       m.deliveryDelayCTime =
0085           ib.book1D("DeliveryDelayCTime", "Observed delivery delay for the data file (ctime).", 4, 0., 4.);
0086 
0087       m.eventsAccepted->getTH1F()->SetCanExtend(TH1::kXaxis);
0088       m.eventsProcessed->getTH1F()->SetCanExtend(TH1::kXaxis);
0089       m.deliveryDelayMTime->getTH1F()->SetCanExtend(TH1::kXaxis);
0090       m.deliveryDelayCTime->getTH1F()->SetCanExtend(TH1::kXaxis);
0091 
0092       m.eventsAccepted->setAxisTitle("Luminosity Section", 1);
0093       m.eventsProcessed->setAxisTitle("Luminosity Section", 1);
0094       m.deliveryDelayMTime->setAxisTitle("Luminosity Section", 1);
0095       m.deliveryDelayCTime->setAxisTitle("Luminosity Section", 1);
0096 
0097       m.eventsAccepted->setAxisTitle("Number of events", 2);
0098       m.eventsProcessed->setAxisTitle("Number of events", 2);
0099       m.deliveryDelayMTime->setAxisTitle("Delay (s.)", 2);
0100       m.deliveryDelayCTime->setAxisTitle("Delay (s.)", 2);
0101 
0102       streams_[stream] = m;
0103     }
0104   };
0105 
0106   double RamdiskMonitor::getRunTimestamp() const {
0107     if (global_start_ != 0)
0108       return global_start_;
0109 
0110     std::string run_global = fmt::sprintf("%s/.run%06d.global", runInputDir_, runNumber_);
0111     struct stat st;
0112     if (::stat(run_global.c_str(), &st) != 0) {
0113       edm::LogWarning("RamdiskMonitor") << "Stat failed: " << run_global;
0114       return 0.;
0115     }
0116 
0117     global_start_ = st.st_mtime;
0118     edm::LogPrint("RamdiskMonitor") << "Run start timestamp: " << global_start_;
0119     return global_start_;
0120   };
0121 
0122   void RamdiskMonitor::analyzeFile(std::string fn, unsigned int run, unsigned int lumi, std::string label) const {
0123     using LumiEntry = dqmservices::DQMFileIterator::LumiEntry;
0124 
0125     // we are disabled, at least for this stream
0126     if (streams_.empty())
0127       return;
0128 
0129     auto itStream = streams_.find(label);
0130     if (itStream == streams_.end()) {
0131       edm::LogPrint("RamdiskMonitor") << "Stream not monitored [" << label << "]: " << fn;
0132       return;
0133     }
0134 
0135     StreamME m = itStream->second;
0136 
0137     // decode json and fill in some histograms
0138     LumiEntry lumi_jsn = LumiEntry::load_json(runPath_, fn, lumi, -1);
0139     m.eventsAccepted->setBinContent(lumi, lumi_jsn.n_events_accepted);
0140     m.eventsProcessed->setBinContent(lumi, lumi_jsn.n_events_processed);
0141 
0142     // collect stat struct and calculate mtimes
0143     struct stat st;
0144     if (::stat(fn.c_str(), &st) != 0) {
0145       edm::LogWarning("RamdiskMonitor") << "Stat failed: " << fn;
0146       return;
0147     }
0148 
0149     // get start offset (from .global)
0150     // abort the calculation if it does not exist
0151     double start_offset = getRunTimestamp();
0152     if (start_offset <= 0)
0153       return;
0154 
0155     // check fff_dqmtools (separate repository)
0156     // for calculation details
0157     double mtime = st.st_mtime;
0158     double ctime = st.st_ctime;
0159 
0160     // timeout from the begging of the run
0161     double start_offset_mtime = mtime - start_offset - LUMI;
0162     double start_offset_ctime = ctime - start_offset - LUMI;
0163     double lumi_offset = (lumi - 1) * LUMI;
0164 
0165     // timeout from the time we think this lumi happenned
0166     double delay_mtime = start_offset_mtime - lumi_offset;
0167     double delay_ctime = start_offset_ctime - lumi_offset;
0168 
0169     m.deliveryDelayMTime->setBinContent(lumi, delay_mtime);
0170     m.deliveryDelayCTime->setBinContent(lumi, delay_ctime);
0171   };
0172 
0173   std::shared_ptr<dqm::rdm::Empty> RamdiskMonitor::globalBeginLuminosityBlock(edm::LuminosityBlock const &,
0174                                                                               edm::EventSetup const &eSetup) const {
0175     // search filesystem to find available lumi section files
0176     using std::filesystem::directory_entry;
0177     using std::filesystem::directory_iterator;
0178 
0179     directory_iterator dend;
0180     for (directory_iterator di(runPath_); di != dend; ++di) {
0181       const boost::regex fn_re("run(\\d+)_ls(\\d+)_([a-zA-Z0-9]+)(_.*)?\\.jsn");
0182 
0183       const std::string filename = di->path().filename().string();
0184       const std::string fn = di->path().string();
0185 
0186       if (filesSeen_.find(filename) != filesSeen_.end()) {
0187         continue;
0188       }
0189 
0190       boost::smatch result;
0191       if (boost::regex_match(filename, result, fn_re)) {
0192         unsigned int run = std::stoi(result[1]);
0193         unsigned int lumi = std::stoi(result[2]);
0194         std::string label = result[3];
0195 
0196         filesSeen_.insert(filename);
0197 
0198         if (run != runNumber_)
0199           continue;
0200 
0201         // check if this is EoR
0202         if ((lumi == 0) && (label == "EoR")) {
0203           // do not handle
0204           continue;
0205         }
0206 
0207         try {
0208           this->analyzeFile(fn, run, lumi, label);
0209         } catch (const std::exception &e) {
0210           // it's likely we have read it too soon
0211           filesSeen_.erase(filename);
0212 
0213           std::string msg("Found, tried to load the json, but failed (");
0214           msg += e.what();
0215           msg += "): ";
0216           edm::LogWarning("RamdiskMonitor") << msg;
0217         }
0218       }
0219     }
0220 
0221     // @TODO lookup info for the current lumi
0222     return std::shared_ptr<dqm::rdm::Empty>();
0223   }
0224 
0225   void RamdiskMonitor::fillDescriptions(edm::ConfigurationDescriptions &d) {
0226     edm::ParameterSetDescription desc;
0227 
0228     desc.setComment(
0229         "Analyses file timestams in the /fff/ramdisk and creates monitor "
0230         "elements.");
0231 
0232     desc.addUntracked<std::vector<std::string>>("streamLabels")->setComment("List of streams to monitor.");
0233 
0234     desc.addUntracked<unsigned int>("runNumber")->setComment("Run number passed via configuration file.");
0235 
0236     desc.addUntracked<std::string>("runInputDir")->setComment("Directory where the DQM files will appear.");
0237 
0238     d.add("RamdiskMonitor", desc);
0239   }
0240 
0241 }  // namespace dqm
0242 
0243 #include "FWCore/Framework/interface/MakerMacros.h"
0244 
0245 typedef dqm::RamdiskMonitor RamdiskMonitor;
0246 DEFINE_FWK_MODULE(RamdiskMonitor);