Components/plugins/DQMFileSaver.cc

0001 #include "DQMServices/Core/interface/DQMStore.h"
0002 #include "DQMServices/Core/interface/LegacyIOHelper.h"
0003
0004 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
0005 #include "FWCore/Framework/interface/Event.h"
0006 #include "FWCore/Framework/interface/Run.h"
0007 #include "FWCore/Framework/interface/LuminosityBlock.h"
0008 #include "FWCore/Framework/interface/GetterOfProducts.h"
0009 #include "FWCore/Framework/interface/ProcessMatch.h"
0010 #include "FWCore/Framework/interface/LuminosityBlock.h"
0011 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0012 #include "FWCore/Version/interface/GetReleaseVersion.h"
0013 #include "FWCore/ServiceRegistry/interface/Service.h"
0014 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0015 #include "FWCore/MessageLogger/interface/JobReport.h"
0016
0017 #include "DataFormats/Histograms/interface/DQMToken.h"
0018
0019 #include <sys/stat.h>
0020
0021 namespace saverDetails {
0022   struct NoCache {};
0023 }  // namespace saverDetails
0024
0025 // NOTE: This module is only save to use in a very restricted set of circumstances:
0026 // - In offline HARVESTING jobs, running single-threaded. RUN and JOB histograms are saved at end of JOB.
0027 // - In multi-run harvesting. JOB histograms are save at end of job.
0028 // - This includes ALCAHARVEST. TODO: check if the data written there is needed for the PCL.
0029 // This module is not used in online. This module is (hopefully?) not used at HLT.
0030 // Online and HLT use modules in DQMServices/FileIO.
0031 class DQMFileSaver : public edm::one::EDAnalyzer<edm::one::WatchRuns, edm::WatchProcessBlock> {
0032 public:
0033   typedef dqm::legacy::DQMStore DQMStore;
0034   typedef dqm::legacy::MonitorElement MonitorElement;
0035   DQMFileSaver(const edm::ParameterSet &ps);
0036
0037 protected:
0038   void beginRun(edm::Run const &, edm::EventSetup const &) override{};
0039   void analyze(const edm::Event &e, const edm::EventSetup &) override;
0040   void endRun(const edm::Run &, const edm::EventSetup &) override;
0041   void endProcessBlock(const edm::ProcessBlock &) override;
0042
0043 private:
0044   void saveForOffline(const std::string &workflow, int run, int lumi);
0045   void saveJobReport(const std::string &filename);
0046
0047   std::string workflow_;
0048   std::string producer_;
0049   std::string stream_label_;
0050   std::string dirName_;
0051   std::string child_;
0052   int version_;
0053   bool runIsComplete_;
0054
0055   int saveByRun_;
0056   bool saveAtJobEnd_;
0057   int forceRunNumber_;
0058
0059   std::string fileBaseName_;
0060
0061   DQMStore *dbe_;
0062   int nrun_;
0063
0064   // needed only for the harvesting step when saving in the endJob
0065   int irun_;
0066
0067   edm::GetterOfProducts<DQMToken> jobmegetter_;
0068   edm::GetterOfProducts<DQMToken> runmegetter_;
0069 };
0070
0071 //--------------------------------------------------------
0072 static void getAnInt(const edm::ParameterSet &ps, int &value, const std::string &name) {
0073   value = ps.getUntrackedParameter<int>(name, value);
0074   if (value < 1 && value != -1)
0075     throw cms::Exception("DQMFileSaver") << "Invalid '" << name << "' parameter '" << value
0076                                          << "'.  Must be -1 or >= 1.";
0077 }
0078
0079 static std::string onlineOfflineFileName(const std::string &fileBaseName,
0080                                          const std::string &suffix,
0081                                          const std::string &workflow,
0082                                          const std::string &child) {
0083   size_t pos = 0;
0084   std::string wflow;
0085   wflow.reserve(workflow.size() + 3);
0086   wflow = workflow;
0087   while ((pos = wflow.find('/', pos)) != std::string::npos)
0088     wflow.replace(pos++, 1, "__");
0089
0090   std::string filename = fileBaseName + suffix + wflow + child + ".root";
0091   return filename;
0092 }
0093
0094 void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi) {
0095   char suffix[64];
0096   sprintf(suffix, "R%09d", run);
0097
0098   std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_);
0099   assert(lumi == 0);
0100
0101   // set run end flag
0102   dbe_->cd();
0103   dbe_->setCurrentFolder("Info/ProvInfo");
0104
0105   // do this, because ProvInfo is not yet run in offline DQM
0106   MonitorElement *me = dbe_->get("Info/ProvInfo/CMSSW");
0107   if (!me)
0108     me = dbe_->bookString("CMSSW", edm::getReleaseVersion().c_str());
0109
0110   me = dbe_->get("Info/ProvInfo/runIsComplete");
0111   if (!me)
0112     me = dbe_->bookFloat("runIsComplete");
0113
0114   if (me) {
0115     if (runIsComplete_)
0116       me->Fill(1.);
0117     else
0118       me->Fill(0.);
0119   }
0120
0121   LegacyIOHelper h(dbe_);
0122   h.save(filename, "", run, /* saveall */ true, "RECREATE");
0123
0124   // save the JobReport
0125   saveJobReport(filename);
0126 }
0127
0128 void DQMFileSaver::saveJobReport(const std::string &filename) {
0129   // Report the file to job report service.
0130   edm::Service<edm::JobReport> jr;
0131   if (jr.isAvailable()) {
0132     std::map<std::string, std::string> info;
0133     info["Source"] = "DQMStore";
0134     info["FileClass"] = "DQM";
0135     jr->reportAnalysisFile(filename, info);
0136   }
0137 }
0138
0139 //--------------------------------------------------------
0140 DQMFileSaver::DQMFileSaver(const edm::ParameterSet &ps)
0141     :
0142
0143       workflow_(""),
0144       producer_("DQM"),
0145       dirName_("."),
0146       child_(""),
0147       version_(1),
0148       runIsComplete_(false),
0149       saveByRun_(-1),
0150       saveAtJobEnd_(false),
0151       forceRunNumber_(-1),
0152       fileBaseName_(""),
0153       dbe_(&*edm::Service<DQMStore>()),
0154       nrun_(0),
0155       irun_(0),
0156       // Abuse ProcessMatch as a "match all".
0157       jobmegetter_(edm::GetterOfProducts<DQMToken>(edm::ProcessMatch("*"), this, edm::InProcess)),
0158       runmegetter_(edm::GetterOfProducts<DQMToken>(edm::ProcessMatch("*"), this, edm::InRun)) {
0159   callWhenNewProductsRegistered([this](edm::BranchDescription const &bd) {
0160     this->jobmegetter_(bd);
0161     this->runmegetter_(bd);
0162   });
0163
0164   workflow_ = ps.getUntrackedParameter<std::string>("workflow", workflow_);
0165   if (workflow_.empty() || workflow_[0] != '/' || *workflow_.rbegin() == '/' ||
0166       std::count(workflow_.begin(), workflow_.end(), '/') != 3 ||
0167       workflow_.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
0168                                   "abcdefghijklmnopqrstuvwxyz"
0169                                   "0123456789"
0170                                   "-_/") != std::string::npos)
0171     throw cms::Exception("DQMFileSaver") << "Invalid 'workflow' parameter '" << workflow_ << "'.  Expected '/A/B/C'.";
0172
0173   // version number to be used in filename
0174   // Note that version *must* always be 1 vor DQMGUI upload.
0175   version_ = ps.getUntrackedParameter<int>("version", version_);
0176   // flag to signal that file contains data from complete run
0177   runIsComplete_ = ps.getUntrackedParameter<bool>("runIsComplete", runIsComplete_);
0178
0179   // Get and check the output directory.
0180   struct stat s;
0181   dirName_ = ps.getUntrackedParameter<std::string>("dirName", dirName_);
0182   if (dirName_.empty() || stat(dirName_.c_str(), &s) == -1)
0183     throw cms::Exception("DQMFileSaver") << "Invalid 'dirName' parameter '" << dirName_ << "'.";
0184
0185   // Find out when and how to save files.  The following contraints apply:
0186   // - For offline allow files to be saved per run, at job end, and run number to be overridden (for mc data).
0187
0188   getAnInt(ps, saveByRun_, "saveByRun");
0189   getAnInt(ps, forceRunNumber_, "forceRunNumber");
0190   saveAtJobEnd_ = ps.getUntrackedParameter<bool>("saveAtJobEnd", saveAtJobEnd_);
0191
0192   // Set up base file name:
0193   // - for online and offline, follow the convention <dirName>/<producer>_V<4digits>_
0194   char version[8];
0195   sprintf(version, "_V%04d_", int(version_));
0196   version[7] = '\0';
0197   fileBaseName_ = dirName_ + "/" + producer_ + version;
0198
0199   // Log some information what we will do.
0200   edm::LogInfo("DQMFileSaver") << "DQM file saving settings:\n"
0201                                << " using base file name '" << fileBaseName_ << "'\n"
0202                                << " forcing run number " << forceRunNumber_ << "\n"
0203                                << " saving every " << saveByRun_ << " run(s)\n"
0204                                << " saving at job end: " << (saveAtJobEnd_ ? "yes" : "no") << "\n";
0205 }
0206
0207 void DQMFileSaver::analyze(const edm::Event &e, const edm::EventSetup &) {
0208   //save by event and save by time are not supported
0209   //anymore in the threaded framework. please use
0210   //savebyLumiSection instead.
0211 }
0212
0213 void DQMFileSaver::endRun(const edm::Run &iRun, const edm::EventSetup &) {
0214   int irun = iRun.id().run();
0215   irun_ = irun;
0216   if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0) {
0217     saveForOffline(workflow_, irun, 0);
0218   }
0219 }
0220
0221 void DQMFileSaver::endProcessBlock(const edm::ProcessBlock &) {
0222   if (saveAtJobEnd_) {
0223     if (forceRunNumber_ > 0)
0224       saveForOffline(workflow_, forceRunNumber_, 0);
0225     else
0226       saveForOffline(workflow_, irun_, 0);
0227   }
0228 }
0229
0230 #include "FWCore/Framework/interface/MakerMacros.h"
0231 DEFINE_FWK_MODULE(DQMFileSaver);