Components/plugins/DQMFileSaver.cc

0001 #include "DQMServices/Core/interface/DQMStore.h"
0002 #include "DQMServices/Core/interface/LegacyIOHelper.h"
0003
0004 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
0005 #include "FWCore/Framework/interface/Event.h"
0006 #include "FWCore/Framework/interface/Run.h"
0007 #include "FWCore/Framework/interface/LuminosityBlock.h"
0008 #include "FWCore/Framework/interface/GetterOfProducts.h"
0009 #include "FWCore/Framework/interface/ProcessMatch.h"
0010 #include "FWCore/Framework/interface/LuminosityBlock.h"
0011 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0012 #include "FWCore/Version/interface/GetReleaseVersion.h"
0013 #include "FWCore/ServiceRegistry/interface/Service.h"
0014 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0015 #include "FWCore/MessageLogger/interface/JobReport.h"
0016
0017 #include "DataFormats/Histograms/interface/DQMToken.h"
0018
0019 #include <sys/stat.h>
0020
0021 namespace saverDetails {
0022   struct NoCache {};
0023 }  // namespace saverDetails
0024
0025 // NOTE: This module is only save to use in a very restricted set of circumstances:
0026 // - In offline HARVESTING jobs, running single-threaded. RUN and JOB histograms are saved at end of JOB.
0027 // - In multi-run harvesting. JOB histograms are save at end of job.
0028 // - This includes ALCAHARVEST. TODO: check if the data written there is needed for the PCL.
0029 // This module is not used in online. This module is (hopefully?) not used at HLT.
0030 // Online and HLT use modules in DQMServices/FileIO.
0031 class DQMFileSaver : public edm::one::EDAnalyzer<edm::one::WatchRuns, edm::WatchProcessBlock> {
0032 public:
0033   typedef dqm::legacy::DQMStore DQMStore;
0034   typedef dqm::legacy::MonitorElement MonitorElement;
0035   DQMFileSaver(const edm::ParameterSet &ps);
0036
0037 protected:
0038   void beginRun(edm::Run const &, edm::EventSetup const &) override{};
0039   void analyze(const edm::Event &e, const edm::EventSetup &) override;
0040   void endRun(const edm::Run &, const edm::EventSetup &) override;
0041   void endProcessBlock(const edm::ProcessBlock &) override;
0042
0043 private:
0044   void saveForOffline(const std::string &workflow, int run, int lumi);
0045   void saveJobReport(const std::string &filename);
0046
0047   std::string workflow_;
0048   std::string producer_;
0049   std::string stream_label_;
0050   std::string dirName_;
0051   std::string child_;
0052   int version_;
0053   bool runIsComplete_;
0054
0055   int saveByRun_;
0056   bool saveAtJobEnd_;
0057   int forceRunNumber_;
0058
0059   std::string fileBaseName_;
0060
0061   DQMStore *dbe_;
0062   int nrun_;
0063
0064   // needed only for the harvesting step when saving in the endJob
0065   int irun_;
0066
0067   // We want to consume all DQMTokens for runs and jobs. But consumesMany is
0068   // confused by the labels, so we sue these getters.
0069   edm::GetterOfProducts<DQMToken> jobmegetter_;
0070   edm::GetterOfProducts<DQMToken> runmegetter_;
0071 };
0072
0073 //--------------------------------------------------------
0074 static void getAnInt(const edm::ParameterSet &ps, int &value, const std::string &name) {
0075   value = ps.getUntrackedParameter<int>(name, value);
0076   if (value < 1 && value != -1)
0077     throw cms::Exception("DQMFileSaver") << "Invalid '" << name << "' parameter '" << value
0078                                          << "'.  Must be -1 or >= 1.";
0079 }
0080
0081 static std::string onlineOfflineFileName(const std::string &fileBaseName,
0082                                          const std::string &suffix,
0083                                          const std::string &workflow,
0084                                          const std::string &child) {
0085   size_t pos = 0;
0086   std::string wflow;
0087   wflow.reserve(workflow.size() + 3);
0088   wflow = workflow;
0089   while ((pos = wflow.find('/', pos)) != std::string::npos)
0090     wflow.replace(pos++, 1, "__");
0091
0092   std::string filename = fileBaseName + suffix + wflow + child + ".root";
0093   return filename;
0094 }
0095
0096 void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi) {
0097   char suffix[64];
0098   sprintf(suffix, "R%09d", run);
0099
0100   std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_);
0101   assert(lumi == 0);
0102
0103   // set run end flag
0104   dbe_->cd();
0105   dbe_->setCurrentFolder("Info/ProvInfo");
0106
0107   // do this, because ProvInfo is not yet run in offline DQM
0108   MonitorElement *me = dbe_->get("Info/ProvInfo/CMSSW");
0109   if (!me)
0110     me = dbe_->bookString("CMSSW", edm::getReleaseVersion().c_str());
0111
0112   me = dbe_->get("Info/ProvInfo/runIsComplete");
0113   if (!me)
0114     me = dbe_->bookFloat("runIsComplete");
0115
0116   if (me) {
0117     if (runIsComplete_)
0118       me->Fill(1.);
0119     else
0120       me->Fill(0.);
0121   }
0122
0123   LegacyIOHelper h(dbe_);
0124   h.save(filename, "", run, /* saveall */ true, "RECREATE");
0125
0126   // save the JobReport
0127   saveJobReport(filename);
0128 }
0129
0130 void DQMFileSaver::saveJobReport(const std::string &filename) {
0131   // Report the file to job report service.
0132   edm::Service<edm::JobReport> jr;
0133   if (jr.isAvailable()) {
0134     std::map<std::string, std::string> info;
0135     info["Source"] = "DQMStore";
0136     info["FileClass"] = "DQM";
0137     jr->reportAnalysisFile(filename, info);
0138   }
0139 }
0140
0141 //--------------------------------------------------------
0142 DQMFileSaver::DQMFileSaver(const edm::ParameterSet &ps)
0143     :
0144
0145       workflow_(""),
0146       producer_("DQM"),
0147       dirName_("."),
0148       child_(""),
0149       version_(1),
0150       runIsComplete_(false),
0151       saveByRun_(-1),
0152       saveAtJobEnd_(false),
0153       forceRunNumber_(-1),
0154       fileBaseName_(""),
0155       dbe_(&*edm::Service<DQMStore>()),
0156       nrun_(0),
0157       irun_(0),
0158       // Abuse ProcessMatch as a "match all".
0159       jobmegetter_(edm::GetterOfProducts<DQMToken>(edm::ProcessMatch("*"), this, edm::InProcess)),
0160       runmegetter_(edm::GetterOfProducts<DQMToken>(edm::ProcessMatch("*"), this, edm::InRun)) {
0161   callWhenNewProductsRegistered([this](edm::BranchDescription const &bd) {
0162     this->jobmegetter_(bd);
0163     this->runmegetter_(bd);
0164   });
0165
0166   workflow_ = ps.getUntrackedParameter<std::string>("workflow", workflow_);
0167   if (workflow_.empty() || workflow_[0] != '/' || *workflow_.rbegin() == '/' ||
0168       std::count(workflow_.begin(), workflow_.end(), '/') != 3 ||
0169       workflow_.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ"
0170                                   "abcdefghijklmnopqrstuvwxyz"
0171                                   "0123456789"
0172                                   "-_/") != std::string::npos)
0173     throw cms::Exception("DQMFileSaver") << "Invalid 'workflow' parameter '" << workflow_ << "'.  Expected '/A/B/C'.";
0174
0175   // version number to be used in filename
0176   // Note that version *must* always be 1 vor DQMGUI upload.
0177   version_ = ps.getUntrackedParameter<int>("version", version_);
0178   // flag to signal that file contains data from complete run
0179   runIsComplete_ = ps.getUntrackedParameter<bool>("runIsComplete", runIsComplete_);
0180
0181   // Get and check the output directory.
0182   struct stat s;
0183   dirName_ = ps.getUntrackedParameter<std::string>("dirName", dirName_);
0184   if (dirName_.empty() || stat(dirName_.c_str(), &s) == -1)
0185     throw cms::Exception("DQMFileSaver") << "Invalid 'dirName' parameter '" << dirName_ << "'.";
0186
0187   // Find out when and how to save files.  The following contraints apply:
0188   // - For offline allow files to be saved per run, at job end, and run number to be overridden (for mc data).
0189
0190   getAnInt(ps, saveByRun_, "saveByRun");
0191   getAnInt(ps, forceRunNumber_, "forceRunNumber");
0192   saveAtJobEnd_ = ps.getUntrackedParameter<bool>("saveAtJobEnd", saveAtJobEnd_);
0193
0194   // Set up base file name:
0195   // - for online and offline, follow the convention <dirName>/<producer>_V<4digits>_
0196   char version[8];
0197   sprintf(version, "_V%04d_", int(version_));
0198   version[7] = '\0';
0199   fileBaseName_ = dirName_ + "/" + producer_ + version;
0200
0201   // Log some information what we will do.
0202   edm::LogInfo("DQMFileSaver") << "DQM file saving settings:\n"
0203                                << " using base file name '" << fileBaseName_ << "'\n"
0204                                << " forcing run number " << forceRunNumber_ << "\n"
0205                                << " saving every " << saveByRun_ << " run(s)\n"
0206                                << " saving at job end: " << (saveAtJobEnd_ ? "yes" : "no") << "\n";
0207 }
0208
0209 void DQMFileSaver::analyze(const edm::Event &e, const edm::EventSetup &) {
0210   //save by event and save by time are not supported
0211   //anymore in the threaded framework. please use
0212   //savebyLumiSection instead.
0213 }
0214
0215 void DQMFileSaver::endRun(const edm::Run &iRun, const edm::EventSetup &) {
0216   int irun = iRun.id().run();
0217   irun_ = irun;
0218   if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0) {
0219     saveForOffline(workflow_, irun, 0);
0220   }
0221 }
0222
0223 void DQMFileSaver::endProcessBlock(const edm::ProcessBlock &) {
0224   if (saveAtJobEnd_) {
0225     if (forceRunNumber_ > 0)
0226       saveForOffline(workflow_, forceRunNumber_, 0);
0227     else
0228       saveForOffline(workflow_, irun_, 0);
0229   }
0230 }
0231
0232 #include "FWCore/Framework/interface/MakerMacros.h"
0233 DEFINE_FWK_MODULE(DQMFileSaver);