FileIO/plugins/DQMFileSaverOnline.cc

0001 #include "DQMServices/Core/interface/DQMStore.h"
0002 #include "DQMServices/Core/interface/LegacyIOHelper.h"
0003 #include "FWCore/Framework/interface/Event.h"
0004 #include "FWCore/Framework/interface/LuminosityBlock.h"
0005 #include "FWCore/Framework/interface/Run.h"
0006 #include "FWCore/MessageLogger/interface/JobReport.h"
0007 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0008 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0009 #include "FWCore/ServiceRegistry/interface/Service.h"
0010 #include "FWCore/Version/interface/GetReleaseVersion.h"
0011 #include "Utilities/OpenSSL/interface/openssl_init.h"
0012 #include "DQMFileSaverOnline.h"
0013
0014 #include <TString.h>
0015 #include <TSystem.h>
0016 #include <sys/stat.h>
0017 #include <sys/types.h>
0018 #include <unistd.h>
0019 #include <fstream>
0020 #include <iostream>
0021 #include <string>
0022 #include <utility>
0023 #include <vector>
0024
0025 #include <filesystem>
0026 #include <boost/iostreams/device/mapped_file.hpp>
0027
0028 using namespace dqm;
0029
0030 DQMFileSaverOnline::DQMFileSaverOnline(const edm::ParameterSet& ps) : DQMFileSaverBase(ps) {
0031   backupLumiCount_ = ps.getUntrackedParameter<int>("backupLumiCount", 1);
0032   keepBackupLumi_ = ps.getUntrackedParameter<bool>("keepBackupLumi", false);
0033 }
0034
0035 DQMFileSaverOnline::~DQMFileSaverOnline() = default;
0036
0037 void DQMFileSaverOnline::saveLumi(const FileParameters& fp) const {
0038   if (backupLumiCount_ > 0) {
0039     if (fp.lumi_ % backupLumiCount_ == 0) {
0040       // actual saving is done here
0041       makeSnapshot(fp, false);
0042     }
0043   }
0044 }
0045
0046 void DQMFileSaverOnline::saveRun(const FileParameters& fp) const { makeSnapshot(fp, true); }
0047
0048 void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp, bool final) const {
0049   int pid = getpid();
0050   char hostname[64];
0051   gethostname(hostname, 64);
0052   hostname[63] = 0;
0053
0054   char suffix[128];
0055   if (!final) {
0056     snprintf(suffix, 127, ".ls%08ld_host%s_pid%08d", fp.lumi_, hostname, pid);
0057   } else {
0058     suffix[0] = 0;
0059   }
0060
0061   std::string prefix = filename(fp, false);
0062
0063   std::string root_fp = prefix + ".root" + suffix;
0064   std::string meta_fp = prefix + ".root.origin" + suffix;
0065
0066   std::string tmp_root_fp = root_fp + ".tmp";
0067   std::string tmp_meta_fp = meta_fp + ".tmp";
0068
0069   // run_ and lumi_ are ignored if dqmstore is not in multithread mode
0070   edm::Service<DQMStore> store;
0071
0072   logFileAction("Writing DQM Root file: ", root_fp);
0073   // logFileAction("Writing DQM Origin file: ", meta_fp);
0074
0075   //char rewrite[128];
0076   //snprintf(rewrite, 128, "\\1Run %ld/\\2/Run summary", fp.run_);
0077
0078   //store->save(tmp_root_fp,                      /* filename      */
0079   //            "",                               /* path          */
0080   //            "^(Reference/)?([^/]+)",          /* pattern       */
0081   //            rewrite,                          /* rewrite       */
0082   //            store->mtEnabled() ? fp.run_ : 0, /* run           */
0083   //            0,                                /* lumi          */
0084   //            fp.saveReference_,                /* ref           */
0085   //            fp.saveReferenceQMin_,            /* ref minStatus */
0086   //            "RECREATE");                      /* fileupdate    */
0087   // TODO: some parameters prepared here are now unused, and the code should
0088   // eventually be removed.
0089   LegacyIOHelper h(&*store);
0090   h.save(tmp_root_fp, "", fp.run_, /* saveall */ true, "RECREATE");
0091
0092   // write metadata
0093   // format.origin: md5:d566a34b27f48d507150a332b189398b 294835
0094   // /home/dqmprolocal/output/DQM_V0001_FED_R000194224.root
0095   std::ofstream meta_fd(tmp_meta_fp);
0096   meta_fd << fillOrigin(tmp_root_fp, root_fp);
0097   meta_fd.close();
0098
0099   checkError("Rename failed: ", root_fp, ::rename(tmp_root_fp.c_str(), root_fp.c_str()));
0100   checkError("Rename failed: ", meta_fp, ::rename(tmp_meta_fp.c_str(), meta_fp.c_str()));
0101
0102   SnapshotFiles files = {root_fp, meta_fp};
0103   if (final) {
0104     // final will never be cleared
0105     appendSnapshot(SnapshotFiles{});
0106
0107     saveJobReport(root_fp);
0108   } else {
0109     appendSnapshot(SnapshotFiles{root_fp, meta_fp});
0110   }
0111 }
0112
0113 void DQMFileSaverOnline::appendSnapshot(SnapshotFiles f) const {
0114   std::lock_guard<std::mutex> lock(snapshots_lock_);
0115
0116   if (!keepBackupLumi_) {
0117     while (!snapshots_.empty()) {
0118       SnapshotFiles& x = snapshots_.front();
0119
0120       // logFileAction("Deleting old snapshot (origin): ", x.meta);
0121       checkError("Unlink failed: ", x.meta, ::unlink(x.meta.c_str()));
0122
0123       logFileAction("Deleting old snapshot (root): ", x.data);
0124       checkError("Unlink failed: ", x.data, ::unlink(x.data.c_str()));
0125
0126       snapshots_.pop_front();
0127     }
0128   }
0129
0130   if (!f.data.empty()) {
0131     snapshots_.push_back(f);
0132   }
0133 }
0134
0135 void DQMFileSaverOnline::checkError(const char* msg, const std::string& file, int status) const {
0136   if (status != 0) {
0137     std::string actual_msg = msg;
0138     actual_msg += std::strerror(status);
0139     logFileAction(actual_msg, file);
0140   }
0141 }
0142
0143 const std::string DQMFileSaverOnline::fillOrigin(const std::string& filename, const std::string& final_filename) {
0144   // format.origin (one line):
0145   //   md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root
0146
0147   cms::openssl_init();
0148   EVP_MD_CTX* mdctx = EVP_MD_CTX_new();
0149   const EVP_MD* md = EVP_get_digestbyname("MD5");
0150   unsigned int md_len = 0;
0151   unsigned char md5[EVP_MAX_MD_SIZE];
0152
0153   boost::iostreams::mapped_file_source fp(filename);
0154
0155   EVP_DigestInit_ex(mdctx, md, nullptr);
0156   EVP_DigestUpdate(mdctx, (unsigned char*)fp.data(), fp.size());
0157   EVP_DigestFinal_ex(mdctx, md5, &md_len);
0158   EVP_MD_CTX_free(mdctx);
0159
0160   std::ostringstream hash;
0161   for (unsigned int i = 0; i < md_len; i++) {
0162     hash << std::hex << std::setfill('0') << std::setw(2) << (int)md5[i];
0163   }
0164
0165   std::ostringstream out;
0166   out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
0167   return out.str();
0168 }
0169
0170 void DQMFileSaverOnline::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0171   edm::ParameterSetDescription desc;
0172   desc.setComment("Saves histograms from DQM store, online workflow.");
0173
0174   desc.addUntracked<int>("backupLumiCount", 10)
0175       ->setComment(
0176           "How often the backup file will be generated, in lumisections (-1 "
0177           "disables).");
0178
0179   desc.addUntracked<bool>("keepBackupLumi", false)
0180       ->setComment(
0181           "Usually the backup old backup is deleted once the new file is "
0182           "available. Setting this to true ensures that no backup files are "
0183           "ever deleted. Useful for ML applications, which use backups as a "
0184           "'history' of what happened during the run.");
0185
0186   DQMFileSaverBase::fillDescription(desc);
0187
0188   // Changed to use addDefault instead of add here because previously
0189   // DQMFileSaverOnline and DQMFileSaverPB both used the module label
0190   // "saver" which caused conflicting cfi filenames to be generated.
0191   // add could be used if unique module labels were given.
0192   descriptions.addDefault(desc);
0193 }
0194
0195 #include "FWCore/Framework/interface/MakerMacros.h"
0196 DEFINE_FWK_MODULE(DQMFileSaverOnline);