Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2025-03-10 23:53:33 

0001 // -*- C++ -*-
0002 //
0003 // Package:     FwkIO
0004 // Class  :     DQMRootSource
0005 //
0006 // Implementation:
0007 //     [Notes on implementation]
0008 //
0009 // Original Author:  Chris Jones
0010 //         Created:  Tue May  3 11:13:47 CDT 2011
0011 //
0012 
0013 // system include files
0014 #include <memory>
0015 
0016 #include "TFile.h"
0017 #include "TString.h"
0018 #include "TTree.h"
0019 #include <map>
0020 #include <string>
0021 #include <vector>
0022 
0023 #include "DQMServices/Core/interface/DQMStore.h"
0024 #include "DataFormats/Histograms/interface/DQMToken.h"
0025 #include "FWCore/ServiceRegistry/interface/Service.h"
0026 
0027 #include "FWCore/Framework/interface/InputSource.h"
0028 #include "FWCore/Sources/interface/PuttableSourceBase.h"
0029 #include "FWCore/Catalog/interface/InputFileCatalog.h"
0030 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0031 #include "FWCore/Utilities/interface/ExceptionPropagate.h"
0032 
0033 #include "FWCore/Framework/interface/RunPrincipal.h"
0034 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
0035 
0036 #include "FWCore/Framework/interface/Run.h"
0037 #include "FWCore/Framework/interface/LuminosityBlock.h"
0038 #include "DataFormats/Provenance/interface/LuminosityBlockID.h"
0039 #include "DataFormats/Provenance/interface/LuminosityBlockRange.h"
0040 #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
0041 
0042 #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
0043 #include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
0044 
0045 #include "FWCore/Framework/interface/InputSourceMacros.h"
0046 #include "FWCore/Framework/interface/FileBlock.h"
0047 
0048 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0049 #include "FWCore/MessageLogger/interface/JobReport.h"
0050 #include "FWCore/Utilities/interface/TimeOfDay.h"
0051 
0052 #include "format.h"
0053 
0054 // class rather than namespace so we can make this a friend of the
0055 // MonitorElement to get access to constructors etc.
0056 struct DQMTTreeIO {
0057   typedef dqm::harvesting::MonitorElement MonitorElement;
0058   typedef dqm::harvesting::DQMStore DQMStore;
0059 
0060   // TODO: this should probably be moved somewhere else
0061   class DQMMergeHelper {
0062   public:
0063     // Utility function to check the consistency of the axis labels
0064     // Taken from TH1::CheckBinLabels which is not public
0065     static bool CheckBinLabels(const TAxis* a1, const TAxis* a2) {
0066       // Check that axis have same labels
0067       THashList* l1 = (const_cast<TAxis*>(a1))->GetLabels();
0068       THashList* l2 = (const_cast<TAxis*>(a2))->GetLabels();
0069 
0070       if (!l1 && !l2)
0071         return true;
0072       if (!l1 || !l2) {
0073         return false;
0074       }
0075 
0076       // Check now labels sizes  are the same
0077       if (l1->GetSize() != l2->GetSize()) {
0078         return false;
0079       }
0080 
0081       for (int i = 1; i <= a1->GetNbins(); ++i) {
0082         std::string_view label1 = a1->GetBinLabel(i);
0083         std::string_view label2 = a2->GetBinLabel(i);
0084         if (label1 != label2) {
0085           return false;
0086         }
0087       }
0088 
0089       return true;
0090     }
0091 
0092     // NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h
0093     static void mergeTogether(TH1* original, TH1* toAdd) {
0094       if (original->CanExtendAllAxes() && toAdd->CanExtendAllAxes()) {
0095         TList list;
0096         list.Add(toAdd);
0097         if (original->Merge(&list) == -1) {
0098           edm::LogError("MergeFailure") << "Failed to merge DQM element " << original->GetName();
0099         }
0100       } else {
0101         // TODO: Redo. This is both more strict than what ROOT checks for yet
0102         // allows cases where ROOT fails with merging.
0103         if (original->GetNbinsX() == toAdd->GetNbinsX() &&
0104             original->GetXaxis()->GetXmin() == toAdd->GetXaxis()->GetXmin() &&
0105             original->GetXaxis()->GetXmax() == toAdd->GetXaxis()->GetXmax() &&
0106             original->GetNbinsY() == toAdd->GetNbinsY() &&
0107             original->GetYaxis()->GetXmin() == toAdd->GetYaxis()->GetXmin() &&
0108             original->GetYaxis()->GetXmax() == toAdd->GetYaxis()->GetXmax() &&
0109             original->GetNbinsZ() == toAdd->GetNbinsZ() &&
0110             original->GetZaxis()->GetXmin() == toAdd->GetZaxis()->GetXmin() &&
0111             original->GetZaxis()->GetXmax() == toAdd->GetZaxis()->GetXmax() &&
0112             CheckBinLabels(original->GetXaxis(), toAdd->GetXaxis()) &&
0113             CheckBinLabels(original->GetYaxis(), toAdd->GetYaxis()) &&
0114             CheckBinLabels(original->GetZaxis(), toAdd->GetZaxis())) {
0115           original->Add(toAdd);
0116         } else {
0117           edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels '"
0118                                         << original->GetName() << "' not merged.";
0119         }
0120       }
0121     }
0122   };
0123 
0124   // This struct allows to find all MEs belonging to a run-lumi pair
0125   // All files will be open at once so m_file property indicates the file where data is saved.
0126   struct FileMetadata {
0127     unsigned int m_run;
0128     unsigned int m_lumi;
0129     ULong64_t m_beginTime;
0130     ULong64_t m_endTime;
0131     ULong64_t m_firstIndex;
0132     ULong64_t m_lastIndex;  // Last is inclusive
0133     unsigned int m_type;
0134     TFile* m_file;
0135 
0136     // This will be used when sorting a vector
0137     bool operator<(const FileMetadata& obj) const {
0138       if (m_run == obj.m_run)
0139         return m_lumi < obj.m_lumi;
0140       else
0141         return m_run < obj.m_run;
0142     }
0143 
0144     void describe() {
0145       std::cout << "read r:" << m_run << " l:" << m_lumi << " bt:" << m_beginTime << " et:" << m_endTime
0146                 << " fi:" << m_firstIndex << " li:" << m_lastIndex << " type:" << m_type << " file: " << m_file
0147                 << std::endl;
0148     }
0149   };
0150 
0151   class TreeReaderBase {
0152   public:
0153     TreeReaderBase(MonitorElementData::Kind kind, MonitorElementData::Scope rescope)
0154         : m_kind(kind), m_rescope(rescope) {}
0155     virtual ~TreeReaderBase() {}
0156 
0157     MonitorElementData::Key makeKey(std::string const& fullname, int run, int lumi) {
0158       MonitorElementData::Key key;
0159       key.kind_ = m_kind;
0160       key.path_.set(fullname, MonitorElementData::Path::Type::DIR_AND_NAME);
0161       if (m_rescope == MonitorElementData::Scope::LUMI) {
0162         // no rescoping
0163         key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI;
0164         key.id_ = edm::LuminosityBlockID(run, lumi);
0165       } else if (m_rescope == MonitorElementData::Scope::RUN) {
0166         // everything becomes run, we'll never see Scope::JOB inside DQMIO files.
0167         key.scope_ = MonitorElementData::Scope::RUN;
0168         key.id_ = edm::LuminosityBlockID(run, 0);
0169       } else if (m_rescope == MonitorElementData::Scope::JOB) {
0170         // Everything is aggregated over the entire job.
0171         key.scope_ = MonitorElementData::Scope::JOB;
0172         key.id_ = edm::LuminosityBlockID(0, 0);
0173       } else {
0174         assert(!"Invalid Scope in rescope option.");
0175       }
0176       return key;
0177     }
0178     virtual void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) = 0;
0179     virtual void setTree(TTree* iTree) = 0;
0180 
0181   protected:
0182     MonitorElementData::Kind m_kind;
0183     MonitorElementData::Scope m_rescope;
0184   };
0185 
0186   template <class T>
0187   class TreeObjectReader : public TreeReaderBase {
0188   public:
0189     TreeObjectReader(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) : TreeReaderBase(kind, rescope) {
0190       assert(m_kind != MonitorElementData::Kind::INT);
0191       assert(m_kind != MonitorElementData::Kind::REAL);
0192       assert(m_kind != MonitorElementData::Kind::STRING);
0193     }
0194 
0195     void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) override {
0196       // This will populate the fields as defined in setTree method
0197       try {
0198         m_tree->GetEntry(iIndex);
0199 
0200         auto key = makeKey(*m_fullName, run, lumi);
0201         auto existing = dqmstore->findOrRecycle(key);
0202         if (existing) {
0203           // TODO: make sure there is sufficient locking here.
0204           DQMMergeHelper::mergeTogether(existing->getTH1(), m_buffer);
0205         } else {
0206           // We make our own MEs here, to avoid a round-trip through the booking API.
0207           MonitorElementData meData;
0208           meData.key_ = key;
0209           meData.value_.object_ = std::unique_ptr<T>((T*)(m_buffer->Clone()));
0210           auto me = new MonitorElement(std::move(meData));
0211           dqmstore->putME(me);
0212         }
0213       } catch (cms::Exception& iExcept) {
0214         using namespace std::string_literals;
0215         iExcept.addContext("failed while reading "s + *m_fullName);
0216         throw;
0217       }
0218     }
0219 
0220     void setTree(TTree* iTree) override {
0221       m_tree = iTree;
0222       m_tree->SetBranchAddress(kFullNameBranch, &m_fullName);
0223       m_tree->SetBranchAddress(kFlagBranch, &m_tag);
0224       m_tree->SetBranchAddress(kValueBranch, &m_buffer);
0225     }
0226 
0227   private:
0228     TTree* m_tree = nullptr;
0229     std::string* m_fullName = nullptr;
0230     T* m_buffer = nullptr;
0231     uint32_t m_tag = 0;
0232   };
0233 
0234   class TreeStringReader : public TreeReaderBase {
0235   public:
0236     TreeStringReader(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) : TreeReaderBase(kind, rescope) {
0237       assert(m_kind == MonitorElementData::Kind::STRING);
0238     }
0239 
0240     void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) override {
0241       // This will populate the fields as defined in setTree method
0242       m_tree->GetEntry(iIndex);
0243 
0244       auto key = makeKey(*m_fullName, run, lumi);
0245       auto existing = dqmstore->findOrRecycle(key);
0246 
0247       if (existing) {
0248         existing->Fill(*m_value);
0249       } else {
0250         // We make our own MEs here, to avoid a round-trip through the booking API.
0251         MonitorElementData meData;
0252         meData.key_ = key;
0253         meData.value_.scalar_.str = *m_value;
0254         auto me = new MonitorElement(std::move(meData));
0255         dqmstore->putME(me);
0256       }
0257     }
0258 
0259     void setTree(TTree* iTree) override {
0260       m_tree = iTree;
0261       m_tree->SetBranchAddress(kFullNameBranch, &m_fullName);
0262       m_tree->SetBranchAddress(kFlagBranch, &m_tag);
0263       m_tree->SetBranchAddress(kValueBranch, &m_value);
0264     }
0265 
0266   private:
0267     TTree* m_tree = nullptr;
0268     std::string* m_fullName = nullptr;
0269     std::string* m_value = nullptr;
0270     uint32_t m_tag = 0;
0271   };
0272 
0273   template <class T>
0274   class TreeSimpleReader : public TreeReaderBase {
0275   public:
0276     TreeSimpleReader(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) : TreeReaderBase(kind, rescope) {
0277       assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL);
0278     }
0279 
0280     void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) override {
0281       // This will populate the fields as defined in setTree method
0282       m_tree->GetEntry(iIndex);
0283 
0284       auto key = makeKey(*m_fullName, run, lumi);
0285       auto existing = dqmstore->findOrRecycle(key);
0286 
0287       if (existing) {
0288         existing->Fill(m_buffer);
0289       } else {
0290         // We make our own MEs here, to avoid a round-trip through the booking API.
0291         MonitorElementData meData;
0292         meData.key_ = key;
0293         if (m_kind == MonitorElementData::Kind::INT)
0294           meData.value_.scalar_.num = m_buffer;
0295         else if (m_kind == MonitorElementData::Kind::REAL)
0296           meData.value_.scalar_.real = m_buffer;
0297         auto me = new MonitorElement(std::move(meData));
0298         dqmstore->putME(me);
0299       }
0300     }
0301 
0302     void setTree(TTree* iTree) override {
0303       m_tree = iTree;
0304       m_tree->SetBranchAddress(kFullNameBranch, &m_fullName);
0305       m_tree->SetBranchAddress(kFlagBranch, &m_tag);
0306       m_tree->SetBranchAddress(kValueBranch, &m_buffer);
0307     }
0308 
0309   private:
0310     TTree* m_tree = nullptr;
0311     std::string* m_fullName = nullptr;
0312     T m_buffer = 0;
0313     uint32_t m_tag = 0;
0314   };
0315 };
0316 
0317 class DQMRootSource : public edm::PuttableSourceBase, DQMTTreeIO {
0318 public:
0319   DQMRootSource(edm::ParameterSet const&, const edm::InputSourceDescription&);
0320   DQMRootSource(const DQMRootSource&) = delete;
0321   ~DQMRootSource() override;
0322 
0323   // ---------- const member functions ---------------------
0324 
0325   const DQMRootSource& operator=(const DQMRootSource&) = delete;  // stop default
0326 
0327   // ---------- static member functions --------------------
0328 
0329   // ---------- member functions ---------------------------
0330   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
0331 
0332 private:
0333   edm::InputSource::ItemTypeInfo getNextItemType() override;
0334 
0335   std::shared_ptr<edm::FileBlock> readFile_() override;
0336   std::shared_ptr<edm::RunAuxiliary> readRunAuxiliary_() override;
0337   std::shared_ptr<edm::LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_() override;
0338   void readRun_(edm::RunPrincipal& rpCache) override;
0339   void readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) override;
0340   void readEvent_(edm::EventPrincipal&) override;
0341 
0342   // Read MEs from m_fileMetadatas to DQMStore  till run or lumi transition
0343   void readElements();
0344   // True if m_currentIndex points to an element that has a different
0345   // run or lumi than the previous element (a transition needs to happen).
0346   // False otherwise.
0347   bool isRunOrLumiTransition() const;
0348   void readNextItemType();
0349 
0350   // These methods will be called by the framework.
0351   // MEs in DQMStore  will be put to products.
0352   void beginRun(edm::Run& run) override;
0353   void beginLuminosityBlock(edm::LuminosityBlock& lumi) override;
0354 
0355   // If the run matches the filterOnRun configuration parameter, the run
0356   // (and all its lumis) will be kept.
0357   // Otherwise, check if a run and a lumi are in the range that needs to be processed.
0358   // Range is retrieved from lumisToProcess configuration parameter.
0359   // If at least one lumi of a run needs to be kept, per run MEs of that run will also be kept.
0360   bool keepIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const;
0361   void logFileAction(char const* msg, char const* fileName) const;
0362 
0363   // ---------- member data --------------------------------
0364 
0365   // Properties from python config
0366   bool m_skipBadFiles;
0367   unsigned int m_filterOnRun;
0368   edm::InputFileCatalog m_catalog;
0369   std::vector<edm::LuminosityBlockRange> m_lumisToProcess;
0370   MonitorElementData::Scope m_rescope;
0371 
0372   edm::InputSource::ItemType m_nextItemType;
0373   // Each ME type gets its own reader
0374   std::vector<std::shared_ptr<TreeReaderBase>> m_treeReaders;
0375 
0376   // Index of currenlty processed row in m_fileMetadatas
0377   unsigned int m_currentIndex;
0378 
0379   // All open DQMIO files
0380   struct OpenFileInfo {
0381     OpenFileInfo(TFile* file, edm::JobReport::Token jrToken) : m_file(file), m_jrToken(jrToken) {}
0382     ~OpenFileInfo() {
0383       edm::Service<edm::JobReport> jr;
0384       jr->inputFileClosed(edm::InputType::Primary, m_jrToken);
0385     }
0386 
0387     OpenFileInfo(OpenFileInfo&&) = default;
0388     OpenFileInfo& operator=(OpenFileInfo&&) = default;
0389 
0390     std::unique_ptr<TFile> m_file;
0391     edm::JobReport::Token m_jrToken;
0392   };
0393   std::vector<OpenFileInfo> m_openFiles;
0394 
0395   // An item here is a row read from DQMIO indices (metadata) table
0396   std::vector<FileMetadata> m_fileMetadatas;
0397 };
0398 
0399 //
0400 // constants, enums and typedefs
0401 //
0402 
0403 //
0404 // static data member definitions
0405 //
0406 
0407 void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
0408   edm::ParameterSetDescription desc;
0409   desc.addUntracked<std::vector<std::string>>("fileNames")->setComment("Names of files to be processed.");
0410   desc.addUntracked<unsigned int>("filterOnRun", 0)->setComment("Just limit the process to the selected run.");
0411   desc.addUntracked<std::string>("reScope", "JOB")
0412       ->setComment(
0413           "Accumulate histograms more coarsely."
0414           " Options: \"\": keep unchanged, \"RUN\": turn LUMI histograms into RUN histograms, \"JOB\": turn everything "
0415           "into JOB histograms.");
0416   desc.addUntracked<bool>("skipBadFiles", false)->setComment("Skip the file if it is not valid");
0417   desc.addUntracked<std::string>("overrideCatalog", std::string())
0418       ->setComment("An alternate file catalog to use instead of the standard site one.");
0419   std::vector<edm::LuminosityBlockRange> defaultLumis;
0420   desc.addUntracked<std::vector<edm::LuminosityBlockRange>>("lumisToProcess", defaultLumis)
0421       ->setComment("Skip any lumi inside the specified run:lumi range.");
0422 
0423   descriptions.addDefault(desc);
0424 }
0425 
0426 //
0427 // constructors and destructor
0428 //
0429 
0430 DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSourceDescription& iDesc)
0431     : edm::PuttableSourceBase(iPSet, iDesc),
0432       m_skipBadFiles(iPSet.getUntrackedParameter<bool>("skipBadFiles", false)),
0433       m_filterOnRun(iPSet.getUntrackedParameter<unsigned int>("filterOnRun", 0)),
0434       m_catalog(iPSet.getUntrackedParameter<std::vector<std::string>>("fileNames"),
0435                 iPSet.getUntrackedParameter<std::string>("overrideCatalog")),
0436       m_lumisToProcess(iPSet.getUntrackedParameter<std::vector<edm::LuminosityBlockRange>>(
0437           "lumisToProcess", std::vector<edm::LuminosityBlockRange>())),
0438       m_rescope(std::map<std::string, MonitorElementData::Scope>{
0439           {"", MonitorElementData::Scope::LUMI},
0440           {"LUMI", MonitorElementData::Scope::LUMI},
0441           {"RUN", MonitorElementData::Scope::RUN},
0442           {"JOB", MonitorElementData::Scope::JOB}}[iPSet.getUntrackedParameter<std::string>("reScope", "JOB")]),
0443       m_nextItemType(edm::InputSource::ItemType::IsFile),
0444       m_treeReaders(kNIndicies, std::shared_ptr<TreeReaderBase>()),
0445       m_currentIndex(0),
0446       m_openFiles(std::vector<OpenFileInfo>()),
0447       m_fileMetadatas(std::vector<FileMetadata>()) {
0448   edm::sortAndRemoveOverlaps(m_lumisToProcess);
0449 
0450   if (m_catalog.fileNames(0).empty()) {
0451     m_nextItemType = edm::InputSource::ItemType::IsStop;
0452   } else {
0453     m_treeReaders[kIntIndex] = std::make_shared<TreeSimpleReader<Long64_t>>(MonitorElementData::Kind::INT, m_rescope);
0454     m_treeReaders[kFloatIndex] = std::make_shared<TreeSimpleReader<double>>(MonitorElementData::Kind::REAL, m_rescope);
0455     m_treeReaders[kStringIndex] = std::make_shared<TreeStringReader>(MonitorElementData::Kind::STRING, m_rescope);
0456     m_treeReaders[kTH1FIndex] = std::make_shared<TreeObjectReader<TH1F>>(MonitorElementData::Kind::TH1F, m_rescope);
0457     m_treeReaders[kTH1SIndex] = std::make_shared<TreeObjectReader<TH1S>>(MonitorElementData::Kind::TH1S, m_rescope);
0458     m_treeReaders[kTH1DIndex] = std::make_shared<TreeObjectReader<TH1D>>(MonitorElementData::Kind::TH1D, m_rescope);
0459     m_treeReaders[kTH1IIndex] = std::make_shared<TreeObjectReader<TH1I>>(MonitorElementData::Kind::TH1I, m_rescope);
0460     m_treeReaders[kTH2FIndex] = std::make_shared<TreeObjectReader<TH2F>>(MonitorElementData::Kind::TH2F, m_rescope);
0461     m_treeReaders[kTH2SIndex] = std::make_shared<TreeObjectReader<TH2S>>(MonitorElementData::Kind::TH2S, m_rescope);
0462     m_treeReaders[kTH2DIndex] = std::make_shared<TreeObjectReader<TH2D>>(MonitorElementData::Kind::TH2D, m_rescope);
0463     m_treeReaders[kTH2PolyIndex] =
0464         std::make_shared<TreeObjectReader<TH2Poly>>(MonitorElementData::Kind::TH2Poly, m_rescope);
0465     m_treeReaders[kTH2IIndex] = std::make_shared<TreeObjectReader<TH2I>>(MonitorElementData::Kind::TH2I, m_rescope);
0466     m_treeReaders[kTH3FIndex] = std::make_shared<TreeObjectReader<TH3F>>(MonitorElementData::Kind::TH3F, m_rescope);
0467     m_treeReaders[kTProfileIndex] =
0468         std::make_shared<TreeObjectReader<TProfile>>(MonitorElementData::Kind::TPROFILE, m_rescope);
0469     m_treeReaders[kTProfile2DIndex] =
0470         std::make_shared<TreeObjectReader<TProfile2D>>(MonitorElementData::Kind::TPROFILE2D, m_rescope);
0471   }
0472 
0473   produces<DQMToken, edm::Transition::BeginRun>("DQMGenerationRecoRun");
0474   produces<DQMToken, edm::Transition::BeginLuminosityBlock>("DQMGenerationRecoLumi");
0475 }
0476 
0477 DQMRootSource::~DQMRootSource() {
0478   for (auto& file : m_openFiles) {
0479     if (file.m_file && file.m_file->IsOpen()) {
0480       logFileAction("Closed file", "");
0481     }
0482   }
0483 }
0484 
0485 //
0486 // member functions
0487 //
0488 
0489 edm::InputSource::ItemTypeInfo DQMRootSource::getNextItemType() { return m_nextItemType; }
0490 
0491 // We will read the metadata of all files and fill m_fileMetadatas vector
0492 std::shared_ptr<edm::FileBlock> DQMRootSource::readFile_() {
0493   const int numFiles = m_catalog.fileNames(0).size();
0494   m_openFiles.reserve(numFiles);
0495 
0496   for (auto& fileitem : m_catalog.fileCatalogItems()) {
0497     TFile* file = nullptr;
0498     std::string pfn;
0499     std::string lfn;
0500     std::list<std::string> exInfo;
0501     //loop over names of a file, each of them corresponds to a data catalog
0502     bool isGoodFile(true);
0503     //get all names of a file, each of them corresponds to a data catalog
0504     const std::vector<std::string>& fNames = fileitem.fileNames();
0505     for (std::vector<std::string>::const_iterator it = fNames.begin(); it != fNames.end(); ++it) {
0506       // Try to open a file
0507       try {
0508         file = TFile::Open(it->c_str());
0509 
0510         // Exception will be trapped so we pull it out ourselves
0511         std::exception_ptr e = edm::threadLocalException::getException();
0512         if (e != std::exception_ptr()) {
0513           edm::threadLocalException::setException(std::exception_ptr());
0514           std::rethrow_exception(e);
0515         }
0516 
0517       } catch (cms::Exception const& e) {
0518         file = nullptr;                       // is there anything we need to free?
0519         if (std::next(it) == fNames.end()) {  //last name corresponding to the last data catalog to try
0520           if (!m_skipBadFiles) {
0521             edm::Exception ex(edm::errors::FileOpenError, "", e);
0522             ex.addContext("Opening DQM Root file");
0523             ex << "\nInput file " << *it << " was not found, could not be opened, or is corrupted.\n";
0524             //report previous exceptions when use other names to open file
0525             for (auto const& s : exInfo)
0526               ex.addAdditionalInfo(s);
0527             throw ex;
0528           }
0529           isGoodFile = false;
0530         }
0531         // save in case of error when trying next name
0532         for (auto const& s : e.additionalInfo())
0533           exInfo.push_back(s);
0534       }
0535 
0536       // Check if a file is usable
0537       if (file && !file->IsZombie()) {
0538         logFileAction("Successfully opened file ", it->c_str());
0539         pfn = *it;
0540         lfn = fileitem.logicalFileName();
0541         break;
0542       } else {
0543         if (std::next(it) == fNames.end()) {
0544           if (!m_skipBadFiles) {
0545             edm::Exception ex(edm::errors::FileOpenError);
0546             ex << "Input file " << *it << " could not be opened.\n";
0547             ex.addContext("Opening DQM Root file");
0548             //report previous exceptions when use other names to open file
0549             for (auto const& s : exInfo)
0550               ex.addAdditionalInfo(s);
0551             throw ex;
0552           }
0553           isGoodFile = false;
0554         }
0555         if (file) {
0556           delete file;
0557           file = nullptr;
0558         }
0559       }
0560     }  //end loop over names of the file
0561 
0562     if (!file || (!isGoodFile && m_skipBadFiles))
0563       continue;
0564 
0565     std::unique_ptr<std::string> guid{file->Get<std::string>(kCmsGuid)};
0566     if (not guid) {
0567       guid = std::make_unique<std::string>(file->GetUUID().AsString());
0568       std::transform(guid->begin(), guid->end(), guid->begin(), (int (*)(int))std::toupper);
0569     }
0570 
0571     edm::Service<edm::JobReport> jr;
0572     auto jrToken = jr->inputFileOpened(
0573         pfn, lfn, std::string(), std::string(), "DQMRootSource", "source", *guid, std::vector<std::string>());
0574     m_openFiles.emplace_back(file, jrToken);
0575 
0576     // Check file format version, which is encoded in the Title of the TFile
0577     if (strcmp(file->GetTitle(), "1") != 0) {
0578       edm::Exception ex(edm::errors::FileReadError);
0579       ex << "Input file " << fNames[0] << " does not appear to be a DQM Root file.\n";
0580     }
0581 
0582     // Read metadata from the file
0583     TTree* indicesTree = dynamic_cast<TTree*>(file->Get(kIndicesTree));
0584     assert(indicesTree != nullptr);
0585 
0586     FileMetadata temp;
0587     // Each line of metadata will be read into the coresponding fields of temp.
0588     indicesTree->SetBranchAddress(kRunBranch, &temp.m_run);
0589     indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi);
0590     indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime);
0591     indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime);
0592     indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type);
0593     indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex);
0594     indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex);
0595 
0596     for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) {
0597       indicesTree->GetEntry(index);
0598       temp.m_file = file;
0599 
0600       if (keepIt(temp.m_run, temp.m_lumi)) {
0601         m_fileMetadatas.push_back(temp);
0602       }
0603     }
0604 
0605   }  //end loop over files
0606 
0607   // Sort to make sure runs and lumis appear in sequential order
0608   std::stable_sort(m_fileMetadatas.begin(), m_fileMetadatas.end());
0609 
0610   // If we have lumisections without matching runs, insert dummy runs here.
0611   unsigned int run = 0;
0612   auto toadd = std::vector<FileMetadata>();
0613   for (auto& metadata : m_fileMetadatas) {
0614     if (run < metadata.m_run && metadata.m_lumi != 0) {
0615       // run transition and lumi transition at the same time!
0616       FileMetadata dummy{};  // zero initialize
0617       dummy.m_run = metadata.m_run;
0618       dummy.m_lumi = 0;
0619       dummy.m_type = kNoTypesStored;
0620       toadd.push_back(dummy);
0621     }
0622     run = metadata.m_run;
0623   }
0624 
0625   if (!toadd.empty()) {
0626     // rather than trying to insert at the right places, just append and sort again.
0627     m_fileMetadatas.insert(m_fileMetadatas.end(), toadd.begin(), toadd.end());
0628     std::stable_sort(m_fileMetadatas.begin(), m_fileMetadatas.end());
0629   }
0630 
0631   //for (auto& metadata : m_fileMetadatas)
0632   //  metadata.describe();
0633 
0634   // Stop if there's nothing to process. Otherwise start the run.
0635   if (m_fileMetadatas.empty())
0636     m_nextItemType = edm::InputSource::ItemType::IsStop;
0637   else
0638     m_nextItemType = edm::InputSource::ItemType::IsRun;
0639 
0640   // We have to return something but not sure why
0641   return std::make_shared<edm::FileBlock>();
0642 }
0643 
0644 std::shared_ptr<edm::RunAuxiliary> DQMRootSource::readRunAuxiliary_() {
0645   FileMetadata metadata = m_fileMetadatas[m_currentIndex];
0646   auto runAux =
0647       edm::RunAuxiliary(metadata.m_run, edm::Timestamp(metadata.m_beginTime), edm::Timestamp(metadata.m_endTime));
0648   return std::make_shared<edm::RunAuxiliary>(runAux);
0649 }
0650 
0651 std::shared_ptr<edm::LuminosityBlockAuxiliary> DQMRootSource::readLuminosityBlockAuxiliary_() {
0652   FileMetadata metadata = m_fileMetadatas[m_currentIndex];
0653   auto lumiAux = edm::LuminosityBlockAuxiliary(edm::LuminosityBlockID(metadata.m_run, metadata.m_lumi),
0654                                                edm::Timestamp(metadata.m_beginTime),
0655                                                edm::Timestamp(metadata.m_endTime));
0656   return std::make_shared<edm::LuminosityBlockAuxiliary>(lumiAux);
0657 }
0658 
0659 void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) {
0660   // Read elements of a current run.
0661   do {
0662     FileMetadata metadata = m_fileMetadatas[m_currentIndex];
0663     if (metadata.m_lumi == 0) {
0664       readElements();
0665     }
0666     m_currentIndex++;
0667   } while (!isRunOrLumiTransition());
0668 
0669   readNextItemType();
0670 
0671   edm::Service<edm::JobReport> jr;
0672   jr->reportInputRunNumber(rpCache.id().run());
0673   rpCache.fillRunPrincipal(processHistoryRegistryForUpdate());
0674 }
0675 
0676 void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) {
0677   // Read elements of a current lumi.
0678   do {
0679     readElements();
0680     m_currentIndex++;
0681   } while (!isRunOrLumiTransition());
0682 
0683   readNextItemType();
0684 
0685   edm::Service<edm::JobReport> jr;
0686   jr->reportInputLumiSection(lbCache.id().run(), lbCache.id().luminosityBlock());
0687   lbCache.fillLuminosityBlockPrincipal(processHistoryRegistry().getMapped(lbCache.aux().processHistoryID()));
0688 }
0689 
0690 void DQMRootSource::readEvent_(edm::EventPrincipal&) {}
0691 
0692 void DQMRootSource::readElements() {
0693   FileMetadata metadata = m_fileMetadatas[m_currentIndex];
0694 
0695   if (metadata.m_type != kNoTypesStored) {
0696     std::shared_ptr<TreeReaderBase> reader = m_treeReaders[metadata.m_type];
0697     TTree* tree = dynamic_cast<TTree*>(metadata.m_file->Get(kTypeNames[metadata.m_type]));
0698     // The Reset() below screws up the tree, so we need to re-read it from file
0699     // before use here.
0700     tree->Refresh();
0701 
0702     reader->setTree(tree);
0703 
0704     ULong64_t index = metadata.m_firstIndex;
0705     ULong64_t endIndex = metadata.m_lastIndex + 1;
0706 
0707     for (; index != endIndex; ++index) {
0708       reader->read(index, edm::Service<DQMStore>().operator->(), metadata.m_run, metadata.m_lumi);
0709     }
0710     // Drop buffers in the TTree. This reduces memory consuption while the tree
0711     // just sits there and waits for the next block to be read.
0712     tree->Reset();
0713   }
0714 }
0715 
0716 bool DQMRootSource::isRunOrLumiTransition() const {
0717   if (m_currentIndex == 0) {
0718     return false;
0719   }
0720 
0721   if (m_currentIndex > m_fileMetadatas.size() - 1) {
0722     // We reached the end
0723     return true;
0724   }
0725 
0726   FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1];
0727   FileMetadata metadata = m_fileMetadatas[m_currentIndex];
0728 
0729   return previousMetadata.m_run != metadata.m_run || previousMetadata.m_lumi != metadata.m_lumi;
0730 }
0731 
0732 void DQMRootSource::readNextItemType() {
0733   if (m_currentIndex == 0) {
0734     m_nextItemType = edm::InputSource::ItemType::IsRun;
0735   } else if (m_currentIndex > m_fileMetadatas.size() - 1) {
0736     // We reached the end
0737     m_nextItemType = edm::InputSource::ItemType::IsStop;
0738   } else {
0739     FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1];
0740     FileMetadata metadata = m_fileMetadatas[m_currentIndex];
0741 
0742     if (previousMetadata.m_run != metadata.m_run) {
0743       m_nextItemType = edm::InputSource::ItemType::IsRun;
0744     } else if (previousMetadata.m_lumi != metadata.m_lumi) {
0745       m_nextItemType = edm::InputSource::ItemType::IsLumi;
0746     }
0747   }
0748 }
0749 
0750 void DQMRootSource::beginRun(edm::Run& run) {
0751   std::unique_ptr<DQMToken> product = std::make_unique<DQMToken>();
0752   run.put(std::move(product), "DQMGenerationRecoRun");
0753 }
0754 
0755 void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) {
0756   std::unique_ptr<DQMToken> product = std::make_unique<DQMToken>();
0757   lumi.put(std::move(product), "DQMGenerationRecoLumi");
0758 }
0759 
0760 bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const {
0761   if (m_filterOnRun != 0 && run != m_filterOnRun) {
0762     return false;
0763   }
0764 
0765   if (m_lumisToProcess.empty()) {
0766     return true;
0767   }
0768 
0769   for (edm::LuminosityBlockRange const& lumiToProcess : m_lumisToProcess) {
0770     if (run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) {
0771       if (lumi >= lumiToProcess.startLumi() && lumi <= lumiToProcess.endLumi()) {
0772         return true;
0773       } else if (lumi == 0) {
0774         return true;
0775       }
0776     }
0777   }
0778   return false;
0779 }
0780 
0781 void DQMRootSource::logFileAction(char const* msg, char const* fileName) const {
0782   edm::LogAbsolute("fileAction") << std::setprecision(0) << edm::TimeOfDay() << msg << fileName;
0783   edm::FlushMessageLog();
0784 }
0785 
0786 //
0787 // const member functions
0788 //
0789 
0790 //
0791 // static member functions
0792 //
0793 DEFINE_FWK_INPUT_SOURCE(DQMRootSource);
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!