Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-12-24 02:18:41

0001 #ifndef FWCore_Framework_MergeableRunProductMetadata_h
0002 #define FWCore_Framework_MergeableRunProductMetadata_h
0003 
0004 /** \class edm::MergeableRunProductMetadata
0005 
0006 This class holds information used to decide how to merge together
0007 mergeable run products when multiple run entries with the same
0008 run number and ProcessHistoryID are read from input files
0009 contiguously.
0010 
0011 Most of the information here is associated with the current
0012 run being processed. Most of it is cleared when a new run
0013 is started. If multiple runs are being processed concurrently,
0014 then there will be an object instantiated for each concurrent
0015 run. (At the present time concurrent runs are not possible, but
0016 there plans to implement that in the future). The primary
0017 RunPrincipal for the current run owns the object.
0018 
0019 This class gets information from the input file from the
0020 StoredMergeableRunProductMetadata object and IndexIntoFile object.
0021 It uses that information to make the decision how to merge
0022 run products read from data, and puts information into the
0023 StoredMergeableRunProductMetadata written into the output file
0024 to use in later processing steps.
0025 
0026 If there are SubProcesses, they use the same object as the top
0027 level process because they share the same input.
0028 
0029 There will be a TWIKI page on the Framework page of the Software
0030 Guide which explains the details about how this works. There
0031 are significant limitations related to what the Framework does
0032 and does not do managing mergeable run products.
0033 
0034 \author W. David Dagenhart, created 9 July, 2018
0035 
0036 */
0037 
0038 #include "DataFormats/Provenance/interface/MergeableRunProductMetadataBase.h"
0039 #include "FWCore/Framework/interface/MergeableRunProductProcesses.h"
0040 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0041 
0042 #include "oneapi/tbb/concurrent_vector.h"
0043 
0044 #include <string>
0045 #include <vector>
0046 
0047 namespace edm {
0048 
0049   class IndexIntoFileItrHolder;
0050   class StoredMergeableRunProductMetadata;
0051 
0052   class MergeableRunProductMetadata : public MergeableRunProductMetadataBase {
0053   public:
0054     enum MergeDecision { MERGE, REPLACE, IGNORE };
0055 
0056     MergeableRunProductMetadata(MergeableRunProductProcesses const&);
0057 
0058     ~MergeableRunProductMetadata() override;
0059 
0060     // Called each time a new input file is opened
0061     void preReadFile();
0062 
0063     // Called each time a run entry is read from an input file. This
0064     // should be called before the run products themselves are read
0065     // because it sets the decision whether to merge, replace, or ignore
0066     // run products as they read.
0067     void readRun(long long inputRunEntry,
0068                  StoredMergeableRunProductMetadata const& inputStoredMergeableRunProductMetadata,
0069                  IndexIntoFileItrHolder const& inputIndexIntoFileItr);
0070 
0071     // Called to record which lumis were processed by the current run
0072     void writeLumi(LuminosityBlockNumber_t lumi);
0073 
0074     void preWriteRun();
0075     void postWriteRun();
0076 
0077     void addEntryToStoredMetadata(StoredMergeableRunProductMetadata&) const;
0078 
0079     MergeDecision getMergeDecision(std::string const& processThatCreatedProduct) const;
0080 
0081     // If Runs were split on lumi boundaries, but then the files were merged
0082     // in a way that made it impossible to properly merge run products, this
0083     // function will return true. The Framework usually does not know
0084     // enough to detect other cases where there is a merging problem.
0085     bool knownImproperlyMerged(std::string const& processThatCreatedProduct) const override;
0086 
0087     std::string const& getProcessName(unsigned int index) const {
0088       return mergeableRunProductProcesses_->getProcessName(index);
0089     }
0090 
0091     class MetadataForProcess {
0092     public:
0093       MetadataForProcess() = default;
0094 
0095       std::vector<LuminosityBlockNumber_t>& lumis() { return lumis_; }
0096       std::vector<LuminosityBlockNumber_t> const& lumis() const { return lumis_; }
0097 
0098       MergeDecision mergeDecision() const { return mergeDecision_; }
0099       void setMergeDecision(MergeDecision v) { mergeDecision_ = v; }
0100 
0101       bool valid() const { return valid_; }
0102       void setValid(bool v) { valid_ = v; }
0103 
0104       bool useIndexIntoFile() const { return useIndexIntoFile_; }
0105       void setUseIndexIntoFile(bool v) { useIndexIntoFile_ = v; }
0106 
0107       bool allLumisProcessed() const { return allLumisProcessed_; }
0108       void setAllLumisProcessed(bool v) { allLumisProcessed_ = v; }
0109 
0110       void reset();
0111 
0112     private:
0113       std::vector<LuminosityBlockNumber_t> lumis_;
0114       MergeDecision mergeDecision_ = MERGE;
0115       bool valid_ = true;
0116       bool useIndexIntoFile_ = false;
0117       bool allLumisProcessed_ = false;
0118     };
0119 
0120     MetadataForProcess const* metadataForOneProcess(std::string const& processName) const;
0121 
0122     // The next few functions are only intended to be used in tests.
0123     std::vector<MetadataForProcess> const& metadataForProcesses() const { return metadataForProcesses_; }
0124 
0125     std::vector<LuminosityBlockNumber_t> const& lumisFromIndexIntoFile() const { return lumisFromIndexIntoFile_; }
0126 
0127     bool gotLumisFromIndexIntoFile() const { return gotLumisFromIndexIntoFile_; }
0128 
0129     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> const& lumisProcessed() const { return lumisProcessed_; }
0130 
0131   private:
0132     void mergeLumisFromIndexIntoFile();
0133 
0134     bool addProcess(StoredMergeableRunProductMetadata& storedMetadata,
0135                     MetadataForProcess const& metadataForProcess,
0136                     unsigned int storedProcessIndex,
0137                     unsigned long long beginProcess,
0138                     unsigned long long endProcess) const;
0139 
0140     MergeableRunProductProcesses const* mergeableRunProductProcesses_;
0141 
0142     // This vector has an entry for each process that has a
0143     // mergeable run product in the input. It has an exact one to
0144     // correspondence with mergeableRunProductProcesses_ and
0145     // is in the same order.
0146     std::vector<MetadataForProcess> metadataForProcesses_;
0147 
0148     std::vector<LuminosityBlockNumber_t> lumisFromIndexIntoFile_;
0149     bool gotLumisFromIndexIntoFile_ = false;
0150 
0151     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> lumisProcessed_;
0152   };
0153 }  // namespace edm
0154 #endif