Framework/interface/MergeableRunProductMetadata.h

0001 #ifndef FWCore_Framework_MergeableRunProductMetadata_h
0002 #define FWCore_Framework_MergeableRunProductMetadata_h
0003
0004 /** \class edm::MergeableRunProductMetadata
0005
0006 This class holds information used to decide how to merge together
0007 mergeable run products when multiple run entries with the same
0008 run number and ProcessHistoryID are read from input files
0009 contiguously.
0010
0011 Most of the information here is associated with the current
0012 run being processed. Most of it is cleared when a new run
0013 is started. If multiple runs are being processed concurrently,
0014 then there will be an object instantiated for each concurrent
0015 run. The primary RunPrincipal for the current run owns the
0016 object.
0017
0018 This class gets information from the input file from the
0019 StoredMergeableRunProductMetadata object and IndexIntoFile object.
0020 It uses that information to make the decision how to merge
0021 run products read from data, and puts information into the
0022 StoredMergeableRunProductMetadata written into the output file
0023 to use in later processing steps.
0024
0025 There is a TWIKI page on the Framework page of the Software
0026 Guide which explains the details about how this works. There
0027 are significant limitations related to what the Framework does
0028 and does not do managing mergeable run products.
0029
0030 \author W. David Dagenhart, created 9 July, 2018
0031
0032 */
0033
0034 #include "DataFormats/Provenance/interface/MergeableRunProductMetadataBase.h"
0035 #include "FWCore/Framework/interface/MergeableRunProductProcesses.h"
0036 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0037
0038 #include "oneapi/tbb/concurrent_vector.h"
0039
0040 #include <string>
0041 #include <vector>
0042
0043 namespace edm {
0044
0045   class IndexIntoFileItrHolder;
0046   class StoredMergeableRunProductMetadata;
0047
0048   class MergeableRunProductMetadata : public MergeableRunProductMetadataBase {
0049   public:
0050     enum MergeDecision { MERGE, REPLACE, IGNORE };
0051
0052     MergeableRunProductMetadata(MergeableRunProductProcesses const&);
0053
0054     ~MergeableRunProductMetadata() override;
0055
0056     // Called each time a new input file is opened
0057     void preReadFile();
0058
0059     // Called each time a run entry is read from an input file. This
0060     // should be called before the run products themselves are read
0061     // because it sets the decision whether to merge, replace, or ignore
0062     // run products as they read.
0063     void readRun(long long inputRunEntry,
0064                  StoredMergeableRunProductMetadata const& inputStoredMergeableRunProductMetadata,
0065                  IndexIntoFileItrHolder const& inputIndexIntoFileItr);
0066
0067     // Called to record which lumis were processed by the current run
0068     void writeLumi(LuminosityBlockNumber_t lumi);
0069
0070     void preWriteRun();
0071     void postWriteRun();
0072
0073     void addEntryToStoredMetadata(StoredMergeableRunProductMetadata&) const;
0074
0075     MergeDecision getMergeDecision(std::string const& processThatCreatedProduct) const;
0076
0077     // If Runs were split on lumi boundaries, but then the files were merged
0078     // in a way that made it impossible to properly merge run products, this
0079     // function will return true. The Framework usually does not know
0080     // enough to detect other cases where there is a merging problem.
0081     bool knownImproperlyMerged(std::string const& processThatCreatedProduct) const override;
0082
0083     std::string const& getProcessName(unsigned int index) const {
0084       return mergeableRunProductProcesses_->getProcessName(index);
0085     }
0086
0087     class MetadataForProcess {
0088     public:
0089       MetadataForProcess() = default;
0090
0091       std::vector<LuminosityBlockNumber_t>& lumis() { return lumis_; }
0092       std::vector<LuminosityBlockNumber_t> const& lumis() const { return lumis_; }
0093
0094       MergeDecision mergeDecision() const { return mergeDecision_; }
0095       void setMergeDecision(MergeDecision v) { mergeDecision_ = v; }
0096
0097       bool valid() const { return valid_; }
0098       void setValid(bool v) { valid_ = v; }
0099
0100       bool useIndexIntoFile() const { return useIndexIntoFile_; }
0101       void setUseIndexIntoFile(bool v) { useIndexIntoFile_ = v; }
0102
0103       bool allLumisProcessed() const { return allLumisProcessed_; }
0104       void setAllLumisProcessed(bool v) { allLumisProcessed_ = v; }
0105
0106       void reset();
0107
0108     private:
0109       std::vector<LuminosityBlockNumber_t> lumis_;
0110       MergeDecision mergeDecision_ = MERGE;
0111       bool valid_ = true;
0112       bool useIndexIntoFile_ = false;
0113       bool allLumisProcessed_ = false;
0114     };
0115
0116     MetadataForProcess const* metadataForOneProcess(std::string const& processName) const;
0117
0118     // The next few functions are only intended to be used in tests.
0119     std::vector<MetadataForProcess> const& metadataForProcesses() const { return metadataForProcesses_; }
0120
0121     std::vector<LuminosityBlockNumber_t> const& lumisFromIndexIntoFile() const { return lumisFromIndexIntoFile_; }
0122
0123     bool gotLumisFromIndexIntoFile() const { return gotLumisFromIndexIntoFile_; }
0124
0125     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> const& lumisProcessed() const { return lumisProcessed_; }
0126
0127   private:
0128     void mergeLumisFromIndexIntoFile();
0129
0130     bool addProcess(StoredMergeableRunProductMetadata& storedMetadata,
0131                     MetadataForProcess const& metadataForProcess,
0132                     unsigned int storedProcessIndex,
0133                     unsigned long long beginProcess,
0134                     unsigned long long endProcess) const;
0135
0136     MergeableRunProductProcesses const* mergeableRunProductProcesses_;
0137
0138     // This vector has an entry for each process that has a
0139     // mergeable run product in the input. It has an exact one to
0140     // correspondence with mergeableRunProductProcesses_ and
0141     // is in the same order.
0142     std::vector<MetadataForProcess> metadataForProcesses_;
0143
0144     std::vector<LuminosityBlockNumber_t> lumisFromIndexIntoFile_;
0145     bool gotLumisFromIndexIntoFile_ = false;
0146
0147     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> lumisProcessed_;
0148   };
0149 }  // namespace edm
0150 #endif