Framework/interface/MergeableRunProductMetadata.h

0001 #ifndef FWCore_Framework_MergeableRunProductMetadata_h
0002 #define FWCore_Framework_MergeableRunProductMetadata_h
0003
0004 /** \class edm::MergeableRunProductMetadata
0005
0006 This class holds information used to decide how to merge together
0007 mergeable run products when multiple run entries with the same
0008 run number and ProcessHistoryID are read from input files
0009 contiguously.
0010
0011 Most of the information here is associated with the current
0012 run being processed. Most of it is cleared when a new run
0013 is started. If multiple runs are being processed concurrently,
0014 then there will be an object instantiated for each concurrent
0015 run. The primary RunPrincipal for the current run owns the
0016 object.
0017
0018 This class gets information from the input file from the
0019 StoredMergeableRunProductMetadata object and IndexIntoFile object.
0020 It uses that information to make the decision how to merge
0021 run products read from data, and puts information into the
0022 StoredMergeableRunProductMetadata written into the output file
0023 to use in later processing steps.
0024
0025 If there are SubProcesses, they use the same object as the top
0026 level process because they share the same input.
0027
0028 There is a TWIKI page on the Framework page of the Software
0029 Guide which explains the details about how this works. There
0030 are significant limitations related to what the Framework does
0031 and does not do managing mergeable run products.
0032
0033 \author W. David Dagenhart, created 9 July, 2018
0034
0035 */
0036
0037 #include "DataFormats/Provenance/interface/MergeableRunProductMetadataBase.h"
0038 #include "FWCore/Framework/interface/MergeableRunProductProcesses.h"
0039 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0040
0041 #include "oneapi/tbb/concurrent_vector.h"
0042
0043 #include <string>
0044 #include <vector>
0045
0046 namespace edm {
0047
0048   class IndexIntoFileItrHolder;
0049   class StoredMergeableRunProductMetadata;
0050
0051   class MergeableRunProductMetadata : public MergeableRunProductMetadataBase {
0052   public:
0053     enum MergeDecision { MERGE, REPLACE, IGNORE };
0054
0055     MergeableRunProductMetadata(MergeableRunProductProcesses const&);
0056
0057     ~MergeableRunProductMetadata() override;
0058
0059     // Called each time a new input file is opened
0060     void preReadFile();
0061
0062     // Called each time a run entry is read from an input file. This
0063     // should be called before the run products themselves are read
0064     // because it sets the decision whether to merge, replace, or ignore
0065     // run products as they read.
0066     void readRun(long long inputRunEntry,
0067                  StoredMergeableRunProductMetadata const& inputStoredMergeableRunProductMetadata,
0068                  IndexIntoFileItrHolder const& inputIndexIntoFileItr);
0069
0070     // Called to record which lumis were processed by the current run
0071     void writeLumi(LuminosityBlockNumber_t lumi);
0072
0073     void preWriteRun();
0074     void postWriteRun();
0075
0076     void addEntryToStoredMetadata(StoredMergeableRunProductMetadata&) const;
0077
0078     MergeDecision getMergeDecision(std::string const& processThatCreatedProduct) const;
0079
0080     // If Runs were split on lumi boundaries, but then the files were merged
0081     // in a way that made it impossible to properly merge run products, this
0082     // function will return true. The Framework usually does not know
0083     // enough to detect other cases where there is a merging problem.
0084     bool knownImproperlyMerged(std::string const& processThatCreatedProduct) const override;
0085
0086     std::string const& getProcessName(unsigned int index) const {
0087       return mergeableRunProductProcesses_->getProcessName(index);
0088     }
0089
0090     class MetadataForProcess {
0091     public:
0092       MetadataForProcess() = default;
0093
0094       std::vector<LuminosityBlockNumber_t>& lumis() { return lumis_; }
0095       std::vector<LuminosityBlockNumber_t> const& lumis() const { return lumis_; }
0096
0097       MergeDecision mergeDecision() const { return mergeDecision_; }
0098       void setMergeDecision(MergeDecision v) { mergeDecision_ = v; }
0099
0100       bool valid() const { return valid_; }
0101       void setValid(bool v) { valid_ = v; }
0102
0103       bool useIndexIntoFile() const { return useIndexIntoFile_; }
0104       void setUseIndexIntoFile(bool v) { useIndexIntoFile_ = v; }
0105
0106       bool allLumisProcessed() const { return allLumisProcessed_; }
0107       void setAllLumisProcessed(bool v) { allLumisProcessed_ = v; }
0108
0109       void reset();
0110
0111     private:
0112       std::vector<LuminosityBlockNumber_t> lumis_;
0113       MergeDecision mergeDecision_ = MERGE;
0114       bool valid_ = true;
0115       bool useIndexIntoFile_ = false;
0116       bool allLumisProcessed_ = false;
0117     };
0118
0119     MetadataForProcess const* metadataForOneProcess(std::string const& processName) const;
0120
0121     // The next few functions are only intended to be used in tests.
0122     std::vector<MetadataForProcess> const& metadataForProcesses() const { return metadataForProcesses_; }
0123
0124     std::vector<LuminosityBlockNumber_t> const& lumisFromIndexIntoFile() const { return lumisFromIndexIntoFile_; }
0125
0126     bool gotLumisFromIndexIntoFile() const { return gotLumisFromIndexIntoFile_; }
0127
0128     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> const& lumisProcessed() const { return lumisProcessed_; }
0129
0130   private:
0131     void mergeLumisFromIndexIntoFile();
0132
0133     bool addProcess(StoredMergeableRunProductMetadata& storedMetadata,
0134                     MetadataForProcess const& metadataForProcess,
0135                     unsigned int storedProcessIndex,
0136                     unsigned long long beginProcess,
0137                     unsigned long long endProcess) const;
0138
0139     MergeableRunProductProcesses const* mergeableRunProductProcesses_;
0140
0141     // This vector has an entry for each process that has a
0142     // mergeable run product in the input. It has an exact one to
0143     // correspondence with mergeableRunProductProcesses_ and
0144     // is in the same order.
0145     std::vector<MetadataForProcess> metadataForProcesses_;
0146
0147     std::vector<LuminosityBlockNumber_t> lumisFromIndexIntoFile_;
0148     bool gotLumisFromIndexIntoFile_ = false;
0149
0150     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> lumisProcessed_;
0151   };
0152 }  // namespace edm
0153 #endif