Framework/interface/MergeableRunProductMetadata.h

0001 #ifndef FWCore_Framework_MergeableRunProductMetadata_h
0002 #define FWCore_Framework_MergeableRunProductMetadata_h
0003
0004 /** \class edm::MergeableRunProductMetadata
0005
0006 This class holds information used to decide how to merge together
0007 mergeable run products when multiple run entries with the same
0008 run number and ProcessHistoryID are read from input files
0009 contiguously.
0010
0011 Most of the information here is associated with the current
0012 run being processed. Most of it is cleared when a new run
0013 is started. If multiple runs are being processed concurrently,
0014 then there will be an object instantiated for each concurrent
0015 run. (At the present time concurrent runs are not possible, but
0016 there plans to implement that in the future). The primary
0017 RunPrincipal for the current run owns the object.
0018
0019 This class gets information from the input file from the
0020 StoredMergeableRunProductMetadata object and IndexIntoFile object.
0021 It uses that information to make the decision how to merge
0022 run products read from data, and puts information into the
0023 StoredMergeableRunProductMetadata written into the output file
0024 to use in later processing steps.
0025
0026 If there are SubProcesses, they use the same object as the top
0027 level process because they share the same input.
0028
0029 There will be a TWIKI page on the Framework page of the Software
0030 Guide which explains the details about how this works. There
0031 are significant limitations related to what the Framework does
0032 and does not do managing mergeable run products.
0033
0034 \author W. David Dagenhart, created 9 July, 2018
0035
0036 */
0037
0038 #include "DataFormats/Provenance/interface/MergeableRunProductMetadataBase.h"
0039 #include "FWCore/Framework/interface/MergeableRunProductProcesses.h"
0040 #include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
0041
0042 #include "oneapi/tbb/concurrent_vector.h"
0043
0044 #include <string>
0045 #include <vector>
0046
0047 namespace edm {
0048
0049   class IndexIntoFileItrHolder;
0050   class StoredMergeableRunProductMetadata;
0051
0052   class MergeableRunProductMetadata : public MergeableRunProductMetadataBase {
0053   public:
0054     enum MergeDecision { MERGE, REPLACE, IGNORE };
0055
0056     MergeableRunProductMetadata(MergeableRunProductProcesses const&);
0057
0058     ~MergeableRunProductMetadata() override;
0059
0060     // Called each time a new input file is opened
0061     void preReadFile();
0062
0063     // Called each time a run entry is read from an input file. This
0064     // should be called before the run products themselves are read
0065     // because it sets the decision whether to merge, replace, or ignore
0066     // run products as they read.
0067     void readRun(long long inputRunEntry,
0068                  StoredMergeableRunProductMetadata const& inputStoredMergeableRunProductMetadata,
0069                  IndexIntoFileItrHolder const& inputIndexIntoFileItr);
0070
0071     // Called to record which lumis were processed by the current run
0072     void writeLumi(LuminosityBlockNumber_t lumi);
0073
0074     void preWriteRun();
0075     void postWriteRun();
0076
0077     void addEntryToStoredMetadata(StoredMergeableRunProductMetadata&) const;
0078
0079     MergeDecision getMergeDecision(std::string const& processThatCreatedProduct) const;
0080
0081     // If Runs were split on lumi boundaries, but then the files were merged
0082     // in a way that made it impossible to properly merge run products, this
0083     // function will return true. The Framework usually does not know
0084     // enough to detect other cases where there is a merging problem.
0085     bool knownImproperlyMerged(std::string const& processThatCreatedProduct) const override;
0086
0087     std::string const& getProcessName(unsigned int index) const {
0088       return mergeableRunProductProcesses_->getProcessName(index);
0089     }
0090
0091     class MetadataForProcess {
0092     public:
0093       MetadataForProcess() = default;
0094
0095       std::vector<LuminosityBlockNumber_t>& lumis() { return lumis_; }
0096       std::vector<LuminosityBlockNumber_t> const& lumis() const { return lumis_; }
0097
0098       MergeDecision mergeDecision() const { return mergeDecision_; }
0099       void setMergeDecision(MergeDecision v) { mergeDecision_ = v; }
0100
0101       bool valid() const { return valid_; }
0102       void setValid(bool v) { valid_ = v; }
0103
0104       bool useIndexIntoFile() const { return useIndexIntoFile_; }
0105       void setUseIndexIntoFile(bool v) { useIndexIntoFile_ = v; }
0106
0107       bool allLumisProcessed() const { return allLumisProcessed_; }
0108       void setAllLumisProcessed(bool v) { allLumisProcessed_ = v; }
0109
0110       void reset();
0111
0112     private:
0113       std::vector<LuminosityBlockNumber_t> lumis_;
0114       MergeDecision mergeDecision_ = MERGE;
0115       bool valid_ = true;
0116       bool useIndexIntoFile_ = false;
0117       bool allLumisProcessed_ = false;
0118     };
0119
0120     MetadataForProcess const* metadataForOneProcess(std::string const& processName) const;
0121
0122     // The next few functions are only intended to be used in tests.
0123     std::vector<MetadataForProcess> const& metadataForProcesses() const { return metadataForProcesses_; }
0124
0125     std::vector<LuminosityBlockNumber_t> const& lumisFromIndexIntoFile() const { return lumisFromIndexIntoFile_; }
0126
0127     bool gotLumisFromIndexIntoFile() const { return gotLumisFromIndexIntoFile_; }
0128
0129     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> const& lumisProcessed() const { return lumisProcessed_; }
0130
0131   private:
0132     void mergeLumisFromIndexIntoFile();
0133
0134     bool addProcess(StoredMergeableRunProductMetadata& storedMetadata,
0135                     MetadataForProcess const& metadataForProcess,
0136                     unsigned int storedProcessIndex,
0137                     unsigned long long beginProcess,
0138                     unsigned long long endProcess) const;
0139
0140     MergeableRunProductProcesses const* mergeableRunProductProcesses_;
0141
0142     // This vector has an entry for each process that has a
0143     // mergeable run product in the input. It has an exact one to
0144     // correspondence with mergeableRunProductProcesses_ and
0145     // is in the same order.
0146     std::vector<MetadataForProcess> metadataForProcesses_;
0147
0148     std::vector<LuminosityBlockNumber_t> lumisFromIndexIntoFile_;
0149     bool gotLumisFromIndexIntoFile_ = false;
0150
0151     oneapi::tbb::concurrent_vector<LuminosityBlockNumber_t> lumisProcessed_;
0152   };
0153 }  // namespace edm
0154 #endif