Input/src/RootFile.h

0001 #ifndef IOPool_Input_RootFile_h
0002 #define IOPool_Input_RootFile_h
0003
0004 /*----------------------------------------------------------------------
0005
0006 RootFile.h // used by ROOT input sources
0007
0008 ----------------------------------------------------------------------*/
0009
0010 #include "RootTree.h"
0011 #include "DataFormats/Provenance/interface/ProductDependencies.h"
0012 #include "DataFormats/Provenance/interface/BranchIDList.h"
0013 #include "DataFormats/Provenance/interface/BranchListIndex.h"
0014 #include "DataFormats/Provenance/interface/EntryDescriptionID.h"  // backward compatibility
0015 #include "DataFormats/Provenance/interface/EventAuxiliary.h"
0016 #include "DataFormats/Provenance/interface/EventEntryDescription.h"  // backward compatibility
0017 #include "DataFormats/Provenance/interface/EventProcessHistoryID.h"  // backward compatibility
0018 #include "DataFormats/Provenance/interface/EventSelectionID.h"
0019 #include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h"
0020 #include "DataFormats/Provenance/interface/FileFormatVersion.h"
0021 #include "DataFormats/Provenance/interface/FileID.h"
0022 #include "DataFormats/Provenance/interface/History.h"
0023 #include "DataFormats/Provenance/interface/IndexIntoFile.h"
0024 #include "DataFormats/Provenance/interface/ParentageID.h"
0025 #include "DataFormats/Provenance/interface/ProvenanceFwd.h"
0026 #include "FWCore/Common/interface/FWCoreCommonFwd.h"
0027 #include "FWCore/Framework/interface/Frameworkfwd.h"
0028 #include "FWCore/Framework/interface/InputSource.h"
0029 #include "FWCore/Utilities/interface/InputType.h"
0030 #include "FWCore/Utilities/interface/get_underlying_safe.h"
0031 #include "FWCore/Utilities/interface/propagate_const.h"
0032
0033 #include "TBranch.h"
0034
0035 #include <array>
0036 #include <map>
0037 #include <memory>
0038 #include <set>
0039 #include <string>
0040 #include <tuple>
0041 #include <vector>
0042
0043 namespace edm {
0044
0045   //------------------------------------------------------------
0046   // Class RootFile: supports file reading.
0047
0048   class BranchID;
0049   class BranchIDListHelper;
0050   class ProductProvenanceRetriever;
0051   struct DaqProvenanceHelper;
0052   class DuplicateChecker;
0053   class EventSkipperByID;
0054   class ProcessHistoryRegistry;
0055   class ProductSelectorRules;
0056   class InputFile;
0057   class ProvenanceReaderBase;
0058   class ProvenanceAdaptor;
0059   class StoredMergeableRunProductMetadata;
0060   class RunHelperBase;
0061   class ThinnedAssociationsHelper;
0062
0063   using EntryDescriptionMap = std::map<EntryDescriptionID, EventEntryDescription>;
0064
0065   class MakeProvenanceReader {
0066   public:
0067     virtual std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0068                                                              DaqProvenanceHelper const* daqProvenanceHelper) const = 0;
0069     virtual ~MakeProvenanceReader() = default;
0070   };
0071
0072   class RootFile {
0073   public:
0074     struct FileOptions {
0075       std::string const& fileName;
0076       std::string const& logicalFileName;
0077       std::shared_ptr<InputFile> filePtr;
0078       bool bypassVersionCheck;
0079       bool enforceGUIDInFileName;
0080     };
0081
0082     struct ProcessingOptions {
0083       std::shared_ptr<EventSkipperByID> eventSkipperByID{};
0084       bool skipAnyEvents = false;
0085       int remainingEvents = -1;
0086       int remainingLumis = -1;
0087       InputSource::ProcessingMode processingMode = InputSource::RunsLumisAndEvents;
0088       bool noRunLumiSort = false;
0089       bool noEventSort = false;
0090       bool usingGoToEvent = false;
0091     };
0092
0093     using TTreeOptions = RootTree::Options;
0094     struct ProductChoices {
0095       ProductSelectorRules const& productSelectorRules;
0096       std::vector<BranchID> const* associationsFromSecondary = nullptr;
0097       bool dropDescendantsOfDroppedProducts = false;
0098       bool labelRawDataLikeMC = false;
0099     };
0100
0101     struct CrossFileInfo {
0102       RunHelperBase* runHelper = nullptr;
0103       std::shared_ptr<BranchIDListHelper> branchIDListHelper{};
0104       ProcessBlockHelper* processBlockHelper = nullptr;
0105       std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper{};
0106       std::shared_ptr<DuplicateChecker> duplicateChecker{};
0107       std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles;  //duplicate checking
0108       std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile;
0109     };
0110
0111     RootFile(FileOptions&& fileOptions,
0112              InputType inputType,
0113              ProcessingOptions&& processingOptions,
0114              TTreeOptions&& ttreeOptions,
0115              ProductChoices&& productChoices,
0116              CrossFileInfo&& crossFileInfo,
0117              unsigned int nStreams,
0118              ProcessHistoryRegistry& processHistoryRegistry,
0119              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs);
0120     ~RootFile();
0121
0122     RootFile(RootFile const&) = delete;             // Disallow copying and moving
0123     RootFile& operator=(RootFile const&) = delete;  // Disallow copying and moving
0124
0125     void reportOpened(std::string const& inputType);
0126     void close();
0127     std::tuple<bool, bool> readCurrentEvent(EventPrincipal& cache,
0128                                             bool assertOnFailure = true,
0129                                             bool readAllProducts = false);
0130     bool readEvent(EventPrincipal& cache, bool readAllProducts = false);
0131
0132     std::shared_ptr<LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_();
0133     std::shared_ptr<RunAuxiliary> readRunAuxiliary_();
0134     std::shared_ptr<RunAuxiliary> readFakeRunAuxiliary_();
0135
0136     void fillProcessBlockHelper_();
0137     bool initializeFirstProcessBlockEntry();
0138     bool endOfProcessBlocksReached() const;
0139     bool nextProcessBlock_(ProcessBlockPrincipal&);
0140     void readProcessBlock_(ProcessBlockPrincipal&);
0141
0142     bool readRun_(RunPrincipal& runPrincipal);
0143     void readFakeRun_(RunPrincipal& runPrincipal);
0144     bool readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal);
0145     std::string const& file() const { return file_; }
0146     std::shared_ptr<ProductRegistry const> productRegistry() const { return productRegistry_; }
0147     // IndexIntoFile::EntryNumber_t const& entryNumber() const {return indexIntoFileIter().entry();}
0148     // LuminosityBlockNumber_t const& luminosityBlockNumber() const {return indexIntoFileIter().lumi();}
0149     // RunNumber_t const& runNumber() const {return indexIntoFileIter().run();}
0150     RootTree const& eventTree() const { return eventTree_; }
0151     RootTree const& lumiTree() const { return lumiTree_; }
0152     RootTree const& runTree() const { return runTree_; }
0153     FileFormatVersion fileFormatVersion() const { return fileFormatVersion_; }
0154     int whyNotFastClonable() const { return whyNotFastClonable_; }
0155     std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
0156     bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; }
0157     bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; }
0158     std::shared_ptr<FileBlock> createFileBlock();
0159     void updateFileBlock(FileBlock&);
0160
0161     bool setEntryAtItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
0162       return (event != 0) ? setEntryAtEvent(run, lumi, event) : (lumi ? setEntryAtLumi(run, lumi) : setEntryAtRun(run));
0163     }
0164     bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;
0165     bool setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event);
0166     bool setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi);
0167     bool setEntryAtRun(RunNumber_t run);
0168     bool setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi);
0169     void setAtEventEntry(IndexIntoFile::EntryNumber_t entry);
0170
0171     void rewind() {
0172       indexIntoFileIter_ = indexIntoFileBegin_;
0173       eventTree_.rewind();
0174       lumiTree_.rewind();
0175       runTree_.rewind();
0176       currentProcessBlockTree_ = 0;
0177       for (auto& processBlockTree : processBlockTrees_) {
0178         processBlockTree->rewindToInvalid();
0179       }
0180     }
0181     void setToLastEntry() { indexIntoFileIter_ = indexIntoFileEnd_; }
0182
0183     bool skipEntries(unsigned int& offset) { return eventTree_.skipEntries(offset); }
0184     bool skipEvents(int& offset);
0185     bool goToEvent(EventID const& eventID);
0186     bool nextEventEntry() { return eventTree_.nextWithCache(); }
0187     IndexIntoFile::EntryType getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event);
0188     std::shared_ptr<BranchIDListHelper const> branchIDListHelper() const {
0189       return get_underlying_safe(branchIDListHelper_);
0190     }
0191     std::shared_ptr<BranchIDListHelper>& branchIDListHelper() { return get_underlying_safe(branchIDListHelper_); }
0192     std::shared_ptr<IndexIntoFile const> indexIntoFileSharedPtr() const {
0193       return get_underlying_safe(indexIntoFileSharedPtr_);
0194     }
0195     std::shared_ptr<IndexIntoFile>& indexIntoFileSharedPtr() { return get_underlying_safe(indexIntoFileSharedPtr_); }
0196     bool wasLastEventJustRead() const;
0197     bool wasFirstEventJustRead() const;
0198     IndexIntoFile::IndexIntoFileItr indexIntoFileIter() const;
0199     void setPosition(IndexIntoFile::IndexIntoFileItr const& position);
0200     void initAssociationsFromSecondary(std::vector<BranchID> const&);
0201
0202     void setSignals(
0203         signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* preEventReadSource,
0204         signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* postEventReadSource);
0205
0206   private:
0207     void makeProcessBlockRootTrees(std::shared_ptr<InputFile> filePtr,
0208                                    TTreeOptions const& ttreeOptions,
0209                                    InputType inputType,
0210                                    StoredProcessBlockHelper const& storedProcessBlockHelper);
0211     bool skipThisEntry();
0212     void setIfFastClonable(int remainingEvents, int remainingLumis);
0213     void validateFile(InputType inputType,
0214                       bool usingGoToEvent,
0215                       std::vector<ProcessHistoryID>& orderedProcessHistoryIDs);
0216     void fillIndexIntoFile();
0217     EventAuxiliary fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry);
0218     EventAuxiliary const& fillThisEventAuxiliary();
0219     void fillEventToProcessBlockIndexes();
0220     bool fillEventHistory(EventAuxiliary& evtAux,
0221                           EventSelectionIDVector& eventSelectionIDs,
0222                           BranchListIndexes& branchListIndexes,
0223                           bool assertOnFailure = true);
0224     std::shared_ptr<LuminosityBlockAuxiliary> fillLumiAuxiliary();
0225     std::shared_ptr<RunAuxiliary> fillRunAuxiliary();
0226     std::string const& newBranchToOldBranch(std::string const& newBranch) const;
0227     void setPresenceInProductRegistry(ProductRegistry&, StoredProcessBlockHelper const&);
0228     void markBranchToBeDropped(bool dropDescendants,
0229                                ProductDescription const& branch,
0230                                std::set<BranchID>& branchesToDrop,
0231                                std::map<BranchID, BranchID> const& droppedToKeptAlias) const;
0232     void dropOnInputAndReorder(ProductRegistry&,
0233                                ProductSelectorRules const&,
0234                                bool dropDescendants,
0235                                InputType,
0236                                StoredProcessBlockHelper&,
0237                                ProcessBlockHelper const*);
0238     void dropProcessesAndReorder(StoredProcessBlockHelper&,
0239                                  std::set<std::string> const& processesWithKeptProcessBlockProducts,
0240                                  ProcessBlockHelper const*);
0241
0242     void readParentageTree(InputType inputType);
0243     void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap,
0244                                   InputType inputType);  // backward compatibility
0245     void readEventHistoryTree();
0246     bool isDuplicateEvent();
0247
0248     void initializeDuplicateChecker(std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
0249                                     std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile);
0250
0251     std::unique_ptr<MakeProvenanceReader> makeProvenanceReaderMaker(InputType inputType);
0252     std::shared_ptr<ProductProvenanceRetriever> makeProductProvenanceRetriever(unsigned int iStreamIndex);
0253
0254     std::shared_ptr<RunAuxiliary const> savedRunAuxiliary() const { return get_underlying_safe(savedRunAuxiliary_); }
0255     std::shared_ptr<RunAuxiliary>& savedRunAuxiliary() { return get_underlying_safe(savedRunAuxiliary_); }
0256
0257     std::shared_ptr<ProductDependencies const> productDependencies() const {
0258       return get_underlying_safe(productDependencies_);
0259     }
0260     std::shared_ptr<ProductDependencies>& productDependencies() { return get_underlying_safe(productDependencies_); }
0261
0262     std::shared_ptr<ProductProvenanceRetriever const> eventProductProvenanceRetriever(size_t index) const {
0263       return get_underlying_safe(eventProductProvenanceRetrievers_[index]);
0264     }
0265     std::shared_ptr<ProductProvenanceRetriever>& eventProductProvenanceRetriever(size_t index) {
0266       return get_underlying_safe(eventProductProvenanceRetrievers_[index]);
0267     }
0268
0269     std::string const file_;
0270     std::string const logicalFile_;
0271     edm::propagate_const<ProcessHistoryRegistry*> processHistoryRegistry_;  // We don't own this
0272     edm::propagate_const<std::shared_ptr<InputFile>> filePtr_;
0273     edm::propagate_const<std::shared_ptr<EventSkipperByID>> eventSkipperByID_;
0274     FileFormatVersion fileFormatVersion_;
0275     FileID fid_;
0276     edm::propagate_const<std::shared_ptr<IndexIntoFile>> indexIntoFileSharedPtr_;
0277     IndexIntoFile& indexIntoFile_;
0278     IndexIntoFile::IndexIntoFileItr indexIntoFileBegin_;
0279     IndexIntoFile::IndexIntoFileItr indexIntoFileEnd_;
0280     IndexIntoFile::IndexIntoFileItr indexIntoFileIter_;
0281     edm::propagate_const<std::unique_ptr<StoredMergeableRunProductMetadata>> storedMergeableRunProductMetadata_;
0282     std::vector<EventProcessHistoryID> eventProcessHistoryIDs_;                   // backward compatibility
0283     std::vector<EventProcessHistoryID>::const_iterator eventProcessHistoryIter_;  // backward compatibility
0284     edm::propagate_const<std::shared_ptr<RunAuxiliary>> savedRunAuxiliary_;
0285     bool skipAnyEvents_;
0286     bool noRunLumiSort_;
0287     bool noEventSort_;
0288     bool enforceGUIDInFileName_;
0289     int whyNotFastClonable_;
0290     std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
0291     bool branchListIndexesUnchanged_;
0292     EventAuxiliary eventAuxCache_;  //Should only be used by fillThisEventAuxiliary()
0293     RootTree eventTree_;
0294     RootTree lumiTree_;
0295     RootTree runTree_;
0296     std::vector<edm::propagate_const<std::unique_ptr<RootTree>>> processBlockTrees_;
0297     unsigned int currentProcessBlockTree_ = 0;
0298     std::vector<edm::propagate_const<RootTree*>> treePointers_;
0299     //Should only be used by fillThisEventAuxiliary()
0300     IndexIntoFile::EntryNumber_t lastEventEntryNumberRead_;
0301     std::shared_ptr<ProductRegistry const> productRegistry_;
0302     std::shared_ptr<BranchIDLists const> branchIDLists_;
0303     edm::propagate_const<std::shared_ptr<BranchIDListHelper>> branchIDListHelper_;
0304     edm::propagate_const<ProcessBlockHelper*> processBlockHelper_;
0305     edm::propagate_const<std::unique_ptr<StoredProcessBlockHelper>> storedProcessBlockHelper_;
0306     edm::propagate_const<std::unique_ptr<ThinnedAssociationsHelper>> fileThinnedAssociationsHelper_;
0307     edm::propagate_const<std::shared_ptr<ThinnedAssociationsHelper>> thinnedAssociationsHelper_;
0308     InputSource::ProcessingMode processingMode_;
0309     edm::propagate_const<RunHelperBase*> runHelper_;
0310     std::map<std::string, std::string> newBranchToOldBranch_;
0311     edm::propagate_const<TTree*> eventHistoryTree_;  // backward compatibility
0312     EventToProcessBlockIndexes eventToProcessBlockIndexes_;
0313     edm::propagate_const<TBranch*> eventToProcessBlockIndexesBranch_;
0314     edm::propagate_const<std::unique_ptr<History>> history_;  // backward compatibility
0315     edm::propagate_const<std::shared_ptr<ProductDependencies>> productDependencies_;
0316     edm::propagate_const<std::shared_ptr<DuplicateChecker>> duplicateChecker_;
0317     edm::propagate_const<std::unique_ptr<ProvenanceAdaptor>> provenanceAdaptor_;  // backward comatibility
0318     edm::propagate_const<std::unique_ptr<MakeProvenanceReader>> provenanceReaderMaker_;
0319     std::vector<edm::propagate_const<std::shared_ptr<ProductProvenanceRetriever>>> eventProductProvenanceRetrievers_;
0320     std::vector<ParentageID> parentageIDLookup_;
0321     edm::propagate_const<std::unique_ptr<DaqProvenanceHelper>> daqProvenanceHelper_;
0322     edm::propagate_const<TClass*> edProductClass_;
0323     InputType inputType_;
0324   };  // class RootFile
0325
0326 }  // namespace edm
0327 #endif