Input/src/RootFile.h

0001 #ifndef IOPool_Input_RootFile_h
0002 #define IOPool_Input_RootFile_h
0003
0004 /*----------------------------------------------------------------------
0005
0006 RootFile.h // used by ROOT input sources
0007
0008 ----------------------------------------------------------------------*/
0009
0010 #include "RootTree.h"
0011 #include "DataFormats/Provenance/interface/BranchChildren.h"
0012 #include "DataFormats/Provenance/interface/BranchIDList.h"
0013 #include "DataFormats/Provenance/interface/BranchListIndex.h"
0014 #include "DataFormats/Provenance/interface/EntryDescriptionID.h"  // backward compatibility
0015 #include "DataFormats/Provenance/interface/EventAuxiliary.h"
0016 #include "DataFormats/Provenance/interface/EventEntryDescription.h"  // backward compatibility
0017 #include "DataFormats/Provenance/interface/EventProcessHistoryID.h"  // backward compatibility
0018 #include "DataFormats/Provenance/interface/EventSelectionID.h"
0019 #include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h"
0020 #include "DataFormats/Provenance/interface/FileFormatVersion.h"
0021 #include "DataFormats/Provenance/interface/FileID.h"
0022 #include "DataFormats/Provenance/interface/History.h"
0023 #include "DataFormats/Provenance/interface/IndexIntoFile.h"
0024 #include "DataFormats/Provenance/interface/ProvenanceFwd.h"
0025 #include "FWCore/Common/interface/FWCoreCommonFwd.h"
0026 #include "FWCore/Framework/interface/Frameworkfwd.h"
0027 #include "FWCore/Framework/interface/InputSource.h"
0028 #include "FWCore/Utilities/interface/InputType.h"
0029 #include "FWCore/Utilities/interface/get_underlying_safe.h"
0030 #include "FWCore/Utilities/interface/propagate_const.h"
0031
0032 #include "TBranch.h"
0033
0034 #include <array>
0035 #include <map>
0036 #include <memory>
0037 #include <set>
0038 #include <string>
0039 #include <tuple>
0040 #include <vector>
0041
0042 namespace edm {
0043
0044   //------------------------------------------------------------
0045   // Class RootFile: supports file reading.
0046
0047   class BranchID;
0048   class BranchIDListHelper;
0049   class ProductProvenanceRetriever;
0050   struct DaqProvenanceHelper;
0051   class DuplicateChecker;
0052   class EventSkipperByID;
0053   class ProcessHistoryRegistry;
0054   class ProductSelectorRules;
0055   class InputFile;
0056   class ProvenanceReaderBase;
0057   class ProvenanceAdaptor;
0058   class StoredMergeableRunProductMetadata;
0059   class RunHelperBase;
0060   class ThinnedAssociationsHelper;
0061
0062   using EntryDescriptionMap = std::map<EntryDescriptionID, EventEntryDescription>;
0063
0064   class MakeProvenanceReader {
0065   public:
0066     virtual std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0067                                                              DaqProvenanceHelper const* daqProvenanceHelper) const = 0;
0068     virtual ~MakeProvenanceReader() = default;
0069   };
0070
0071   class RootFile {
0072   public:
0073     // Constructor used by RootPrimaryFileSequence
0074     RootFile(std::string const& fileName,
0075              ProcessConfiguration const& processConfiguration,
0076              std::string const& logicalFileName,
0077              std::shared_ptr<InputFile> filePtr,
0078              std::shared_ptr<EventSkipperByID> eventSkipperByID,
0079              bool skipAnyEvents,
0080              int remainingEvents,
0081              int remainingLumis,
0082              unsigned int nStreams,
0083              unsigned int treeCacheSize,
0084              int treeMaxVirtualSize,
0085              InputSource::ProcessingMode processingMode,
0086              RunHelperBase* runHelper,
0087              bool noRunLumiSort,
0088              bool noEventSort,
0089              ProductSelectorRules const& productSelectorRules,
0090              InputType inputType,
0091              std::shared_ptr<BranchIDListHelper> branchIDListHelper,
0092              ProcessBlockHelper*,
0093              std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper,
0094              std::vector<BranchID> const* associationsFromSecondary,
0095              std::shared_ptr<DuplicateChecker> duplicateChecker,
0096              bool dropDescendantsOfDroppedProducts,
0097              ProcessHistoryRegistry& processHistoryRegistry,
0098              std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
0099              std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
0100              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
0101              bool bypassVersionCheck,
0102              bool labelRawDataLikeMC,
0103              bool usingGoToEvent,
0104              bool enablePrefetching,
0105              bool enforceGUIDInFileName);
0106
0107     // Constructor used by RootSecondaryFileSequence
0108     RootFile(std::string const& fileName,
0109              ProcessConfiguration const& processConfiguration,
0110              std::string const& logicalFileName,
0111              std::shared_ptr<InputFile> filePtr,
0112              unsigned int nStreams,
0113              int treeMaxVirtualSize,
0114              InputSource::ProcessingMode processingMode,
0115              RunHelperBase* runHelper,
0116              ProductSelectorRules const& productSelectorRules,
0117              InputType inputType,
0118              std::shared_ptr<BranchIDListHelper> branchIDListHelper,
0119              std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper,
0120              std::vector<BranchID> const* associationsFromSecondary,
0121              bool dropDescendantsOfDroppedProducts,
0122              ProcessHistoryRegistry& processHistoryRegistry,
0123              std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
0124              std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
0125              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
0126              bool bypassVersionCheck,
0127              bool labelRawDataLikeMC,
0128              bool enablePrefetching,
0129              bool enforceGUIDInFileName)
0130         : RootFile(fileName,
0131                    processConfiguration,
0132                    logicalFileName,
0133                    filePtr,
0134                    nullptr,
0135                    false,
0136                    -1,
0137                    -1,
0138                    nStreams,
0139                    0U,
0140                    treeMaxVirtualSize,
0141                    processingMode,
0142                    runHelper,
0143                    false,
0144                    false,
0145                    productSelectorRules,
0146                    inputType,
0147                    branchIDListHelper,
0148                    nullptr,
0149                    thinnedAssociationsHelper,
0150                    associationsFromSecondary,
0151                    nullptr,
0152                    dropDescendantsOfDroppedProducts,
0153                    processHistoryRegistry,
0154                    indexesIntoFiles,
0155                    currentIndexIntoFile,
0156                    orderedProcessHistoryIDs,
0157                    bypassVersionCheck,
0158                    labelRawDataLikeMC,
0159                    false,
0160                    enablePrefetching,
0161                    enforceGUIDInFileName) {}
0162
0163     // Constructor used by RootEmbeddedFileSequence
0164     RootFile(std::string const& fileName,
0165              ProcessConfiguration const& processConfiguration,
0166              std::string const& logicalFileName,
0167              std::shared_ptr<InputFile> filePtr,
0168              unsigned int nStreams,
0169              unsigned int treeCacheSize,
0170              int treeMaxVirtualSize,
0171              RunHelperBase* runHelper,
0172              ProductSelectorRules const& productSelectorRules,
0173              InputType inputType,
0174              ProcessHistoryRegistry& processHistoryRegistry,
0175              std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
0176              std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
0177              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
0178              bool bypassVersionCheck,
0179              bool enablePrefetching,
0180              bool enforceGUIDInFileName)
0181         : RootFile(fileName,
0182                    processConfiguration,
0183                    logicalFileName,
0184                    filePtr,
0185                    nullptr,
0186                    false,
0187                    -1,
0188                    -1,
0189                    nStreams,
0190                    treeCacheSize,
0191                    treeMaxVirtualSize,
0192                    InputSource::RunsLumisAndEvents,
0193                    runHelper,
0194                    false,
0195                    false,
0196                    productSelectorRules,
0197                    inputType,
0198                    nullptr,
0199                    nullptr,
0200                    nullptr,
0201                    nullptr,
0202                    nullptr,
0203                    false,
0204                    processHistoryRegistry,
0205                    indexesIntoFiles,
0206                    currentIndexIntoFile,
0207                    orderedProcessHistoryIDs,
0208                    bypassVersionCheck,
0209                    false,
0210                    false,
0211                    enablePrefetching,
0212                    enforceGUIDInFileName) {}
0213
0214     ~RootFile();
0215
0216     RootFile(RootFile const&) = delete;             // Disallow copying and moving
0217     RootFile& operator=(RootFile const&) = delete;  // Disallow copying and moving
0218
0219     void reportOpened(std::string const& inputType);
0220     void close();
0221     std::tuple<bool, bool> readCurrentEvent(EventPrincipal& cache, bool assertOnFailure = true);
0222     bool readEvent(EventPrincipal& cache);
0223
0224     std::shared_ptr<LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_();
0225     std::shared_ptr<RunAuxiliary> readRunAuxiliary_();
0226     std::shared_ptr<RunAuxiliary> readFakeRunAuxiliary_();
0227
0228     void fillProcessBlockHelper_();
0229     bool initializeFirstProcessBlockEntry();
0230     bool endOfProcessBlocksReached() const;
0231     bool nextProcessBlock_(ProcessBlockPrincipal&);
0232     void readProcessBlock_(ProcessBlockPrincipal&);
0233
0234     bool readRun_(RunPrincipal& runPrincipal);
0235     void readFakeRun_(RunPrincipal& runPrincipal);
0236     bool readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal);
0237     std::string const& file() const { return file_; }
0238     std::shared_ptr<ProductRegistry const> productRegistry() const { return productRegistry_; }
0239     // IndexIntoFile::EntryNumber_t const& entryNumber() const {return indexIntoFileIter().entry();}
0240     // LuminosityBlockNumber_t const& luminosityBlockNumber() const {return indexIntoFileIter().lumi();}
0241     // RunNumber_t const& runNumber() const {return indexIntoFileIter().run();}
0242     RootTree const& eventTree() const { return eventTree_; }
0243     RootTree const& lumiTree() const { return lumiTree_; }
0244     RootTree const& runTree() const { return runTree_; }
0245     FileFormatVersion fileFormatVersion() const { return fileFormatVersion_; }
0246     int whyNotFastClonable() const { return whyNotFastClonable_; }
0247     std::array<bool, NumBranchTypes> const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; }
0248     bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; }
0249     bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; }
0250     std::shared_ptr<FileBlock> createFileBlock();
0251     void updateFileBlock(FileBlock&);
0252
0253     bool setEntryAtItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
0254       return (event != 0) ? setEntryAtEvent(run, lumi, event) : (lumi ? setEntryAtLumi(run, lumi) : setEntryAtRun(run));
0255     }
0256     bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;
0257     bool setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event);
0258     bool setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi);
0259     bool setEntryAtRun(RunNumber_t run);
0260     bool setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi);
0261     void setAtEventEntry(IndexIntoFile::EntryNumber_t entry);
0262
0263     void rewind() {
0264       indexIntoFileIter_ = indexIntoFileBegin_;
0265       eventTree_.rewind();
0266       lumiTree_.rewind();
0267       runTree_.rewind();
0268       currentProcessBlockTree_ = 0;
0269       for (auto& processBlockTree : processBlockTrees_) {
0270         processBlockTree->rewindToInvalid();
0271       }
0272     }
0273     void setToLastEntry() { indexIntoFileIter_ = indexIntoFileEnd_; }
0274
0275     bool skipEntries(unsigned int& offset) { return eventTree_.skipEntries(offset); }
0276     bool skipEvents(int& offset);
0277     bool goToEvent(EventID const& eventID);
0278     bool nextEventEntry() { return eventTree_.nextWithCache(); }
0279     IndexIntoFile::EntryType getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event);
0280     std::shared_ptr<BranchIDListHelper const> branchIDListHelper() const {
0281       return get_underlying_safe(branchIDListHelper_);
0282     }
0283     std::shared_ptr<BranchIDListHelper>& branchIDListHelper() { return get_underlying_safe(branchIDListHelper_); }
0284     std::shared_ptr<IndexIntoFile const> indexIntoFileSharedPtr() const {
0285       return get_underlying_safe(indexIntoFileSharedPtr_);
0286     }
0287     std::shared_ptr<IndexIntoFile>& indexIntoFileSharedPtr() { return get_underlying_safe(indexIntoFileSharedPtr_); }
0288     bool wasLastEventJustRead() const;
0289     bool wasFirstEventJustRead() const;
0290     IndexIntoFile::IndexIntoFileItr indexIntoFileIter() const;
0291     void setPosition(IndexIntoFile::IndexIntoFileItr const& position);
0292     void initAssociationsFromSecondary(std::vector<BranchID> const&);
0293
0294     void setSignals(
0295         signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* preEventReadSource,
0296         signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* postEventReadSource);
0297
0298   private:
0299     void makeProcessBlockRootTrees(std::shared_ptr<InputFile> filePtr,
0300                                    int treeMaxVirtualSize,
0301                                    bool enablePrefetching,
0302                                    InputType inputType,
0303                                    StoredProcessBlockHelper const& storedProcessBlockHelper);
0304     bool skipThisEntry();
0305     void setIfFastClonable(int remainingEvents, int remainingLumis);
0306     void validateFile(InputType inputType, bool usingGoToEvent);
0307     void fillIndexIntoFile();
0308     EventAuxiliary fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry);
0309     EventAuxiliary const& fillThisEventAuxiliary();
0310     void fillEventToProcessBlockIndexes();
0311     bool fillEventHistory(EventAuxiliary& evtAux,
0312                           EventSelectionIDVector& eventSelectionIDs,
0313                           BranchListIndexes& branchListIndexes,
0314                           bool assertOnFailure = true);
0315     std::shared_ptr<LuminosityBlockAuxiliary> fillLumiAuxiliary();
0316     std::shared_ptr<RunAuxiliary> fillRunAuxiliary();
0317     std::string const& newBranchToOldBranch(std::string const& newBranch) const;
0318     void setPresenceInProductRegistry(ProductRegistry&, StoredProcessBlockHelper const&);
0319     void markBranchToBeDropped(bool dropDescendants,
0320                                BranchDescription const& branch,
0321                                std::set<BranchID>& branchesToDrop,
0322                                std::map<BranchID, BranchID> const& droppedToKeptAlias) const;
0323     void dropOnInputAndReorder(ProductRegistry&,
0324                                ProductSelectorRules const&,
0325                                bool dropDescendants,
0326                                InputType,
0327                                StoredProcessBlockHelper&,
0328                                ProcessBlockHelper const*);
0329     void dropProcessesAndReorder(StoredProcessBlockHelper&,
0330                                  std::set<std::string> const& processesWithKeptProcessBlockProducts,
0331                                  ProcessBlockHelper const*);
0332
0333     void readParentageTree(InputType inputType);
0334     void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap,
0335                                   InputType inputType);  // backward compatibility
0336     void readEventHistoryTree();
0337     bool isDuplicateEvent();
0338
0339     void initializeDuplicateChecker(std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
0340                                     std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile);
0341
0342     std::unique_ptr<MakeProvenanceReader> makeProvenanceReaderMaker(InputType inputType);
0343     std::shared_ptr<ProductProvenanceRetriever> makeProductProvenanceRetriever(unsigned int iStreamIndex);
0344
0345     std::shared_ptr<RunAuxiliary const> savedRunAuxiliary() const { return get_underlying_safe(savedRunAuxiliary_); }
0346     std::shared_ptr<RunAuxiliary>& savedRunAuxiliary() { return get_underlying_safe(savedRunAuxiliary_); }
0347
0348     std::shared_ptr<BranchChildren const> branchChildren() const { return get_underlying_safe(branchChildren_); }
0349     std::shared_ptr<BranchChildren>& branchChildren() { return get_underlying_safe(branchChildren_); }
0350
0351     std::shared_ptr<ProductProvenanceRetriever const> eventProductProvenanceRetriever(size_t index) const {
0352       return get_underlying_safe(eventProductProvenanceRetrievers_[index]);
0353     }
0354     std::shared_ptr<ProductProvenanceRetriever>& eventProductProvenanceRetriever(size_t index) {
0355       return get_underlying_safe(eventProductProvenanceRetrievers_[index]);
0356     }
0357
0358     std::string const file_;
0359     std::string const logicalFile_;
0360     ProcessConfiguration const& processConfiguration_;
0361     edm::propagate_const<ProcessHistoryRegistry*> processHistoryRegistry_;  // We don't own this
0362     edm::propagate_const<std::shared_ptr<InputFile>> filePtr_;
0363     edm::propagate_const<std::shared_ptr<EventSkipperByID>> eventSkipperByID_;
0364     FileFormatVersion fileFormatVersion_;
0365     FileID fid_;
0366     edm::propagate_const<std::shared_ptr<IndexIntoFile>> indexIntoFileSharedPtr_;
0367     IndexIntoFile& indexIntoFile_;
0368     std::vector<ProcessHistoryID>& orderedProcessHistoryIDs_;
0369     IndexIntoFile::IndexIntoFileItr indexIntoFileBegin_;
0370     IndexIntoFile::IndexIntoFileItr indexIntoFileEnd_;
0371     IndexIntoFile::IndexIntoFileItr indexIntoFileIter_;
0372     edm::propagate_const<std::unique_ptr<StoredMergeableRunProductMetadata>> storedMergeableRunProductMetadata_;
0373     std::vector<EventProcessHistoryID> eventProcessHistoryIDs_;                   // backward compatibility
0374     std::vector<EventProcessHistoryID>::const_iterator eventProcessHistoryIter_;  // backward compatibility
0375     edm::propagate_const<std::shared_ptr<RunAuxiliary>> savedRunAuxiliary_;
0376     bool skipAnyEvents_;
0377     bool noRunLumiSort_;
0378     bool noEventSort_;
0379     bool enforceGUIDInFileName_;
0380     int whyNotFastClonable_;
0381     std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
0382     bool branchListIndexesUnchanged_;
0383     EventAuxiliary eventAuxCache_;  //Should only be used by fillThisEventAuxiliary()
0384     RootTree eventTree_;
0385     RootTree lumiTree_;
0386     RootTree runTree_;
0387     std::vector<edm::propagate_const<std::unique_ptr<RootTree>>> processBlockTrees_;
0388     unsigned int currentProcessBlockTree_ = 0;
0389     std::vector<edm::propagate_const<RootTree*>> treePointers_;
0390     //Should only be used by fillThisEventAuxiliary()
0391     IndexIntoFile::EntryNumber_t lastEventEntryNumberRead_;
0392     std::shared_ptr<ProductRegistry const> productRegistry_;
0393     std::shared_ptr<BranchIDLists const> branchIDLists_;
0394     edm::propagate_const<std::shared_ptr<BranchIDListHelper>> branchIDListHelper_;
0395     edm::propagate_const<ProcessBlockHelper*> processBlockHelper_;
0396     edm::propagate_const<std::unique_ptr<StoredProcessBlockHelper>> storedProcessBlockHelper_;
0397     edm::propagate_const<std::unique_ptr<ThinnedAssociationsHelper>> fileThinnedAssociationsHelper_;
0398     edm::propagate_const<std::shared_ptr<ThinnedAssociationsHelper>> thinnedAssociationsHelper_;
0399     InputSource::ProcessingMode processingMode_;
0400     edm::propagate_const<RunHelperBase*> runHelper_;
0401     std::map<std::string, std::string> newBranchToOldBranch_;
0402     edm::propagate_const<TTree*> eventHistoryTree_;  // backward compatibility
0403     EventToProcessBlockIndexes eventToProcessBlockIndexes_;
0404     edm::propagate_const<TBranch*> eventToProcessBlockIndexesBranch_;
0405     edm::propagate_const<std::unique_ptr<History>> history_;  // backward compatibility
0406     edm::propagate_const<std::shared_ptr<BranchChildren>> branchChildren_;
0407     edm::propagate_const<std::shared_ptr<DuplicateChecker>> duplicateChecker_;
0408     edm::propagate_const<std::unique_ptr<ProvenanceAdaptor>> provenanceAdaptor_;  // backward comatibility
0409     edm::propagate_const<std::unique_ptr<MakeProvenanceReader>> provenanceReaderMaker_;
0410     std::vector<edm::propagate_const<std::shared_ptr<ProductProvenanceRetriever>>> eventProductProvenanceRetrievers_;
0411     std::vector<ParentageID> parentageIDLookup_;
0412     edm::propagate_const<std::unique_ptr<DaqProvenanceHelper>> daqProvenanceHelper_;
0413     edm::propagate_const<TClass*> edProductClass_;
0414     InputType inputType_;
0415   };  // class RootFile
0416
0417 }  // namespace edm
0418 #endif