File indexing completed on 2022-12-28 23:27:24
0001
0002
0003
0004 #include "RootFile.h"
0005 #include "DuplicateChecker.h"
0006 #include "InputFile.h"
0007 #include "ProvenanceAdaptor.h"
0008 #include "RunHelper.h"
0009
0010 #include "DataFormats/Common/interface/setIsMergeable.h"
0011 #include "DataFormats/Common/interface/ThinnedAssociation.h"
0012 #include "DataFormats/Provenance/interface/BranchDescription.h"
0013 #include "DataFormats/Provenance/interface/BranchIDListHelper.h"
0014 #include "DataFormats/Provenance/interface/BranchType.h"
0015 #include "DataFormats/Provenance/interface/EventEntryInfo.h"
0016 #include "DataFormats/Provenance/interface/ParameterSetBlob.h"
0017 #include "DataFormats/Provenance/interface/ParentageRegistry.h"
0018 #include "DataFormats/Provenance/interface/ProcessHistoryID.h"
0019 #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
0020 #include "DataFormats/Provenance/interface/ProductRegistry.h"
0021 #include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h"
0022 #include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h"
0023 #include "DataFormats/Provenance/interface/StoredProductProvenance.h"
0024 #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h"
0025 #include "DataFormats/Provenance/interface/RunID.h"
0026 #include "FWCore/Common/interface/ProcessBlockHelper.h"
0027 #include "FWCore/Framework/interface/FileBlock.h"
0028 #include "FWCore/Framework/interface/EventPrincipal.h"
0029 #include "FWCore/Framework/interface/ProductSelector.h"
0030 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
0031 #include "FWCore/Framework/interface/MergeableRunProductMetadata.h"
0032 #include "FWCore/Framework/interface/ProcessBlockPrincipal.h"
0033 #include "FWCore/Framework/interface/RunPrincipal.h"
0034 #include "FWCore/Framework/interface/SharedResourcesAcquirer.h"
0035 #include "FWCore/Framework/interface/SharedResourcesRegistry.h"
0036 #include "FWCore/Framework/interface/DelayedReader.h"
0037 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0038 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0039 #include "FWCore/ParameterSet/interface/Registry.h"
0040 #include "FWCore/Sources/interface/EventSkipperByID.h"
0041 #include "FWCore/Sources/interface/DaqProvenanceHelper.h"
0042 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0043 #include "FWCore/Utilities/interface/Algorithms.h"
0044 #include "FWCore/Utilities/interface/do_nothing_deleter.h"
0045 #include "FWCore/Utilities/interface/EDMException.h"
0046 #include "FWCore/Utilities/interface/FriendlyName.h"
0047 #include "FWCore/Utilities/interface/GlobalIdentifier.h"
0048 #include "FWCore/Utilities/interface/ReleaseVersion.h"
0049 #include "FWCore/Utilities/interface/stemFromPath.h"
0050 #include "FWCore/Utilities/interface/thread_safety_macros.h"
0051 #include "FWCore/Version/interface/GetReleaseVersion.h"
0052 #include "IOPool/Common/interface/getWrapperBasePtr.h"
0053
0054 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0055 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0056
0057
0058 #include "DataFormats/Provenance/interface/EventAux.h"
0059 #include "DataFormats/Provenance/interface/LuminosityBlockAux.h"
0060 #include "DataFormats/Provenance/interface/RunAux.h"
0061 #include "FWCore/ParameterSet/interface/ParameterSetConverter.h"
0062
0063 #include "Rtypes.h"
0064 #include "TClass.h"
0065 #include "TString.h"
0066 #include "TTree.h"
0067 #include "TTreeCache.h"
0068
0069 #include <algorithm>
0070 #include <cassert>
0071 #include <list>
0072
0073 namespace edm {
0074
0075
0076 class MakeDummyProvenanceReader : public MakeProvenanceReader {
0077 public:
0078 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0079 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0080 };
0081 class MakeOldProvenanceReader : public MakeProvenanceReader {
0082 public:
0083 MakeOldProvenanceReader(std::unique_ptr<EntryDescriptionMap>&& entryDescriptionMap)
0084 : MakeProvenanceReader(), entryDescriptionMap_(std::move(entryDescriptionMap)) {}
0085 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0086 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0087
0088 private:
0089 edm::propagate_const<std::unique_ptr<EntryDescriptionMap>> entryDescriptionMap_;
0090 };
0091 class MakeFullProvenanceReader : public MakeProvenanceReader {
0092 public:
0093 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0094 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0095 };
0096 class MakeReducedProvenanceReader : public MakeProvenanceReader {
0097 public:
0098 MakeReducedProvenanceReader(std::vector<ParentageID> const& parentageIDLookup)
0099 : parentageIDLookup_(parentageIDLookup) {}
0100 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0101 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0102
0103 private:
0104 std::vector<ParentageID> const& parentageIDLookup_;
0105 };
0106
0107 namespace {
0108 void checkReleaseVersion(std::vector<ProcessHistory> processHistoryVector, std::string const& fileName) {
0109 std::string releaseVersion = getReleaseVersion();
0110 releaseversion::DecomposedReleaseVersion currentRelease(releaseVersion);
0111 for (auto const& ph : processHistoryVector) {
0112 for (auto const& pc : ph) {
0113 if (releaseversion::isEarlierRelease(currentRelease, pc.releaseVersion())) {
0114 throw Exception(errors::FormatIncompatibility)
0115 << "The release you are using, " << getReleaseVersion() << " , predates\n"
0116 << "a release (" << pc.releaseVersion() << ") used in writing the input file, " << fileName << ".\n"
0117 << "Forward compatibility cannot be supported.\n";
0118 }
0119 }
0120 }
0121 }
0122 }
0123
0124
0125 class RootFileEventFinder : public IndexIntoFile::EventFinder {
0126 public:
0127 explicit RootFileEventFinder(RootTree& eventTree) : eventTree_(eventTree) {}
0128 ~RootFileEventFinder() override {}
0129
0130 EventNumber_t getEventNumberOfEntry(roottree::EntryNumber entry) const override {
0131 roottree::EntryNumber saveEntry = eventTree_.entryNumber();
0132 eventTree_.setEntryNumber(entry);
0133 EventAuxiliary eventAux;
0134 EventAuxiliary* pEvAux = &eventAux;
0135 eventTree_.fillAux<EventAuxiliary>(pEvAux);
0136 eventTree_.setEntryNumber(saveEntry);
0137 return eventAux.event();
0138 }
0139
0140 private:
0141 RootTree& eventTree_;
0142 };
0143
0144
0145 RootFile::RootFile(std::string const& fileName,
0146 ProcessConfiguration const& processConfiguration,
0147 std::string const& logicalFileName,
0148 std::shared_ptr<InputFile> filePtr,
0149 std::shared_ptr<EventSkipperByID> eventSkipperByID,
0150 bool skipAnyEvents,
0151 int remainingEvents,
0152 int remainingLumis,
0153 unsigned int nStreams,
0154 unsigned int treeCacheSize,
0155 int treeMaxVirtualSize,
0156 InputSource::ProcessingMode processingMode,
0157 RunHelperBase* runHelper,
0158 bool noRunLumiSort,
0159 bool noEventSort,
0160 ProductSelectorRules const& productSelectorRules,
0161 InputType inputType,
0162 std::shared_ptr<BranchIDListHelper> branchIDListHelper,
0163 ProcessBlockHelper* processBlockHelper,
0164 std::shared_ptr<ThinnedAssociationsHelper> thinnedAssociationsHelper,
0165 std::vector<BranchID> const* associationsFromSecondary,
0166 std::shared_ptr<DuplicateChecker> duplicateChecker,
0167 bool dropDescendants,
0168 ProcessHistoryRegistry& processHistoryRegistry,
0169 std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
0170 std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
0171 std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
0172 bool bypassVersionCheck,
0173 bool labelRawDataLikeMC,
0174 bool usingGoToEvent,
0175 bool enablePrefetching,
0176 bool enforceGUIDInFileName)
0177 : file_(fileName),
0178 logicalFile_(logicalFileName),
0179 processConfiguration_(processConfiguration),
0180 processHistoryRegistry_(&processHistoryRegistry),
0181 filePtr_(filePtr),
0182 eventSkipperByID_(eventSkipperByID),
0183 fileFormatVersion_(),
0184 fid_(),
0185 indexIntoFileSharedPtr_(new IndexIntoFile),
0186 indexIntoFile_(*indexIntoFileSharedPtr_),
0187 orderedProcessHistoryIDs_(orderedProcessHistoryIDs),
0188 indexIntoFileBegin_(indexIntoFile_.begin(
0189 noRunLumiSort ? IndexIntoFile::entryOrder
0190 : (noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder))),
0191 indexIntoFileEnd_(indexIntoFileBegin_),
0192 indexIntoFileIter_(indexIntoFileBegin_),
0193 storedMergeableRunProductMetadata_((inputType == InputType::Primary) ? new StoredMergeableRunProductMetadata
0194 : nullptr),
0195 eventProcessHistoryIDs_(),
0196 eventProcessHistoryIter_(eventProcessHistoryIDs_.begin()),
0197 savedRunAuxiliary_(),
0198 skipAnyEvents_(skipAnyEvents),
0199 noRunLumiSort_(noRunLumiSort),
0200 noEventSort_(noEventSort),
0201 enforceGUIDInFileName_(enforceGUIDInFileName),
0202 whyNotFastClonable_(0),
0203 hasNewlyDroppedBranch_(),
0204 branchListIndexesUnchanged_(false),
0205 eventAuxCache_(),
0206 eventTree_(filePtr,
0207 InEvent,
0208 nStreams,
0209 treeMaxVirtualSize,
0210 treeCacheSize,
0211 roottree::defaultLearningEntries,
0212 enablePrefetching,
0213 inputType),
0214 lumiTree_(filePtr,
0215 InLumi,
0216 1,
0217 treeMaxVirtualSize,
0218 roottree::defaultNonEventCacheSize,
0219 roottree::defaultNonEventLearningEntries,
0220 enablePrefetching,
0221 inputType),
0222 runTree_(filePtr,
0223 InRun,
0224 1,
0225 treeMaxVirtualSize,
0226 roottree::defaultNonEventCacheSize,
0227 roottree::defaultNonEventLearningEntries,
0228 enablePrefetching,
0229 inputType),
0230 treePointers_(),
0231 lastEventEntryNumberRead_(IndexIntoFile::invalidEntry),
0232 productRegistry_(),
0233 branchIDLists_(),
0234 branchIDListHelper_(branchIDListHelper),
0235 processBlockHelper_(processBlockHelper),
0236 fileThinnedAssociationsHelper_(),
0237 thinnedAssociationsHelper_(thinnedAssociationsHelper),
0238 processingMode_(processingMode),
0239 runHelper_(runHelper),
0240 newBranchToOldBranch_(),
0241 eventHistoryTree_(nullptr),
0242 eventToProcessBlockIndexesBranch_(
0243 inputType == InputType::Primary
0244 ? eventTree_.tree()->GetBranch(poolNames::eventToProcessBlockIndexesBranchName().c_str())
0245 : nullptr),
0246 history_(),
0247 branchChildren_(new BranchChildren),
0248 duplicateChecker_(duplicateChecker),
0249 provenanceAdaptor_(),
0250 provenanceReaderMaker_(),
0251 eventProductProvenanceRetrievers_(),
0252 parentageIDLookup_(),
0253 daqProvenanceHelper_(),
0254 edProductClass_(TypeWithDict::byName("edm::WrapperBase").getClass()),
0255 inputType_(inputType) {
0256 hasNewlyDroppedBranch_.fill(false);
0257
0258 treePointers_.resize(3);
0259 treePointers_[InEvent] = &eventTree_;
0260 treePointers_[InLumi] = &lumiTree_;
0261 treePointers_[InRun] = &runTree_;
0262
0263
0264
0265 std::unique_ptr<TTree> metaDataTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::metaDataTreeName().c_str())));
0266 if (nullptr == metaDataTree.get()) {
0267 throw Exception(errors::FileReadError)
0268 << "Could not find tree " << poolNames::metaDataTreeName() << " in the input file.\n";
0269 }
0270
0271
0272
0273
0274 FileFormatVersion* fftPtr = &fileFormatVersion_;
0275 if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) {
0276 TBranch* fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str());
0277 fft->SetAddress(&fftPtr);
0278 roottree::getEntry(fft, 0);
0279 metaDataTree->SetBranchAddress(poolNames::fileFormatVersionBranchName().c_str(), &fftPtr);
0280 }
0281
0282 FileID* fidPtr = &fid_;
0283 if (metaDataTree->FindBranch(poolNames::fileIdentifierBranchName().c_str()) != nullptr) {
0284 metaDataTree->SetBranchAddress(poolNames::fileIdentifierBranchName().c_str(), &fidPtr);
0285 }
0286
0287 IndexIntoFile* iifPtr = &indexIntoFile_;
0288 if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) {
0289 metaDataTree->SetBranchAddress(poolNames::indexIntoFileBranchName().c_str(), &iifPtr);
0290 }
0291
0292 storedProcessBlockHelper_ = std::make_unique<StoredProcessBlockHelper>();
0293 StoredProcessBlockHelper& storedProcessBlockHelper = *storedProcessBlockHelper_;
0294 StoredProcessBlockHelper* pStoredProcessBlockHelper = storedProcessBlockHelper_.get();
0295 if (inputType == InputType::Primary) {
0296 if (metaDataTree->FindBranch(poolNames::processBlockHelperBranchName().c_str()) != nullptr) {
0297 metaDataTree->SetBranchAddress(poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper);
0298 }
0299 }
0300
0301 StoredMergeableRunProductMetadata* smrc = nullptr;
0302 if (inputType == InputType::Primary) {
0303 smrc = &*storedMergeableRunProductMetadata_;
0304 if (metaDataTree->FindBranch(poolNames::mergeableRunProductMetadataBranchName().c_str()) != nullptr) {
0305 metaDataTree->SetBranchAddress(poolNames::mergeableRunProductMetadataBranchName().c_str(), &smrc);
0306 }
0307 }
0308
0309
0310
0311 ProductRegistry inputProdDescReg;
0312 ProductRegistry* ppReg = &inputProdDescReg;
0313 metaDataTree->SetBranchAddress(poolNames::productDescriptionBranchName().c_str(), (&ppReg));
0314
0315 using PsetMap = std::map<ParameterSetID, ParameterSetBlob>;
0316 PsetMap psetMap;
0317 PsetMap* psetMapPtr = &psetMap;
0318 if (metaDataTree->FindBranch(poolNames::parameterSetMapBranchName().c_str()) != nullptr) {
0319
0320 assert(!fileFormatVersion().parameterSetsTree());
0321 metaDataTree->SetBranchAddress(poolNames::parameterSetMapBranchName().c_str(), &psetMapPtr);
0322 } else {
0323 assert(fileFormatVersion().parameterSetsTree());
0324
0325 std::unique_ptr<TTree> psetTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::parameterSetsTreeName().c_str())));
0326 if (nullptr == psetTree.get()) {
0327 throw Exception(errors::FileReadError)
0328 << "Could not find tree " << poolNames::parameterSetsTreeName() << " in the input file.\n";
0329 }
0330
0331 using IdToBlobs = std::pair<ParameterSetID, ParameterSetBlob>;
0332 IdToBlobs idToBlob;
0333 IdToBlobs* pIdToBlob = &idToBlob;
0334 psetTree->SetBranchAddress(poolNames::idToParameterSetBlobsBranchName().c_str(), &pIdToBlob);
0335
0336 std::unique_ptr<TTreeCache> psetTreeCache =
0337 roottree::trainCache(psetTree.get(), *filePtr_, roottree::defaultNonEventCacheSize, "*");
0338 psetTreeCache->SetEnablePrefetching(false);
0339 filePtr_->SetCacheRead(psetTreeCache.get());
0340 for (Long64_t i = 0; i != psetTree->GetEntries(); ++i) {
0341 psetTree->GetEntry(i);
0342 psetMap.insert(idToBlob);
0343 }
0344 filePtr_->SetCacheRead(nullptr);
0345 }
0346
0347
0348 ProcessHistoryRegistry::collection_type pHistMap;
0349 ProcessHistoryRegistry::collection_type* pHistMapPtr = &pHistMap;
0350 if (metaDataTree->FindBranch(poolNames::processHistoryMapBranchName().c_str()) != nullptr) {
0351 metaDataTree->SetBranchAddress(poolNames::processHistoryMapBranchName().c_str(), &pHistMapPtr);
0352 }
0353
0354 ProcessHistoryRegistry::vector_type pHistVector;
0355 ProcessHistoryRegistry::vector_type* pHistVectorPtr = &pHistVector;
0356 if (metaDataTree->FindBranch(poolNames::processHistoryBranchName().c_str()) != nullptr) {
0357 metaDataTree->SetBranchAddress(poolNames::processHistoryBranchName().c_str(), &pHistVectorPtr);
0358 }
0359
0360
0361 ProcessConfigurationVector processConfigurations;
0362 ProcessConfigurationVector* procConfigVectorPtr = &processConfigurations;
0363 if (metaDataTree->FindBranch(poolNames::processConfigurationBranchName().c_str()) != nullptr) {
0364 metaDataTree->SetBranchAddress(poolNames::processConfigurationBranchName().c_str(), &procConfigVectorPtr);
0365 }
0366
0367 auto branchIDListsAPtr = std::make_unique<BranchIDLists>();
0368 BranchIDLists* branchIDListsPtr = branchIDListsAPtr.get();
0369 if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) != nullptr) {
0370 metaDataTree->SetBranchAddress(poolNames::branchIDListBranchName().c_str(), &branchIDListsPtr);
0371 }
0372
0373 ThinnedAssociationsHelper* thinnedAssociationsHelperPtr;
0374 if (inputType != InputType::SecondarySource) {
0375 fileThinnedAssociationsHelper_ =
0376 std::make_unique<ThinnedAssociationsHelper>();
0377 thinnedAssociationsHelperPtr = fileThinnedAssociationsHelper_.get();
0378 if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) != nullptr) {
0379 metaDataTree->SetBranchAddress(poolNames::thinnedAssociationsHelperBranchName().c_str(),
0380 &thinnedAssociationsHelperPtr);
0381 }
0382 }
0383
0384 BranchChildren* branchChildrenBuffer = branchChildren_.get();
0385 if (metaDataTree->FindBranch(poolNames::productDependenciesBranchName().c_str()) != nullptr) {
0386 metaDataTree->SetBranchAddress(poolNames::productDependenciesBranchName().c_str(), &branchChildrenBuffer);
0387 }
0388
0389
0390 std::vector<EventProcessHistoryID>* eventHistoryIDsPtr = &eventProcessHistoryIDs_;
0391 if (metaDataTree->FindBranch(poolNames::eventHistoryBranchName().c_str()) != nullptr) {
0392 metaDataTree->SetBranchAddress(poolNames::eventHistoryBranchName().c_str(), &eventHistoryIDsPtr);
0393 }
0394
0395 if (metaDataTree->FindBranch(poolNames::moduleDescriptionMapBranchName().c_str()) != nullptr) {
0396 if (metaDataTree->GetBranch(poolNames::moduleDescriptionMapBranchName().c_str())->GetSplitLevel() != 0) {
0397 metaDataTree->SetBranchStatus((poolNames::moduleDescriptionMapBranchName() + ".*").c_str(), false);
0398 } else {
0399 metaDataTree->SetBranchStatus(poolNames::moduleDescriptionMapBranchName().c_str(), false);
0400 }
0401 }
0402
0403
0404 roottree::getEntry(metaDataTree.get(), 0);
0405
0406 eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin();
0407
0408
0409 readEventHistoryTree();
0410
0411 ParameterSetConverter::ParameterSetIdConverter psetIdConverter;
0412 if (!fileFormatVersion().triggerPathsTracked()) {
0413 ParameterSetConverter converter(psetMap, psetIdConverter, fileFormatVersion().parameterSetsByReference());
0414 } else {
0415
0416 pset::Registry& psetRegistry = *pset::Registry::instance();
0417 for (auto const& psetEntry : psetMap) {
0418 ParameterSet pset(psetEntry.second.pset());
0419 pset.setID(psetEntry.first);
0420
0421 if (inputType != InputType::SecondarySource) {
0422 psetRegistry.insertMapped(pset);
0423 }
0424 }
0425 }
0426 if (!fileFormatVersion().splitProductIDs()) {
0427
0428
0429 provenanceAdaptor_ = std::make_unique<ProvenanceAdaptor>(
0430 inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, true);
0431
0432 branchIDLists_ = provenanceAdaptor_->branchIDLists();
0433 } else {
0434 if (!fileFormatVersion().triggerPathsTracked()) {
0435
0436
0437 provenanceAdaptor_ = std::make_unique<ProvenanceAdaptor>(
0438 inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, false);
0439 }
0440
0441 if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) == nullptr) {
0442 throw Exception(errors::EventCorruption) << "Failed to find branchIDLists branch in metaData tree.\n";
0443 }
0444 branchIDLists_.reset(branchIDListsAPtr.release());
0445 }
0446
0447 if (fileFormatVersion().hasThinnedAssociations()) {
0448 if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) == nullptr) {
0449 throw Exception(errors::EventCorruption)
0450 << "Failed to find thinnedAssociationsHelper branch in metaData tree.\n";
0451 }
0452 }
0453
0454 if (!bypassVersionCheck) {
0455 checkReleaseVersion(pHistVector, file());
0456 }
0457
0458 if (labelRawDataLikeMC) {
0459 std::string const rawData("FEDRawDataCollection");
0460 std::string const source("source");
0461 ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
0462 BranchKey finder(rawData, source, "", "");
0463 ProductRegistry::ProductList::iterator it = pList.lower_bound(finder);
0464 if (it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source) {
0465
0466
0467
0468 it->second.init();
0469
0470 daqProvenanceHelper_ = std::make_unique<DaqProvenanceHelper>(it->second.unwrappedTypeID());
0471
0472 BranchDescription const& newBD = daqProvenanceHelper_->branchDescription();
0473
0474 daqProvenanceHelper_->saveInfo(it->second, newBD);
0475
0476 newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), it->second.branchName()));
0477
0478 pList.erase(it);
0479
0480 it = pList.lower_bound(finder);
0481 assert(!(it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source));
0482
0483 inputProdDescReg.copyProduct(newBD);
0484
0485 daqProvenanceHelper_->fixMetaData(processConfigurations, pHistVector);
0486 daqProvenanceHelper_->fixMetaData(*branchIDLists_);
0487 daqProvenanceHelper_->fixMetaData(*branchChildren_);
0488 }
0489 }
0490
0491 for (auto const& history : pHistVector) {
0492 processHistoryRegistry.registerProcessHistory(history);
0493 }
0494
0495 eventTree_.trainCache(BranchTypeToAuxiliaryBranchName(InEvent).c_str());
0496
0497
0498
0499 if (inputType == InputType::Primary) {
0500 branchListIndexesUnchanged_ = branchIDListHelper_->updateFromInput(*branchIDLists_);
0501 }
0502
0503 validateFile(inputType, usingGoToEvent);
0504
0505
0506
0507
0508 provenanceReaderMaker_ = std::unique_ptr<MakeProvenanceReader>(makeProvenanceReaderMaker(inputType).release());
0509
0510
0511 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
0512 whyNotFastClonable_ += FileBlock::EventsOrLumisSelectedByID;
0513 }
0514
0515 initializeDuplicateChecker(indexesIntoFiles, currentIndexIntoFile);
0516 indexIntoFileIter_ = indexIntoFileBegin_ = indexIntoFile_.begin(
0517 noRunLumiSort ? IndexIntoFile::entryOrder
0518 : (noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder));
0519 indexIntoFileEnd_ = indexIntoFile_.end(
0520 noRunLumiSort ? IndexIntoFile::entryOrder
0521 : (noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder));
0522 runHelper_->setForcedRunOffset(indexIntoFileBegin_ == indexIntoFileEnd_ ? 1 : indexIntoFileBegin_.run());
0523 eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin();
0524
0525 makeProcessBlockRootTrees(filePtr, treeMaxVirtualSize, enablePrefetching, inputType, storedProcessBlockHelper);
0526
0527 setPresenceInProductRegistry(inputProdDescReg, storedProcessBlockHelper);
0528
0529 auto newReg = std::make_unique<ProductRegistry>();
0530
0531
0532 {
0533 ProductRegistry::ProductList const& prodList = inputProdDescReg.productList();
0534 for (auto const& product : prodList) {
0535 BranchDescription const& prod = product.second;
0536 std::string newFriendlyName = friendlyname::friendlyName(prod.className());
0537 if (newFriendlyName == prod.friendlyClassName()) {
0538 newReg->copyProduct(prod);
0539 } else {
0540 if (fileFormatVersion().splitProductIDs()) {
0541 throw Exception(errors::UnimplementedFeature)
0542 << "Cannot change friendly class name algorithm without more development work\n"
0543 << "to update BranchIDLists and ThinnedAssociationsHelper. Contact the framework group.\n";
0544 }
0545 BranchDescription newBD(prod);
0546 newBD.updateFriendlyClassName();
0547 newReg->copyProduct(newBD);
0548 newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), prod.branchName()));
0549 }
0550 }
0551
0552 dropOnInputAndReorder(
0553 *newReg, productSelectorRules, dropDescendants, inputType, storedProcessBlockHelper, processBlockHelper);
0554
0555 if (inputType == InputType::SecondaryFile) {
0556 thinnedAssociationsHelper->updateFromSecondaryInput(*fileThinnedAssociationsHelper_,
0557 *associationsFromSecondary);
0558 } else if (inputType == InputType::Primary) {
0559 processBlockHelper->initializeFromPrimaryInput(storedProcessBlockHelper);
0560 thinnedAssociationsHelper->updateFromPrimaryInput(*fileThinnedAssociationsHelper_);
0561 }
0562
0563 if (inputType == InputType::Primary) {
0564 for (auto& product : newReg->productListUpdator()) {
0565 setIsMergeable(product.second);
0566 }
0567 }
0568
0569 for (auto& product : newReg->productListUpdator()) {
0570 product.second.setOnDemand(true);
0571 }
0572
0573 for (auto& processBlockTree : processBlockTrees_) {
0574 treePointers_.push_back(processBlockTree.get());
0575 }
0576
0577
0578 newReg->setFrozen(inputType != InputType::Primary);
0579 productRegistry_.reset(newReg.release());
0580 }
0581
0582
0583 ProductRegistry::ProductList const& prodList = productRegistry()->productList();
0584
0585 {
0586 std::vector<size_t> nBranches(treePointers_.size(), 0);
0587 for (auto const& product : prodList) {
0588 if (product.second.branchType() == InProcess) {
0589 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
0590 auto it = std::find(processes.begin(), processes.end(), product.second.processName());
0591 if (it != processes.end()) {
0592 auto index = std::distance(processes.begin(), it);
0593 ++nBranches[numberOfRunLumiEventProductTrees + index];
0594 }
0595 } else {
0596 ++nBranches[product.second.branchType()];
0597 }
0598 }
0599
0600 int i = 0;
0601 for (auto& t : treePointers_) {
0602 t->numberOfBranchesToAdd(nBranches[i]);
0603 ++i;
0604 }
0605 }
0606 for (auto const& product : prodList) {
0607 BranchDescription const& prod = product.second;
0608 if (prod.branchType() == InProcess) {
0609 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
0610 auto it = std::find(processes.begin(), processes.end(), prod.processName());
0611 if (it != processes.end()) {
0612 auto index = std::distance(processes.begin(), it);
0613 treePointers_[numberOfRunLumiEventProductTrees + index]->addBranch(prod,
0614 newBranchToOldBranch(prod.branchName()));
0615 }
0616 } else {
0617 treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName()));
0618 }
0619 }
0620
0621
0622 setIfFastClonable(remainingEvents, remainingLumis);
0623
0624
0625 indexIntoFile_.doneFileInitialization();
0626
0627
0628 eventTree_.resetTraining();
0629
0630
0631 runTree_.trainCache("*");
0632 lumiTree_.trainCache("*");
0633 for (auto& processBlockTree : processBlockTrees_) {
0634 processBlockTree->trainCache("*");
0635 }
0636 }
0637
0638 RootFile::~RootFile() {}
0639
0640 void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) {
0641
0642
0643 std::unique_ptr<TTree> entryDescriptionTree(
0644 dynamic_cast<TTree*>(filePtr_->Get(poolNames::entryDescriptionTreeName().c_str())));
0645 if (nullptr == entryDescriptionTree.get()) {
0646 throw Exception(errors::FileReadError)
0647 << "Could not find tree " << poolNames::entryDescriptionTreeName() << " in the input file.\n";
0648 }
0649
0650 EntryDescriptionID idBuffer;
0651 EntryDescriptionID* pidBuffer = &idBuffer;
0652 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), &pidBuffer);
0653
0654 EventEntryDescription entryDescriptionBuffer;
0655 EventEntryDescription* pEntryDescriptionBuffer = &entryDescriptionBuffer;
0656 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), &pEntryDescriptionBuffer);
0657
0658
0659 ParentageRegistry& registry = *ParentageRegistry::instance();
0660
0661 for (Long64_t i = 0, numEntries = entryDescriptionTree->GetEntries(); i < numEntries; ++i) {
0662 roottree::getEntry(entryDescriptionTree.get(), i);
0663 if (idBuffer != entryDescriptionBuffer.id()) {
0664 throw Exception(errors::EventCorruption) << "Corruption of EntryDescription tree detected.\n";
0665 }
0666 entryDescriptionMap.insert(std::make_pair(entryDescriptionBuffer.id(), entryDescriptionBuffer));
0667 Parentage parents;
0668 parents.setParents(entryDescriptionBuffer.parents());
0669 if (daqProvenanceHelper_) {
0670 ParentageID const oldID = parents.id();
0671 daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate());
0672 ParentageID newID = parents.id();
0673 if (newID != oldID) {
0674 daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID);
0675 }
0676 }
0677
0678 if (inputType != InputType::SecondarySource) {
0679 registry.insertMapped(parents);
0680 }
0681 }
0682 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), nullptr);
0683 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), nullptr);
0684 }
0685
0686 void RootFile::readParentageTree(InputType inputType) {
0687
0688
0689 std::unique_ptr<TTree> parentageTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::parentageTreeName().c_str())));
0690 if (nullptr == parentageTree.get()) {
0691 throw Exception(errors::FileReadError)
0692 << "Could not find tree " << poolNames::parentageTreeName() << " in the input file.\n";
0693 }
0694
0695 Parentage parents;
0696 Parentage* pParentageBuffer = &parents;
0697 parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), &pParentageBuffer);
0698
0699 ParentageRegistry& registry = *ParentageRegistry::instance();
0700
0701 parentageIDLookup_.reserve(parentageTree->GetEntries());
0702 for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) {
0703 roottree::getEntry(parentageTree.get(), i);
0704 if (daqProvenanceHelper_) {
0705 ParentageID const oldID = parents.id();
0706 daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate());
0707 ParentageID newID = parents.id();
0708 if (newID != oldID) {
0709 daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID);
0710 }
0711 }
0712
0713 if (inputType != InputType::SecondarySource) {
0714 registry.insertMapped(parents);
0715 }
0716 parentageIDLookup_.push_back(parents.id());
0717 }
0718 parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), nullptr);
0719 }
0720
0721 void RootFile::setIfFastClonable(int remainingEvents, int remainingLumis) {
0722 if (fileFormatVersion().noMetaDataTrees() and !fileFormatVersion().storedProductProvenanceUsed()) {
0723
0724 whyNotFastClonable_ += FileBlock::FileTooOld;
0725 return;
0726 }
0727 if (!fileFormatVersion().splitProductIDs()) {
0728 whyNotFastClonable_ += FileBlock::FileTooOld;
0729 return;
0730 }
0731 if (processingMode_ != InputSource::RunsLumisAndEvents) {
0732 whyNotFastClonable_ += FileBlock::NotProcessingEvents;
0733 return;
0734 }
0735
0736 IndexIntoFile::IndexIntoFileItr it = indexIntoFileBegin_;
0737 while (it != indexIntoFileEnd_ && it.getEntryType() != IndexIntoFile::kEvent) {
0738 ++it;
0739 }
0740 if (it == indexIntoFileEnd_) {
0741 whyNotFastClonable_ += FileBlock::NoEventsInFile;
0742 return;
0743 }
0744
0745
0746 IndexIntoFile::SortOrder sortOrder =
0747 (noRunLumiSort_ ? IndexIntoFile::entryOrder
0748 : (noEventSort_ ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder));
0749 if (!indexIntoFile_.iterationWillBeInEntryOrder(sortOrder)) {
0750 whyNotFastClonable_ += (noEventSort_ ? FileBlock::RunOrLumiNotContiguous : FileBlock::EventsToBeSorted);
0751 }
0752 if (skipAnyEvents_) {
0753 whyNotFastClonable_ += FileBlock::InitialEventsSkipped;
0754 }
0755 if (remainingEvents >= 0 && eventTree_.entries() > remainingEvents) {
0756 whyNotFastClonable_ += FileBlock::MaxEventsTooSmall;
0757 }
0758 if (remainingLumis >= 0 && lumiTree_.entries() > remainingLumis) {
0759 whyNotFastClonable_ += FileBlock::MaxLumisTooSmall;
0760 }
0761 if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) {
0762 whyNotFastClonable_ += FileBlock::DuplicateEventsRemoved;
0763 }
0764 }
0765
0766 std::shared_ptr<FileBlock> RootFile::createFileBlock() {
0767 std::vector<TTree*> processBlockTrees;
0768 std::vector<std::string> processesWithProcessBlockTrees;
0769 processBlockTrees.reserve(processBlockTrees_.size());
0770 processesWithProcessBlockTrees.reserve(processBlockTrees_.size());
0771 for (auto& processBlockTree : processBlockTrees_) {
0772 processBlockTrees.push_back(processBlockTree->tree());
0773 processesWithProcessBlockTrees.push_back(processBlockTree->processName());
0774 }
0775 return std::make_shared<FileBlock>(fileFormatVersion(),
0776 eventTree_.tree(),
0777 eventTree_.metaTree(),
0778 lumiTree_.tree(),
0779 lumiTree_.metaTree(),
0780 runTree_.tree(),
0781 runTree_.metaTree(),
0782 std::move(processBlockTrees),
0783 std::move(processesWithProcessBlockTrees),
0784 whyNotFastClonable(),
0785 hasNewlyDroppedBranch(),
0786 file_,
0787 branchListIndexesUnchanged(),
0788 modifiedIDs(),
0789 branchChildren());
0790 }
0791
0792 void RootFile::updateFileBlock(FileBlock& fileBlock) {
0793 std::vector<TTree*> processBlockTrees;
0794 std::vector<std::string> processesWithProcessBlockTrees;
0795 processBlockTrees.reserve(processBlockTrees_.size());
0796 processesWithProcessBlockTrees.reserve(processBlockTrees_.size());
0797 for (auto& processBlockTree : processBlockTrees_) {
0798 processBlockTrees.push_back(processBlockTree->tree());
0799 processesWithProcessBlockTrees.push_back(processBlockTree->processName());
0800 }
0801 fileBlock.updateTTreePointers(eventTree_.tree(),
0802 eventTree_.metaTree(),
0803 lumiTree_.tree(),
0804 lumiTree_.metaTree(),
0805 runTree_.tree(),
0806 runTree_.metaTree(),
0807 std::move(processBlockTrees),
0808 std::move(processesWithProcessBlockTrees));
0809 }
0810
0811 std::string const& RootFile::newBranchToOldBranch(std::string const& newBranch) const {
0812 std::map<std::string, std::string>::const_iterator it = newBranchToOldBranch_.find(newBranch);
0813 if (it != newBranchToOldBranch_.end()) {
0814 return it->second;
0815 }
0816 return newBranch;
0817 }
0818
0819 IndexIntoFile::IndexIntoFileItr RootFile::indexIntoFileIter() const { return indexIntoFileIter_; }
0820
0821 void RootFile::setPosition(IndexIntoFile::IndexIntoFileItr const& position) {
0822 indexIntoFileIter_.copyPosition(position);
0823 }
0824
0825 void RootFile::initAssociationsFromSecondary(std::vector<BranchID> const& associationsFromSecondary) {
0826 thinnedAssociationsHelper_->initAssociationsFromSecondary(associationsFromSecondary,
0827 *fileThinnedAssociationsHelper_);
0828 }
0829
0830 bool RootFile::skipThisEntry() {
0831 if (indexIntoFileIter_ == indexIntoFileEnd_) {
0832 return false;
0833 }
0834
0835 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
0836
0837 if (eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.lumi(), 0U)) {
0838 return true;
0839 }
0840
0841
0842 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent) {
0843 auto eventAux = fillEventAuxiliary(indexIntoFileIter_.entry());
0844 if (eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.lumi(), eventAux.id().event())) {
0845 return true;
0846 }
0847 }
0848
0849
0850 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun && eventSkipperByID_->skippingLumis()) {
0851
0852 if (indexIntoFileIter_.peekAheadAtLumi() == IndexIntoFile::invalidLumi) {
0853 return true;
0854 }
0855
0856 do {
0857 if (!eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.peekAheadAtLumi(), 0U)) {
0858 return false;
0859 }
0860 } while (indexIntoFileIter_.skipLumiInRun());
0861 return true;
0862 }
0863 }
0864 return false;
0865 }
0866
0867 bool RootFile::isDuplicateEvent() {
0868 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent);
0869 if (duplicateChecker_.get() == nullptr) {
0870 return false;
0871 }
0872 auto const eventAux = fillEventAuxiliary(indexIntoFileIter_.entry());
0873 return duplicateChecker_->isDuplicateAndCheckActive(indexIntoFileIter_.processHistoryIDIndex(),
0874 indexIntoFileIter_.run(),
0875 indexIntoFileIter_.lumi(),
0876 eventAux.id().event(),
0877 file_);
0878 }
0879
0880 bool RootFile::containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const {
0881 return indexIntoFile_.containsItem(run, lumi, event);
0882 }
0883
0884 IndexIntoFile::EntryType RootFile::getNextItemType(RunNumber_t& run,
0885 LuminosityBlockNumber_t& lumi,
0886 EventNumber_t& event) {
0887
0888 while (skipThisEntry()) {
0889 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun) {
0890 indexIntoFileIter_.advanceToNextRun();
0891 } else if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi) {
0892 indexIntoFileIter_.advanceToNextLumiOrRun();
0893 } else {
0894 ++indexIntoFileIter_;
0895 }
0896 }
0897
0898 IndexIntoFile::EntryType entryType = indexIntoFileIter_.getEntryType();
0899 if (entryType == IndexIntoFile::kEnd) {
0900 return IndexIntoFile::kEnd;
0901 }
0902 if (entryType == IndexIntoFile::kRun) {
0903 run = indexIntoFileIter_.run();
0904 runHelper_->checkForNewRun(run, indexIntoFileIter_.peekAheadAtLumi());
0905 return IndexIntoFile::kRun;
0906 } else if (processingMode_ == InputSource::Runs) {
0907 indexIntoFileIter_.advanceToNextRun();
0908 return getNextItemType(run, lumi, event);
0909 }
0910 if (entryType == IndexIntoFile::kLumi) {
0911 run = indexIntoFileIter_.run();
0912 lumi = indexIntoFileIter_.lumi();
0913 return IndexIntoFile::kLumi;
0914 } else if (processingMode_ == InputSource::RunsAndLumis) {
0915 indexIntoFileIter_.advanceToNextLumiOrRun();
0916 return getNextItemType(run, lumi, event);
0917 }
0918 if (isDuplicateEvent()) {
0919 ++indexIntoFileIter_;
0920 return getNextItemType(run, lumi, event);
0921 }
0922 run = indexIntoFileIter_.run();
0923 lumi = indexIntoFileIter_.lumi();
0924 auto eventAux = fillEventAuxiliary(indexIntoFileIter_.entry());
0925 event = eventAux.event();
0926 return IndexIntoFile::kEvent;
0927 }
0928
0929 bool RootFile::wasLastEventJustRead() const {
0930 IndexIntoFile::IndexIntoFileItr itr(indexIntoFileIter_);
0931 itr.advanceToEvent();
0932 return itr.getEntryType() == IndexIntoFile::kEnd;
0933 }
0934
0935 bool RootFile::wasFirstEventJustRead() const {
0936 IndexIntoFile::IndexIntoFileItr itr(indexIntoFileIter_);
0937 int phIndex;
0938 RunNumber_t run;
0939 LuminosityBlockNumber_t lumi;
0940 IndexIntoFile::EntryNumber_t eventEntry;
0941 itr.skipEventBackward(phIndex, run, lumi, eventEntry);
0942 itr.skipEventBackward(phIndex, run, lumi, eventEntry);
0943 return eventEntry == IndexIntoFile::invalidEntry;
0944 }
0945
0946 namespace {
0947 struct RunItem {
0948 RunItem(ProcessHistoryID const& phid, RunNumber_t const& run) : phid_(phid), run_(run) {}
0949 ProcessHistoryID phid_;
0950 RunNumber_t run_;
0951 };
0952 struct RunItemSortByRun {
0953 bool operator()(RunItem const& a, RunItem const& b) const { return a.run_ < b.run_; }
0954 };
0955 struct RunItemSortByRunPhid {
0956 bool operator()(RunItem const& a, RunItem const& b) const {
0957 return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.phid_ < b.phid_);
0958 }
0959 };
0960 struct LumiItem {
0961 LumiItem(ProcessHistoryID const& phid,
0962 RunNumber_t const& run,
0963 LuminosityBlockNumber_t const& lumi,
0964 IndexIntoFile::EntryNumber_t const& entry)
0965 : phid_(phid),
0966 run_(run),
0967 lumi_(lumi),
0968 firstEventEntry_(entry),
0969 lastEventEntry_(entry == IndexIntoFile::invalidEntry ? IndexIntoFile::invalidEntry : entry + 1) {}
0970 ProcessHistoryID phid_;
0971 RunNumber_t run_;
0972 LuminosityBlockNumber_t lumi_;
0973 IndexIntoFile::EntryNumber_t firstEventEntry_;
0974 IndexIntoFile::EntryNumber_t lastEventEntry_;
0975 };
0976 struct LumiItemSortByRunLumi {
0977 bool operator()(LumiItem const& a, LumiItem const& b) const {
0978 return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.lumi_ < b.lumi_);
0979 }
0980 };
0981 struct LumiItemSortByRunLumiPhid {
0982 bool operator()(LumiItem const& a, LumiItem const& b) const {
0983 if (a.run_ < b.run_)
0984 return true;
0985 if (b.run_ < a.run_)
0986 return false;
0987 if (a.lumi_ < b.lumi_)
0988 return true;
0989 if (b.lumi_ < a.lumi_)
0990 return false;
0991 return a.phid_ < b.phid_;
0992 }
0993 };
0994 }
0995
0996 void RootFile::fillIndexIntoFile() {
0997
0998
0999
1000
1001
1002
1003
1004 assert(!fileFormatVersion().hasIndexIntoFile());
1005
1006 typedef std::list<LumiItem> LumiList;
1007 LumiList lumis;
1008
1009 typedef std::set<LuminosityBlockID> RunLumiSet;
1010 RunLumiSet runLumiSet;
1011
1012 typedef std::list<RunItem> RunList;
1013 RunList runs;
1014
1015 typedef std::set<RunNumber_t> RunSet;
1016 RunSet runSet;
1017
1018 typedef std::set<RunItem, RunItemSortByRunPhid> RunItemSet;
1019 RunItemSet runItemSet;
1020
1021 typedef std::map<RunNumber_t, ProcessHistoryID> PHIDMap;
1022 PHIDMap phidMap;
1023
1024 RunNumber_t prevRun = 0;
1025 LuminosityBlockNumber_t prevLumi = 0;
1026 ProcessHistoryID prevPhid;
1027 bool iFirst = true;
1028
1029 indexIntoFile_.unsortedEventNumbers().clear();
1030 indexIntoFile_.unsortedEventNumbers().reserve(eventTree_.entries());
1031
1032
1033 EventSelectionIDVector eventSelectionIDs;
1034 BranchListIndexes branchListIndexes;
1035 while (eventTree_.next()) {
1036 bool newRun = false;
1037 bool newLumi = false;
1038 auto evtAux = fillThisEventAuxiliary();
1039 fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes);
1040
1041
1042
1043
1044 indexIntoFile_.unsortedEventNumbers().push_back(evtAux.event());
1045
1046 ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(evtAux.processHistoryID());
1047
1048 if (iFirst || prevPhid != reducedPHID || prevRun != evtAux.run()) {
1049 iFirst = false;
1050 newRun = newLumi = true;
1051 } else if (prevLumi != evtAux.luminosityBlock()) {
1052 newLumi = true;
1053 }
1054 prevPhid = reducedPHID;
1055 prevRun = evtAux.run();
1056 prevLumi = evtAux.luminosityBlock();
1057 if (newLumi) {
1058 lumis.emplace_back(
1059 reducedPHID, evtAux.run(), evtAux.luminosityBlock(), eventTree_.entryNumber());
1060 runLumiSet.insert(LuminosityBlockID(evtAux.run(), evtAux.luminosityBlock()));
1061 } else {
1062 LumiItem& currentLumi = lumis.back();
1063 assert(currentLumi.lastEventEntry_ == eventTree_.entryNumber());
1064 ++currentLumi.lastEventEntry_;
1065 }
1066 if (newRun) {
1067
1068 RunItem item(reducedPHID, evtAux.run());
1069 if (runItemSet.insert(item).second) {
1070 runs.push_back(std::move(item));
1071 runSet.insert(evtAux.run());
1072 phidMap.insert(std::make_pair(evtAux.run(), reducedPHID));
1073 }
1074 }
1075 }
1076
1077 eventTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1078 lastEventEntryNumberRead_ = IndexIntoFile::invalidEntry;
1079
1080
1081
1082 typedef std::map<RunNumber_t, IndexIntoFile::EntryNumber_t> RunMap;
1083 RunMap runMap;
1084
1085 typedef std::vector<RunItem> RunVector;
1086 RunVector emptyRuns;
1087
1088 if (runTree_.isValid()) {
1089 while (runTree_.next()) {
1090
1091
1092 std::shared_ptr<RunAuxiliary> runAux = fillRunAuxiliary();
1093 ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(runAux->processHistoryID());
1094
1095 if (runSet.insert(runAux->run()).second) {
1096
1097 emptyRuns.emplace_back(reducedPHID, runAux->run());
1098 }
1099 runMap.insert(std::make_pair(runAux->run(), runTree_.entryNumber()));
1100 phidMap.insert(std::make_pair(runAux->run(), reducedPHID));
1101 }
1102
1103 runTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1104 }
1105
1106
1107 RunItemSortByRun runItemSortByRun;
1108 stable_sort_all(emptyRuns, runItemSortByRun);
1109
1110 RunList::iterator itRuns = runs.begin(), endRuns = runs.end();
1111 for (auto const& emptyRun : emptyRuns) {
1112 for (; itRuns != endRuns; ++itRuns) {
1113 if (runItemSortByRun(emptyRun, *itRuns)) {
1114 break;
1115 }
1116 }
1117 runs.insert(itRuns, emptyRun);
1118 }
1119
1120
1121
1122 typedef std::vector<LumiItem> LumiVector;
1123 LumiVector emptyLumis;
1124
1125 typedef std::map<LuminosityBlockID, IndexIntoFile::EntryNumber_t> RunLumiMap;
1126 RunLumiMap runLumiMap;
1127
1128 if (lumiTree_.isValid()) {
1129 while (lumiTree_.next()) {
1130
1131 std::shared_ptr<LuminosityBlockAuxiliary> lumiAux = fillLumiAuxiliary();
1132 LuminosityBlockID lumiID = LuminosityBlockID(lumiAux->run(), lumiAux->luminosityBlock());
1133 if (runLumiSet.insert(lumiID).second) {
1134
1135
1136
1137
1138 PHIDMap::const_iterator iPhidMap = phidMap.find(lumiAux->run());
1139 assert(iPhidMap != phidMap.end());
1140 emptyLumis.emplace_back(
1141 iPhidMap->second, lumiAux->run(), lumiAux->luminosityBlock(), IndexIntoFile::invalidEntry);
1142 }
1143 runLumiMap.insert(std::make_pair(lumiID, lumiTree_.entryNumber()));
1144 }
1145
1146 lumiTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1147 }
1148
1149
1150 LumiItemSortByRunLumi lumiItemSortByRunLumi;
1151 stable_sort_all(emptyLumis, lumiItemSortByRunLumi);
1152
1153 LumiList::iterator itLumis = lumis.begin(), endLumis = lumis.end();
1154 for (auto const& emptyLumi : emptyLumis) {
1155 for (; itLumis != endLumis; ++itLumis) {
1156 if (lumiItemSortByRunLumi(emptyLumi, *itLumis)) {
1157 break;
1158 }
1159 }
1160 lumis.insert(itLumis, emptyLumi);
1161 }
1162
1163
1164
1165 typedef std::map<RunItem, int, RunItemSortByRunPhid> RunCountMap;
1166 RunCountMap runCountMap;
1167 std::vector<ProcessHistoryID>& phids = indexIntoFile_.setProcessHistoryIDs();
1168 assert(phids.empty());
1169 std::vector<IndexIntoFile::RunOrLumiEntry>& entries = indexIntoFile_.setRunOrLumiEntries();
1170 assert(entries.empty());
1171 int rcount = 0;
1172 for (auto& run : runs) {
1173 RunCountMap::const_iterator countMapItem = runCountMap.find(run);
1174 if (countMapItem == runCountMap.end()) {
1175 countMapItem = runCountMap.insert(std::make_pair(run, rcount)).first;
1176 assert(countMapItem != runCountMap.end());
1177 ++rcount;
1178 }
1179 std::vector<ProcessHistoryID>::const_iterator phidItem = find_in_all(phids, run.phid_);
1180 if (phidItem == phids.end()) {
1181 phids.push_back(run.phid_);
1182 phidItem = phids.end() - 1;
1183 }
1184 entries.emplace_back(countMapItem->second,
1185 IndexIntoFile::invalidEntry,
1186 runMap[run.run_],
1187 phidItem - phids.begin(),
1188 run.run_,
1189 0U,
1190 IndexIntoFile::invalidEntry,
1191 IndexIntoFile::invalidEntry);
1192 }
1193
1194
1195 typedef std::map<LumiItem, int, LumiItemSortByRunLumiPhid> LumiCountMap;
1196 LumiCountMap lumiCountMap;
1197 int lcount = 0;
1198 for (auto& lumi : lumis) {
1199 RunCountMap::const_iterator runCountMapItem = runCountMap.find(RunItem(lumi.phid_, lumi.run_));
1200 assert(runCountMapItem != runCountMap.end());
1201 LumiCountMap::const_iterator countMapItem = lumiCountMap.find(lumi);
1202 if (countMapItem == lumiCountMap.end()) {
1203 countMapItem = lumiCountMap.insert(std::make_pair(lumi, lcount)).first;
1204 assert(countMapItem != lumiCountMap.end());
1205 ++lcount;
1206 }
1207 std::vector<ProcessHistoryID>::const_iterator phidItem = find_in_all(phids, lumi.phid_);
1208 assert(phidItem != phids.end());
1209 entries.emplace_back(runCountMapItem->second,
1210 countMapItem->second,
1211 runLumiMap[LuminosityBlockID(lumi.run_, lumi.lumi_)],
1212 phidItem - phids.begin(),
1213 lumi.run_,
1214 lumi.lumi_,
1215 lumi.firstEventEntry_,
1216 lumi.lastEventEntry_);
1217 }
1218 stable_sort_all(entries);
1219 }
1220
1221 void RootFile::validateFile(InputType inputType, bool usingGoToEvent) {
1222 if (!fid_.isValid()) {
1223 fid_ = FileID(createGlobalIdentifier());
1224 }
1225 if (!eventTree_.isValid()) {
1226 throw Exception(errors::EventCorruption) << "'Events' tree is corrupted or not present\n"
1227 << "in the input file.\n";
1228 }
1229 if (enforceGUIDInFileName_) {
1230 auto guidFromName = stemFromPath(file_);
1231 if (guidFromName != fid_.fid()) {
1232 throw edm::Exception(edm::errors::FileNameInconsistentWithGUID)
1233 << "GUID " << guidFromName << " extracted from file name " << file_
1234 << " is inconsistent with the GUID read from the file " << fid_.fid();
1235 }
1236 }
1237
1238 if (fileFormatVersion().hasIndexIntoFile()) {
1239 if (runTree().entries() > 0) {
1240 assert(!indexIntoFile_.empty());
1241 }
1242 if (!fileFormatVersion().useReducedProcessHistoryID()) {
1243 if (daqProvenanceHelper_) {
1244 std::vector<ProcessHistoryID>& phidVec = indexIntoFile_.setProcessHistoryIDs();
1245 for (auto& phid : phidVec) {
1246 phid = daqProvenanceHelper_->mapProcessHistoryID(phid);
1247 }
1248 }
1249 indexIntoFile_.reduceProcessHistoryIDs(*processHistoryRegistry_);
1250 }
1251 } else {
1252 assert(indexIntoFile_.empty());
1253 fillIndexIntoFile();
1254 }
1255
1256 indexIntoFile_.fixIndexes(orderedProcessHistoryIDs_);
1257 indexIntoFile_.setNumberOfEvents(eventTree_.entries());
1258 indexIntoFile_.setEventFinder(
1259 std::shared_ptr<IndexIntoFile::EventFinder>(std::make_shared<RootFileEventFinder>(eventTree_)));
1260
1261
1262 bool needEventNumbers = false;
1263 bool needIndexesForDuplicateChecker =
1264 duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled();
1265 if (inputType != InputType::Primary || needIndexesForDuplicateChecker || usingGoToEvent) {
1266 needEventNumbers = true;
1267 }
1268 bool needEventEntries = false;
1269 if (inputType != InputType::Primary || !noEventSort_) {
1270
1271 needEventEntries = true;
1272 }
1273 indexIntoFile_.fillEventNumbersOrEntries(needEventNumbers, needEventEntries);
1274 }
1275
1276 void RootFile::reportOpened(std::string const& inputType) {
1277
1278 std::string const label = "source";
1279 std::string moduleName = "PoolSource";
1280 filePtr_->inputFileOpened(logicalFile_, inputType, moduleName, label, fid_.fid(), eventTree_.branchNames());
1281 }
1282
1283 void RootFile::close() {
1284
1285 eventHistoryTree_ = nullptr;
1286 for (auto& treePointer : treePointers_) {
1287 treePointer->close();
1288 treePointer = nullptr;
1289 }
1290 filePtr_->Close();
1291 filePtr_ = nullptr;
1292 }
1293
1294 EventAuxiliary const& RootFile::fillThisEventAuxiliary() {
1295 if (lastEventEntryNumberRead_ == eventTree_.entryNumber()) {
1296
1297 return eventAuxCache_;
1298 }
1299 if (fileFormatVersion().newAuxiliary()) {
1300 EventAuxiliary* pEvAux = &eventAuxCache_;
1301 eventTree_.fillAux<EventAuxiliary>(pEvAux);
1302 } else {
1303
1304 EventAux eventAux;
1305 EventAux* pEvAux = &eventAux;
1306 eventTree_.fillAux<EventAux>(pEvAux);
1307 conversion(eventAux, eventAuxCache_);
1308 }
1309 lastEventEntryNumberRead_ = eventTree_.entryNumber();
1310 return eventAuxCache_;
1311 }
1312
1313 EventAuxiliary RootFile::fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry) {
1314 eventTree_.setEntryNumber(entry);
1315 return fillThisEventAuxiliary();
1316 }
1317
1318 void RootFile::fillEventToProcessBlockIndexes() {
1319 TBranch* eventToProcessBlockIndexesBranch = get_underlying_safe(eventToProcessBlockIndexesBranch_);
1320 if (eventToProcessBlockIndexesBranch == nullptr) {
1321 if (processBlockHelper_.get() == nullptr) {
1322 eventToProcessBlockIndexes_.setIndex(0);
1323 } else {
1324 eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset());
1325 }
1326 } else {
1327 if (processBlockHelper_->cacheIndexVectorsPerFile().back() == 1u) {
1328 eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset());
1329 } else {
1330 EventToProcessBlockIndexes* pEventToProcessBlockIndexes = &eventToProcessBlockIndexes_;
1331 eventTree_.fillBranchEntry(eventToProcessBlockIndexesBranch, pEventToProcessBlockIndexes);
1332 unsigned int updatedIndex = eventToProcessBlockIndexes_.index() + processBlockHelper_->outerOffset();
1333 eventToProcessBlockIndexes_.setIndex(updatedIndex);
1334 }
1335 }
1336 }
1337
1338 bool RootFile::fillEventHistory(EventAuxiliary& evtAux,
1339 EventSelectionIDVector& eventSelectionIDs,
1340 BranchListIndexes& branchListIndexes,
1341 bool assertOnFailure) {
1342
1343
1344
1345 if (fileFormatVersion().eventHistoryBranch()) {
1346
1347 EventID id(evtAux.id().run(), 0, evtAux.id().event());
1348 if (eventProcessHistoryIter_->eventID() != id) {
1349 EventProcessHistoryID target(id, ProcessHistoryID());
1350 eventProcessHistoryIter_ = lower_bound_all(eventProcessHistoryIDs_, target);
1351 assert(eventProcessHistoryIter_->eventID() == id);
1352 }
1353 evtAux.setProcessHistoryID(eventProcessHistoryIter_->processHistoryID());
1354 ++eventProcessHistoryIter_;
1355 } else if (fileFormatVersion().eventHistoryTree()) {
1356
1357 History* pHistory = history_.get();
1358 TBranch* eventHistoryBranch = eventHistoryTree_->GetBranch(poolNames::eventHistoryBranchName().c_str());
1359 if (!eventHistoryBranch) {
1360 throw Exception(errors::EventCorruption) << "Failed to find history branch in event history tree.\n";
1361 }
1362 eventHistoryBranch->SetAddress(&pHistory);
1363 roottree::getEntry(eventHistoryTree_, eventTree_.entryNumber());
1364 evtAux.setProcessHistoryID(history_->processHistoryID());
1365 eventSelectionIDs.swap(history_->eventSelectionIDs());
1366 branchListIndexes.swap(history_->branchListIndexes());
1367 } else if (fileFormatVersion().noMetaDataTrees()) {
1368
1369 EventSelectionIDVector* pESV = &eventSelectionIDs;
1370 TBranch* eventSelectionIDBranch = eventTree_.tree()->GetBranch(poolNames::eventSelectionsBranchName().c_str());
1371 assert(eventSelectionIDBranch != nullptr);
1372 eventTree_.fillBranchEntry(eventSelectionIDBranch, pESV);
1373 BranchListIndexes* pBLI = &branchListIndexes;
1374 TBranch* branchListIndexesBranch = eventTree_.tree()->GetBranch(poolNames::branchListIndexesBranchName().c_str());
1375 assert(branchListIndexesBranch != nullptr);
1376 eventTree_.fillBranchEntry(branchListIndexesBranch, pBLI);
1377 }
1378 if (provenanceAdaptor_) {
1379 evtAux.setProcessHistoryID(provenanceAdaptor_->convertID(evtAux.processHistoryID()));
1380 for (auto& esID : eventSelectionIDs) {
1381 esID = provenanceAdaptor_->convertID(esID);
1382 }
1383 }
1384 if (daqProvenanceHelper_) {
1385 evtAux.setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(evtAux.processHistoryID()));
1386 }
1387 if (!fileFormatVersion().splitProductIDs()) {
1388
1389 provenanceAdaptor_->branchListIndexes(branchListIndexes);
1390 }
1391 if (branchIDListHelper_) {
1392 return branchIDListHelper_->fixBranchListIndexes(branchListIndexes, assertOnFailure);
1393 }
1394 return true;
1395 }
1396
1397 std::shared_ptr<LuminosityBlockAuxiliary> RootFile::fillLumiAuxiliary() {
1398 auto lumiAuxiliary = std::make_shared<LuminosityBlockAuxiliary>();
1399 if (fileFormatVersion().newAuxiliary()) {
1400 LuminosityBlockAuxiliary* pLumiAux = lumiAuxiliary.get();
1401 lumiTree_.fillAux<LuminosityBlockAuxiliary>(pLumiAux);
1402 } else {
1403 LuminosityBlockAux lumiAux;
1404 LuminosityBlockAux* pLumiAux = &lumiAux;
1405 lumiTree_.fillAux<LuminosityBlockAux>(pLumiAux);
1406 conversion(lumiAux, *lumiAuxiliary);
1407 }
1408 if (provenanceAdaptor_) {
1409 lumiAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(lumiAuxiliary->processHistoryID()));
1410 }
1411 if (daqProvenanceHelper_) {
1412 lumiAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(lumiAuxiliary->processHistoryID()));
1413 }
1414 if (lumiAuxiliary->luminosityBlock() == 0 && !fileFormatVersion().runsAndLumis()) {
1415 lumiAuxiliary->id() = LuminosityBlockID(RunNumber_t(1), LuminosityBlockNumber_t(1));
1416 }
1417 return lumiAuxiliary;
1418 }
1419
1420 std::shared_ptr<RunAuxiliary> RootFile::fillRunAuxiliary() {
1421 auto runAuxiliary = std::make_shared<RunAuxiliary>();
1422 if (fileFormatVersion().newAuxiliary()) {
1423 RunAuxiliary* pRunAux = runAuxiliary.get();
1424 runTree_.fillAux<RunAuxiliary>(pRunAux);
1425 } else {
1426 RunAux runAux;
1427 RunAux* pRunAux = &runAux;
1428 runTree_.fillAux<RunAux>(pRunAux);
1429 conversion(runAux, *runAuxiliary);
1430 }
1431 if (provenanceAdaptor_) {
1432 runAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(runAuxiliary->processHistoryID()));
1433 }
1434 if (daqProvenanceHelper_) {
1435 runAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(runAuxiliary->processHistoryID()));
1436 }
1437 return runAuxiliary;
1438 }
1439
1440 bool RootFile::skipEvents(int& offset) {
1441 while (offset > 0 && indexIntoFileIter_ != indexIntoFileEnd_) {
1442 int phIndexOfSkippedEvent = IndexIntoFile::invalidIndex;
1443 RunNumber_t runOfSkippedEvent = IndexIntoFile::invalidRun;
1444 LuminosityBlockNumber_t lumiOfSkippedEvent = IndexIntoFile::invalidLumi;
1445 IndexIntoFile::EntryNumber_t skippedEventEntry = IndexIntoFile::invalidEntry;
1446
1447 indexIntoFileIter_.skipEventForward(
1448 phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, skippedEventEntry);
1449
1450
1451 if (skippedEventEntry == IndexIntoFile::invalidEntry)
1452 break;
1453
1454 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
1455 auto const evtAux = fillEventAuxiliary(skippedEventEntry);
1456 if (eventSkipperByID_->skipIt(runOfSkippedEvent, lumiOfSkippedEvent, evtAux.id().event())) {
1457 continue;
1458 }
1459 }
1460 if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) {
1461 auto const evtAux = fillEventAuxiliary(skippedEventEntry);
1462 if (duplicateChecker_->isDuplicateAndCheckActive(
1463 phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, evtAux.id().event(), file_)) {
1464 continue;
1465 }
1466 }
1467 --offset;
1468 }
1469
1470 while (offset < 0) {
1471 if (duplicateChecker_) {
1472 duplicateChecker_->disable();
1473 }
1474
1475 int phIndexOfEvent = IndexIntoFile::invalidIndex;
1476 RunNumber_t runOfEvent = IndexIntoFile::invalidRun;
1477 LuminosityBlockNumber_t lumiOfEvent = IndexIntoFile::invalidLumi;
1478 IndexIntoFile::EntryNumber_t eventEntry = IndexIntoFile::invalidEntry;
1479
1480 indexIntoFileIter_.skipEventBackward(phIndexOfEvent, runOfEvent, lumiOfEvent, eventEntry);
1481
1482 if (eventEntry == IndexIntoFile::invalidEntry)
1483 break;
1484
1485 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
1486 auto const evtAux = fillEventAuxiliary(eventEntry);
1487 if (eventSkipperByID_->skipIt(runOfEvent, lumiOfEvent, evtAux.id().event())) {
1488 continue;
1489 }
1490 }
1491 ++offset;
1492 }
1493 return (indexIntoFileIter_ == indexIntoFileEnd_);
1494 }
1495
1496 bool RootFile::goToEvent(EventID const& eventID) {
1497 indexIntoFile_.fillEventNumbers();
1498
1499 if (duplicateChecker_) {
1500 duplicateChecker_->disable();
1501 }
1502
1503 IndexIntoFile::SortOrder sortOrder = IndexIntoFile::numericalOrder;
1504 if (noEventSort_)
1505 sortOrder = IndexIntoFile::firstAppearanceOrder;
1506 if (noRunLumiSort_) {
1507 sortOrder = IndexIntoFile::entryOrder;
1508 }
1509
1510 IndexIntoFile::IndexIntoFileItr iter =
1511 indexIntoFile_.findPosition(sortOrder, eventID.run(), eventID.luminosityBlock(), eventID.event());
1512
1513 if (iter == indexIntoFile_.end(sortOrder)) {
1514 return false;
1515 }
1516 indexIntoFileIter_ = iter;
1517 return true;
1518 }
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532 bool RootFile::readEvent(EventPrincipal& principal) {
1533 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1534 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent);
1535
1536 auto [found, succeeded] = readCurrentEvent(principal, false);
1537 auto const& evtAux = principal.aux();
1538
1539 runHelper_->checkRunConsistency(evtAux.run(), indexIntoFileIter_.run());
1540 runHelper_->checkLumiConsistency(evtAux.luminosityBlock(), indexIntoFileIter_.lumi());
1541
1542 ++indexIntoFileIter_;
1543 return succeeded;
1544 }
1545
1546
1547 std::tuple<bool, bool> RootFile::readCurrentEvent(EventPrincipal& principal, bool assertOnFailure) {
1548 bool found = true;
1549 bool succeeded = true;
1550 if (!eventTree_.current()) {
1551 found = false;
1552 return {found, succeeded};
1553 }
1554 auto evtAux = fillThisEventAuxiliary();
1555 if (!fileFormatVersion().lumiInEventID()) {
1556
1557 const_cast<EventID&>(evtAux.id()).setLuminosityBlockNumber(evtAux.oldLuminosityBlock());
1558 evtAux.resetObsoleteInfo();
1559 }
1560 fillEventToProcessBlockIndexes();
1561 EventSelectionIDVector eventSelectionIDs;
1562 BranchListIndexes branchListIndexes;
1563 if (!fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes, assertOnFailure)) {
1564 succeeded = false;
1565 }
1566 runHelper_->overrideRunNumber(evtAux.id(), evtAux.isRealData());
1567
1568
1569 eventTree_.insertEntryForIndex(principal.transitionIndex());
1570 auto history = processHistoryRegistry_->getMapped(evtAux.processHistoryID());
1571 principal.fillEventPrincipal(evtAux,
1572 history,
1573 std::move(eventSelectionIDs),
1574 std::move(branchListIndexes),
1575 eventToProcessBlockIndexes_,
1576 *(makeProductProvenanceRetriever(principal.streamID().value())),
1577 eventTree_.resetAndGetRootDelayedReader());
1578
1579
1580
1581 ProcessHistoryID idToCheck = (daqProvenanceHelper_ && fileFormatVersion().useReducedProcessHistoryID()
1582 ? *daqProvenanceHelper_->oldProcessHistoryID()
1583 : evtAux.processHistoryID());
1584 ProcessHistoryID const& reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(idToCheck);
1585 assert(reducedPHID == indexIntoFile_.processHistoryID(indexIntoFileIter_.processHistoryIDIndex()));
1586
1587
1588 filePtr_->eventReadFromFile();
1589 return {found, succeeded};
1590 }
1591
1592 void RootFile::setAtEventEntry(IndexIntoFile::EntryNumber_t entry) { eventTree_.setEntryNumber(entry); }
1593
1594 std::shared_ptr<RunAuxiliary> RootFile::readRunAuxiliary_() {
1595 if (runHelper_->fakeNewRun()) {
1596 auto runAuxiliary = std::make_shared<RunAuxiliary>(*savedRunAuxiliary());
1597 runHelper_->overrideRunNumber(runAuxiliary->id());
1598 return runAuxiliary;
1599 }
1600 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1601 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun);
1602
1603
1604 if (!runTree_.isValid()) {
1605
1606
1607 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisRun();
1608 assert(eventEntry != IndexIntoFile::invalidEntry);
1609 assert(eventTree_.current(eventEntry));
1610 auto const evtAux = fillEventAuxiliary(eventEntry);
1611
1612 RunID run = RunID(indexIntoFileIter_.run());
1613 runHelper_->overrideRunNumber(run);
1614 savedRunAuxiliary_ = std::make_shared<RunAuxiliary>(run.run(), evtAux.time(), Timestamp::invalidTimestamp());
1615 return savedRunAuxiliary();
1616 }
1617
1618 runTree_.setEntryNumber(indexIntoFileIter_.entry());
1619 std::shared_ptr<RunAuxiliary> runAuxiliary = fillRunAuxiliary();
1620 assert(runAuxiliary->run() == indexIntoFileIter_.run());
1621 runHelper_->overrideRunNumber(runAuxiliary->id());
1622 filePtr_->reportInputRunNumber(runAuxiliary->run());
1623
1624 if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1625 runAuxiliary->setEndTime(Timestamp::invalidTimestamp());
1626 }
1627
1628
1629
1630
1631 if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp() ||
1632 !fileFormatVersion().processHistorySameWithinRun()) {
1633 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisRun();
1634
1635 if (eventEntry != IndexIntoFile::invalidEntry) {
1636 assert(eventTree_.current(eventEntry));
1637 auto evtAux = fillEventAuxiliary(eventEntry);
1638
1639
1640 if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1641 runAuxiliary->setBeginTime(evtAux.time());
1642 }
1643
1644
1645
1646
1647
1648 if (!fileFormatVersion().processHistorySameWithinRun()) {
1649 EventSelectionIDVector eventSelectionIDs;
1650 BranchListIndexes branchListIndexes;
1651 fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes);
1652 runAuxiliary->setProcessHistoryID(evtAux.processHistoryID());
1653 }
1654 }
1655 }
1656 savedRunAuxiliary_ = runAuxiliary;
1657 return runAuxiliary;
1658 }
1659
1660 void RootFile::fillProcessBlockHelper_() {
1661 assert(inputType_ == InputType::Primary);
1662 std::vector<unsigned int> nEntries;
1663 nEntries.reserve(processBlockTrees_.size());
1664 for (auto const& processBlockTree : processBlockTrees_) {
1665 nEntries.push_back(processBlockTree->entries());
1666 }
1667 processBlockHelper_->fillFromPrimaryInput(*storedProcessBlockHelper_, nEntries);
1668 storedProcessBlockHelper_ =
1669 std::make_unique<StoredProcessBlockHelper>();
1670 }
1671
1672 bool RootFile::initializeFirstProcessBlockEntry() {
1673 if (processBlockTrees_[currentProcessBlockTree_]->entryNumber() == IndexIntoFile::invalidEntry) {
1674 processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0);
1675 assert(processBlockTrees_[currentProcessBlockTree_]->current());
1676 return true;
1677 }
1678 return false;
1679 }
1680
1681 bool RootFile::endOfProcessBlocksReached() const { return currentProcessBlockTree_ >= processBlockTrees_.size(); }
1682
1683 bool RootFile::nextProcessBlock_(ProcessBlockPrincipal&) {
1684 assert(inputType_ == InputType::Primary);
1685 if (endOfProcessBlocksReached()) {
1686 return false;
1687 }
1688 if (initializeFirstProcessBlockEntry()) {
1689 return true;
1690 }
1691
1692
1693
1694 assert(processBlockTrees_[currentProcessBlockTree_]->current());
1695
1696 if (processBlockTrees_[currentProcessBlockTree_]->nextWithCache()) {
1697 return true;
1698 }
1699
1700 ++currentProcessBlockTree_;
1701 if (endOfProcessBlocksReached()) {
1702 return false;
1703 }
1704
1705
1706 processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0);
1707 assert(processBlockTrees_[currentProcessBlockTree_]->current());
1708 return true;
1709 }
1710
1711 void RootFile::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) {
1712 assert(inputType_ == InputType::Primary);
1713 RootTree* rootTree = processBlockTrees_[currentProcessBlockTree_].get();
1714 rootTree->insertEntryForIndex(0);
1715 assert(!rootTree->processName().empty());
1716 processBlockPrincipal.fillProcessBlockPrincipal(rootTree->processName(), rootTree->resetAndGetRootDelayedReader());
1717 }
1718
1719 bool RootFile::readRun_(RunPrincipal& runPrincipal) {
1720 bool shouldProcessRun = indexIntoFileIter_.shouldProcessRun();
1721
1722 MergeableRunProductMetadata* mergeableRunProductMetadata = nullptr;
1723 if (shouldProcessRun) {
1724 if (inputType_ == InputType::Primary) {
1725 mergeableRunProductMetadata = runPrincipal.mergeableRunProductMetadata();
1726 RootTree::EntryNumber const& entryNumber = runTree_.entryNumber();
1727 assert(entryNumber >= 0);
1728 mergeableRunProductMetadata->readRun(
1729 entryNumber, *storedMergeableRunProductMetadata_, IndexIntoFileItrHolder(indexIntoFileIter_));
1730 }
1731 }
1732
1733 if (!runHelper_->fakeNewRun()) {
1734 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1735 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun);
1736 ++indexIntoFileIter_;
1737 }
1738
1739 if (!runTree_.isValid()) {
1740 return shouldProcessRun;
1741 }
1742
1743 if (shouldProcessRun) {
1744
1745 runTree_.insertEntryForIndex(0);
1746 runPrincipal.fillRunPrincipal(*processHistoryRegistry_, runTree_.resetAndGetRootDelayedReader());
1747
1748 runPrincipal.readAllFromSourceAndMergeImmediately(mergeableRunProductMetadata);
1749 runPrincipal.setShouldWriteRun(RunPrincipal::kYes);
1750 } else {
1751 runPrincipal.fillRunPrincipal(*processHistoryRegistry_, nullptr);
1752 if (runPrincipal.shouldWriteRun() != RunPrincipal::kYes) {
1753 runPrincipal.setShouldWriteRun(RunPrincipal::kNo);
1754 }
1755 }
1756 return shouldProcessRun;
1757 }
1758
1759 std::shared_ptr<LuminosityBlockAuxiliary> RootFile::readLuminosityBlockAuxiliary_() {
1760 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1761 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi);
1762
1763 if (!lumiTree_.isValid()) {
1764 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisLumi();
1765 assert(eventEntry != IndexIntoFile::invalidEntry);
1766 assert(eventTree_.current(eventEntry));
1767 auto const evtAux = fillEventAuxiliary(eventEntry);
1768
1769 LuminosityBlockID lumi = LuminosityBlockID(indexIntoFileIter_.run(), indexIntoFileIter_.lumi());
1770 runHelper_->overrideRunNumber(lumi);
1771 return std::make_shared<LuminosityBlockAuxiliary>(
1772 lumi.run(), lumi.luminosityBlock(), evtAux.time(), Timestamp::invalidTimestamp());
1773 }
1774
1775 lumiTree_.setEntryNumber(indexIntoFileIter_.entry());
1776 std::shared_ptr<LuminosityBlockAuxiliary> lumiAuxiliary = fillLumiAuxiliary();
1777 assert(lumiAuxiliary->run() == indexIntoFileIter_.run());
1778 assert(lumiAuxiliary->luminosityBlock() == indexIntoFileIter_.lumi());
1779 runHelper_->overrideRunNumber(lumiAuxiliary->id());
1780 filePtr_->reportInputLumiSection(lumiAuxiliary->run(), lumiAuxiliary->luminosityBlock());
1781 if (lumiAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1782 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisLumi();
1783 if (eventEntry != IndexIntoFile::invalidEntry) {
1784 assert(eventTree_.current(eventEntry));
1785 auto const evtAux = fillEventAuxiliary(eventEntry);
1786
1787 lumiAuxiliary->setBeginTime(evtAux.time());
1788 }
1789 lumiAuxiliary->setEndTime(Timestamp::invalidTimestamp());
1790 }
1791 if (!fileFormatVersion().processHistorySameWithinRun() && savedRunAuxiliary_) {
1792 lumiAuxiliary->setProcessHistoryID(savedRunAuxiliary_->processHistoryID());
1793 }
1794 return lumiAuxiliary;
1795 }
1796
1797 bool RootFile::readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) {
1798 bool shouldProcessLumi = indexIntoFileIter_.shouldProcessLumi();
1799 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1800 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi);
1801
1802 if (!lumiTree_.isValid()) {
1803 ++indexIntoFileIter_;
1804 return shouldProcessLumi;
1805 }
1806
1807 if (shouldProcessLumi) {
1808 lumiTree_.setEntryNumber(indexIntoFileIter_.entry());
1809
1810 lumiTree_.insertEntryForIndex(0);
1811 auto history = processHistoryRegistry_->getMapped(lumiPrincipal.aux().processHistoryID());
1812 lumiPrincipal.fillLuminosityBlockPrincipal(history, lumiTree_.resetAndGetRootDelayedReader());
1813
1814 lumiPrincipal.readAllFromSourceAndMergeImmediately();
1815 lumiPrincipal.setShouldWriteLumi(LuminosityBlockPrincipal::kYes);
1816 } else {
1817 auto history = processHistoryRegistry_->getMapped(lumiPrincipal.aux().processHistoryID());
1818 lumiPrincipal.fillLuminosityBlockPrincipal(history, nullptr);
1819 if (lumiPrincipal.shouldWriteLumi() != LuminosityBlockPrincipal::kYes) {
1820 lumiPrincipal.setShouldWriteLumi(LuminosityBlockPrincipal::kNo);
1821 }
1822 }
1823 ++indexIntoFileIter_;
1824 return shouldProcessLumi;
1825 }
1826
1827 bool RootFile::setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
1828 indexIntoFileIter_ = indexIntoFile_.findEventPosition(run, lumi, event);
1829 if (indexIntoFileIter_ == indexIntoFileEnd_)
1830 return false;
1831 eventTree_.setEntryNumber(indexIntoFileIter_.entry());
1832 return true;
1833 }
1834
1835 bool RootFile::setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) {
1836 indexIntoFileIter_ = indexIntoFile_.findLumiPosition(run, lumi);
1837 if (indexIntoFileIter_ == indexIntoFileEnd_)
1838 return false;
1839 lumiTree_.setEntryNumber(indexIntoFileIter_.entry());
1840 return true;
1841 }
1842
1843 bool RootFile::setEntryAtRun(RunNumber_t run) {
1844 indexIntoFileIter_ = indexIntoFile_.findRunPosition(run);
1845 if (indexIntoFileIter_ == indexIntoFileEnd_)
1846 return false;
1847 runTree_.setEntryNumber(indexIntoFileIter_.entry());
1848 return true;
1849 }
1850
1851 bool RootFile::setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) {
1852 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent) {
1853 ++indexIntoFileIter_;
1854 }
1855 indexIntoFileIter_.advanceToEvent();
1856 if (indexIntoFileIter_.getEntryType() != IndexIntoFile::kEvent)
1857 return false;
1858 if (run != indexIntoFileIter_.run())
1859 return false;
1860 if (lumi != indexIntoFileIter_.lumi())
1861 return false;
1862
1863
1864 fillEventAuxiliary(indexIntoFileIter_.entry());
1865 return true;
1866 }
1867
1868 void RootFile::readEventHistoryTree() {
1869
1870 if (fileFormatVersion().eventHistoryTree()) {
1871 history_ = std::make_unique<History>();
1872 eventHistoryTree_ = dynamic_cast<TTree*>(filePtr_->Get(poolNames::eventHistoryTreeName().c_str()));
1873 if (!eventHistoryTree_) {
1874 throw Exception(errors::EventCorruption) << "Failed to find the event history tree.\n";
1875 }
1876 }
1877 }
1878
1879 void RootFile::initializeDuplicateChecker(
1880 std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
1881 std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile) {
1882 if (duplicateChecker_ && !duplicateChecker_->checkDisabled()) {
1883 if (eventTree_.next()) {
1884 auto const evtAux = fillThisEventAuxiliary();
1885
1886 duplicateChecker_->inputFileOpened(evtAux.isRealData(), indexIntoFile_, indexesIntoFiles, currentIndexIntoFile);
1887 }
1888 eventTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1889 }
1890 }
1891
1892 void RootFile::setPresenceInProductRegistry(ProductRegistry& inputProdDescReg,
1893 StoredProcessBlockHelper const& storedProcessBlockHelper) {
1894
1895
1896
1897
1898 ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
1899 for (auto& product : pList) {
1900 BranchDescription& prod = product.second;
1901
1902
1903
1904
1905 prod.initBranchName();
1906 if (prod.branchType() == InProcess) {
1907 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
1908 auto it = std::find(processes.begin(), processes.end(), prod.processName());
1909 if (it != processes.end()) {
1910 auto index = std::distance(processes.begin(), it);
1911 processBlockTrees_[index]->setPresence(prod, newBranchToOldBranch(prod.branchName()));
1912 } else {
1913
1914
1915 prod.setDropped(true);
1916 }
1917 } else {
1918 treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName()));
1919 }
1920 if (prod.present()) {
1921 prod.initFromDictionary();
1922 }
1923 }
1924 }
1925
1926 void RootFile::markBranchToBeDropped(bool dropDescendants,
1927 BranchDescription const& branch,
1928 std::set<BranchID>& branchesToDrop,
1929 std::map<BranchID, BranchID> const& droppedToKeptAlias) const {
1930 if (dropDescendants) {
1931 branchChildren_->appendToDescendants(branch, branchesToDrop, droppedToKeptAlias);
1932 } else {
1933 branchesToDrop.insert(branch.branchID());
1934 }
1935 }
1936
1937 void RootFile::dropOnInputAndReorder(ProductRegistry& reg,
1938 ProductSelectorRules const& rules,
1939 bool dropDescendants,
1940 InputType inputType,
1941 StoredProcessBlockHelper& storedProcessBlockHelper,
1942 ProcessBlockHelper const* processBlockHelper) {
1943 ProductRegistry::ProductList& prodList = reg.productListUpdator();
1944
1945
1946
1947 std::map<BranchID, BranchID> droppedToKeptAlias;
1948 for (auto const& product : prodList) {
1949 BranchDescription const& prod = product.second;
1950 if (prod.branchID() != prod.originalBranchID() && prod.present()) {
1951 droppedToKeptAlias[prod.originalBranchID()] = prod.branchID();
1952 }
1953 }
1954
1955
1956
1957 ProductSelector productSelector;
1958 productSelector.initialize(rules, reg.allBranchDescriptions());
1959
1960
1961
1962 std::set<BranchID> branchesToDrop;
1963 std::vector<BranchDescription const*> associationDescriptions;
1964 for (auto const& product : prodList) {
1965 BranchDescription const& prod = product.second;
1966 if (inputType != InputType::Primary && prod.branchType() == InProcess) {
1967 markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1968 } else if (prod.unwrappedType() == typeid(ThinnedAssociation) && prod.present()) {
1969
1970 if (inputType != InputType::SecondarySource) {
1971 associationDescriptions.push_back(&prod);
1972 } else {
1973 markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1974 }
1975 } else if (!productSelector.selected(prod)) {
1976 markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1977 }
1978 }
1979
1980 if (inputType != InputType::SecondarySource) {
1981
1982
1983
1984
1985
1986
1987
1988 std::set<BranchID> keptProductsInEvent;
1989 for (auto const& product : prodList) {
1990 BranchDescription const& prod = product.second;
1991 if (branchesToDrop.find(prod.branchID()) == branchesToDrop.end() && prod.present() &&
1992 prod.branchType() == InEvent) {
1993 keptProductsInEvent.insert(prod.branchID());
1994 }
1995 }
1996
1997
1998 std::map<BranchID, bool> keepAssociation;
1999 fileThinnedAssociationsHelper_->selectAssociationProducts(
2000 associationDescriptions, keptProductsInEvent, keepAssociation);
2001
2002 for (auto association : associationDescriptions) {
2003 if (!keepAssociation[association->branchID()]) {
2004 markBranchToBeDropped(dropDescendants, *association, branchesToDrop, droppedToKeptAlias);
2005 }
2006 }
2007
2008
2009 auto temp = std::make_unique<ThinnedAssociationsHelper>();
2010 for (auto const& associationBranches : fileThinnedAssociationsHelper_->data()) {
2011 auto iter = keepAssociation.find(associationBranches.association());
2012 if (iter != keepAssociation.end() && iter->second) {
2013 temp->addAssociation(associationBranches);
2014 }
2015 }
2016
2017 fileThinnedAssociationsHelper_ = std::unique_ptr<ThinnedAssociationsHelper>(temp.release());
2018 }
2019
2020
2021 std::set<std::string> processesWithKeptProcessBlockProducts;
2022 std::set<BranchID>::const_iterator branchesToDropEnd = branchesToDrop.end();
2023 for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
2024 BranchDescription const& prod = it->second;
2025 bool drop = branchesToDrop.find(prod.branchID()) != branchesToDropEnd;
2026 if (drop) {
2027 if (!prod.dropped()) {
2028 if (productSelector.selected(prod) && prod.unwrappedType() != typeid(ThinnedAssociation) &&
2029 prod.branchType() != InProcess) {
2030 LogWarning("RootFile") << "Branch '" << prod.branchName() << "' is being dropped from the input\n"
2031 << "of file '" << file_ << "' because it is dependent on a branch\n"
2032 << "that was explicitly dropped.\n";
2033 }
2034 if (prod.branchType() == InProcess) {
2035 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
2036 auto it = std::find(processes.begin(), processes.end(), prod.processName());
2037 assert(it != processes.end());
2038 auto index = std::distance(processes.begin(), it);
2039 processBlockTrees_[index]->dropBranch(newBranchToOldBranch(prod.branchName()));
2040 } else {
2041 treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName()));
2042 }
2043 hasNewlyDroppedBranch_[prod.branchType()] = true;
2044 }
2045 ProductRegistry::ProductList::iterator icopy = it;
2046 ++it;
2047 prodList.erase(icopy);
2048 } else {
2049 if (prod.branchType() == InProcess && prod.present()) {
2050 processesWithKeptProcessBlockProducts.insert(prod.processName());
2051 }
2052 ++it;
2053 }
2054 }
2055
2056 dropProcessesAndReorder(storedProcessBlockHelper, processesWithKeptProcessBlockProducts, processBlockHelper);
2057
2058
2059 if (inputType == InputType::SecondaryFile) {
2060 TString tString;
2061 for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
2062 BranchDescription const& prod = it->second;
2063 if (prod.present() and prod.branchType() != InEvent and prod.branchType() != InProcess) {
2064 TClass* cp = prod.wrappedType().getClass();
2065 void* p = cp->New();
2066 int offset = cp->GetBaseClassOffset(edProductClass_);
2067 std::unique_ptr<WrapperBase> edp = getWrapperBasePtr(p, offset);
2068 if (edp->isMergeable()) {
2069 treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName()));
2070 ProductRegistry::ProductList::iterator icopy = it;
2071 ++it;
2072 prodList.erase(icopy);
2073 } else {
2074 ++it;
2075 }
2076 } else
2077 ++it;
2078 }
2079 }
2080 }
2081
2082 void RootFile::dropProcessesAndReorder(StoredProcessBlockHelper& storedProcessBlockHelper,
2083 std::set<std::string> const& processesWithKeptProcessBlockProducts,
2084 ProcessBlockHelper const* processBlockHelper) {
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097 if (processBlockTrees_.empty()) {
2098 return;
2099 }
2100
2101 std::vector<unsigned int> nEntries;
2102 nEntries.reserve(processBlockTrees_.size());
2103 for (auto const& processBlockTree : processBlockTrees_) {
2104 nEntries.push_back(processBlockTree->entries());
2105 }
2106
2107 bool firstInputFile = !processBlockHelper->initializedFromInput();
2108 bool isModified = false;
2109 std::vector<unsigned int> finalIndexToStoredIndex;
2110
2111 if (firstInputFile) {
2112 isModified = processBlockHelper->firstFileDropProcessesAndReorderStored(
2113 storedProcessBlockHelper, processesWithKeptProcessBlockProducts, nEntries, finalIndexToStoredIndex);
2114 } else {
2115 isModified =
2116 processBlockHelper->dropProcessesAndReorderStored(storedProcessBlockHelper,
2117 processesWithKeptProcessBlockProducts,
2118 nEntries,
2119 finalIndexToStoredIndex,
2120 processBlockHelper->processesWithProcessBlockProducts());
2121 }
2122
2123
2124
2125
2126 if (isModified) {
2127 std::vector<edm::propagate_const<std::unique_ptr<RootTree>>> newProcessBlockTrees;
2128 unsigned int nFinalProducts = storedProcessBlockHelper.processesWithProcessBlockProducts().size();
2129 for (unsigned int j = 0; j < nFinalProducts; ++j) {
2130 unsigned int iStored = finalIndexToStoredIndex[j];
2131 newProcessBlockTrees.push_back(std::move(processBlockTrees_[iStored]));
2132 }
2133 processBlockTrees_.swap(newProcessBlockTrees);
2134 }
2135 }
2136
2137 void RootFile::setSignals(
2138 signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* preEventReadSource,
2139 signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* postEventReadSource) {
2140 eventTree_.setSignals(preEventReadSource, postEventReadSource);
2141 }
2142
2143 void RootFile::makeProcessBlockRootTrees(std::shared_ptr<InputFile> filePtr,
2144 int treeMaxVirtualSize,
2145 bool enablePrefetching,
2146 InputType inputType,
2147 StoredProcessBlockHelper const& storedProcessBlockHelper) {
2148
2149
2150
2151
2152
2153
2154 for (auto const& process : storedProcessBlockHelper.processesWithProcessBlockProducts()) {
2155 processBlockTrees_.emplace_back(std::make_unique<RootTree>(filePtr,
2156 InProcess,
2157 process,
2158 1,
2159 treeMaxVirtualSize,
2160 roottree::defaultNonEventCacheSize,
2161 roottree::defaultNonEventLearningEntries,
2162 enablePrefetching,
2163 inputType));
2164 }
2165 }
2166
2167 std::unique_ptr<MakeProvenanceReader> RootFile::makeProvenanceReaderMaker(InputType inputType) {
2168 if (fileFormatVersion_.storedProductProvenanceUsed()) {
2169 readParentageTree(inputType);
2170 return std::make_unique<MakeReducedProvenanceReader>(parentageIDLookup_);
2171 } else if (fileFormatVersion_.splitProductIDs()) {
2172 readParentageTree(inputType);
2173 return std::make_unique<MakeFullProvenanceReader>();
2174 } else if (fileFormatVersion_.perEventProductIDs()) {
2175 auto entryDescriptionMap = std::make_unique<EntryDescriptionMap>();
2176 readEntryDescriptionTree(*entryDescriptionMap, inputType);
2177 return std::make_unique<MakeOldProvenanceReader>(std::move(entryDescriptionMap));
2178 } else {
2179 return std::make_unique<MakeDummyProvenanceReader>();
2180 }
2181 }
2182
2183 std::shared_ptr<ProductProvenanceRetriever> RootFile::makeProductProvenanceRetriever(unsigned int iStreamID) {
2184 if (eventProductProvenanceRetrievers_.size() <= iStreamID) {
2185 eventProductProvenanceRetrievers_.resize(iStreamID + 1);
2186 }
2187 if (!eventProductProvenanceRetrievers_[iStreamID]) {
2188
2189 eventProductProvenanceRetrievers_[iStreamID] = std::make_shared<ProductProvenanceRetriever>(
2190 provenanceReaderMaker_->makeReader(eventTree_, daqProvenanceHelper_.get()));
2191 }
2192 eventProductProvenanceRetrievers_[iStreamID]->reset();
2193 return eventProductProvenanceRetriever(iStreamID);
2194 }
2195
2196 class ReducedProvenanceReader : public ProvenanceReaderBase {
2197 public:
2198 ReducedProvenanceReader(RootTree* iRootTree,
2199 std::vector<ParentageID> const& iParentageIDLookup,
2200 DaqProvenanceHelper const* daqProvenanceHelper);
2201
2202 std::set<ProductProvenance> readProvenance(unsigned int) const override;
2203
2204 private:
2205 void readProvenanceAsync(WaitingTaskHolder task,
2206 ModuleCallingContext const* moduleCallingContext,
2207 unsigned int transitionIndex,
2208 std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2209
2210 edm::propagate_const<RootTree*> rootTree_;
2211 edm::propagate_const<TBranch*> provBranch_;
2212 StoredProductProvenanceVector provVector_;
2213 StoredProductProvenanceVector const* pProvVector_;
2214 std::vector<ParentageID> const& parentageIDLookup_;
2215 DaqProvenanceHelper const* daqProvenanceHelper_;
2216 std::shared_ptr<std::recursive_mutex> mutex_;
2217 SharedResourcesAcquirer acquirer_;
2218 };
2219
2220 ReducedProvenanceReader::ReducedProvenanceReader(RootTree* iRootTree,
2221 std::vector<ParentageID> const& iParentageIDLookup,
2222 DaqProvenanceHelper const* daqProvenanceHelper)
2223 : ProvenanceReaderBase(),
2224 rootTree_(iRootTree),
2225 pProvVector_(&provVector_),
2226 parentageIDLookup_(iParentageIDLookup),
2227 daqProvenanceHelper_(daqProvenanceHelper),
2228 mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2229 acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {
2230 provBranch_ =
2231 rootTree_->tree()->GetBranch(BranchTypeToProductProvenanceBranchName(rootTree_->branchType()).c_str());
2232 }
2233
2234 namespace {
2235 using SignalType = signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)>;
2236 template <typename R>
2237 void readProvenanceAsyncImpl(R const* iThis,
2238 SerialTaskQueueChain& chain,
2239 WaitingTaskHolder task,
2240 unsigned int transitionIndex,
2241 std::atomic<const std::set<ProductProvenance>*>& writeTo,
2242 ModuleCallingContext const* iContext,
2243 SignalType const* pre,
2244 SignalType const* post) {
2245 if (nullptr == writeTo.load()) {
2246
2247 WaitingTaskHolder taskHolder{task};
2248 auto pWriteTo = &writeTo;
2249
2250 auto serviceToken = ServiceRegistry::instance().presentToken();
2251
2252 chain.push(
2253 *taskHolder.group(),
2254 [holder = std::move(taskHolder),
2255 pWriteTo,
2256 iThis,
2257 transitionIndex,
2258 iContext,
2259 pre,
2260 post,
2261 serviceToken]() mutable {
2262 if (nullptr == pWriteTo->load()) {
2263 ServiceRegistry::Operate operate(serviceToken);
2264 std::unique_ptr<const std::set<ProductProvenance>> prov;
2265 try {
2266 if (pre) {
2267 pre->emit(*(iContext->getStreamContext()), *iContext);
2268 }
2269 prov = std::make_unique<const std::set<ProductProvenance>>(iThis->readProvenance(transitionIndex));
2270 if (post) {
2271 post->emit(*(iContext->getStreamContext()), *iContext);
2272 }
2273
2274 } catch (...) {
2275 if (post) {
2276 post->emit(*(iContext->getStreamContext()), *iContext);
2277 }
2278
2279 holder.doneWaiting(std::current_exception());
2280 return;
2281 }
2282 const std::set<ProductProvenance>* expected = nullptr;
2283
2284 if (pWriteTo->compare_exchange_strong(expected, prov.get())) {
2285 prov.release();
2286 }
2287 }
2288 holder.doneWaiting(std::exception_ptr());
2289 });
2290 }
2291 }
2292 }
2293
2294 void ReducedProvenanceReader::readProvenanceAsync(WaitingTaskHolder task,
2295 ModuleCallingContext const* moduleCallingContext,
2296 unsigned int transitionIndex,
2297 std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2298 readProvenanceAsyncImpl(this,
2299 acquirer_.serialQueueChain(),
2300 task,
2301 transitionIndex,
2302 writeTo,
2303 moduleCallingContext,
2304 rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2305 rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2306 }
2307
2308 std::set<ProductProvenance> ReducedProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2309 {
2310 std::lock_guard<std::recursive_mutex> guard(*mutex_);
2311 ReducedProvenanceReader* me = const_cast<ReducedProvenanceReader*>(this);
2312 me->rootTree_->fillBranchEntry(
2313 me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_);
2314 }
2315 std::set<ProductProvenance> retValue;
2316 if (daqProvenanceHelper_) {
2317 for (auto const& prov : provVector_) {
2318 BranchID bid(prov.branchID_);
2319 retValue.emplace(daqProvenanceHelper_->mapBranchID(BranchID(prov.branchID_)),
2320 daqProvenanceHelper_->mapParentageID(parentageIDLookup_[prov.parentageIDIndex_]));
2321 }
2322 } else {
2323 for (auto const& prov : provVector_) {
2324 if (prov.parentageIDIndex_ >= parentageIDLookup_.size()) {
2325 throw edm::Exception(errors::LogicError)
2326 << "ReducedProvenanceReader::ReadProvenance\n"
2327 << "The parentage ID index value " << prov.parentageIDIndex_
2328 << " is out of bounds. The maximum value is " << parentageIDLookup_.size() - 1 << ".\n"
2329 << "This should never happen.\n"
2330 << "Please report this to the framework developers.";
2331 }
2332 retValue.emplace(BranchID(prov.branchID_), parentageIDLookup_[prov.parentageIDIndex_]);
2333 }
2334 }
2335 return retValue;
2336 }
2337
2338 class FullProvenanceReader : public ProvenanceReaderBase {
2339 public:
2340 explicit FullProvenanceReader(RootTree* rootTree, DaqProvenanceHelper const* daqProvenanceHelper);
2341 ~FullProvenanceReader() override {}
2342 std::set<ProductProvenance> readProvenance(unsigned int transitionIndex) const override;
2343
2344 private:
2345 void readProvenanceAsync(WaitingTaskHolder task,
2346 ModuleCallingContext const* moduleCallingContext,
2347 unsigned int transitionIndex,
2348 std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2349
2350 RootTree* rootTree_;
2351 ProductProvenanceVector infoVector_;
2352
2353 CMS_SA_ALLOW mutable ProductProvenanceVector* pInfoVector_;
2354 DaqProvenanceHelper const* daqProvenanceHelper_;
2355 std::shared_ptr<std::recursive_mutex> mutex_;
2356 SharedResourcesAcquirer acquirer_;
2357 };
2358
2359 FullProvenanceReader::FullProvenanceReader(RootTree* rootTree, DaqProvenanceHelper const* daqProvenanceHelper)
2360 : ProvenanceReaderBase(),
2361 rootTree_(rootTree),
2362 infoVector_(),
2363 pInfoVector_(&infoVector_),
2364 daqProvenanceHelper_(daqProvenanceHelper),
2365 mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2366 acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {}
2367
2368 void FullProvenanceReader::readProvenanceAsync(WaitingTaskHolder task,
2369 ModuleCallingContext const* moduleCallingContext,
2370 unsigned int transitionIndex,
2371 std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2372 readProvenanceAsyncImpl(this,
2373 acquirer_.serialQueueChain(),
2374 task,
2375 transitionIndex,
2376 writeTo,
2377 moduleCallingContext,
2378 rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2379 rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2380 }
2381
2382 std::set<ProductProvenance> FullProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2383 {
2384 std::lock_guard<std::recursive_mutex> guard(*mutex_);
2385 rootTree_->fillBranchEntryMeta(
2386 rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex), pInfoVector_);
2387 }
2388 std::set<ProductProvenance> retValue;
2389 if (daqProvenanceHelper_) {
2390 for (auto const& info : infoVector_) {
2391 retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()),
2392 daqProvenanceHelper_->mapParentageID(info.parentageID()));
2393 }
2394 } else {
2395 for (auto const& info : infoVector_) {
2396 retValue.emplace(info);
2397 }
2398 }
2399 return retValue;
2400 }
2401
2402 class OldProvenanceReader : public ProvenanceReaderBase {
2403 public:
2404 explicit OldProvenanceReader(RootTree* rootTree,
2405 EntryDescriptionMap const& theMap,
2406 DaqProvenanceHelper const* daqProvenanceHelper);
2407 ~OldProvenanceReader() override {}
2408 std::set<ProductProvenance> readProvenance(unsigned int transitionIndex) const override;
2409
2410 private:
2411 void readProvenanceAsync(WaitingTaskHolder task,
2412 ModuleCallingContext const* moduleCallingContext,
2413 unsigned int transitionIndex,
2414 std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2415
2416 edm::propagate_const<RootTree*> rootTree_;
2417 std::vector<EventEntryInfo> infoVector_;
2418
2419 CMS_SA_ALLOW mutable std::vector<EventEntryInfo>* pInfoVector_;
2420 EntryDescriptionMap const& entryDescriptionMap_;
2421 DaqProvenanceHelper const* daqProvenanceHelper_;
2422 std::shared_ptr<std::recursive_mutex> mutex_;
2423 SharedResourcesAcquirer acquirer_;
2424 };
2425
2426 OldProvenanceReader::OldProvenanceReader(RootTree* rootTree,
2427 EntryDescriptionMap const& theMap,
2428 DaqProvenanceHelper const* daqProvenanceHelper)
2429 : ProvenanceReaderBase(),
2430 rootTree_(rootTree),
2431 infoVector_(),
2432 pInfoVector_(&infoVector_),
2433 entryDescriptionMap_(theMap),
2434 daqProvenanceHelper_(daqProvenanceHelper),
2435 mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2436 acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {}
2437
2438 void OldProvenanceReader::readProvenanceAsync(WaitingTaskHolder task,
2439 ModuleCallingContext const* moduleCallingContext,
2440 unsigned int transitionIndex,
2441 std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2442 readProvenanceAsyncImpl(this,
2443 acquirer_.serialQueueChain(),
2444 task,
2445 transitionIndex,
2446 writeTo,
2447 moduleCallingContext,
2448 rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2449 rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2450 }
2451
2452 std::set<ProductProvenance> OldProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2453 {
2454 std::lock_guard<std::recursive_mutex> guard(*mutex_);
2455 rootTree_->branchEntryInfoBranch()->SetAddress(&pInfoVector_);
2456 roottree::getEntry(rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex));
2457 }
2458 std::set<ProductProvenance> retValue;
2459 for (auto const& info : infoVector_) {
2460 EntryDescriptionMap::const_iterator iter = entryDescriptionMap_.find(info.entryDescriptionID());
2461 assert(iter != entryDescriptionMap_.end());
2462 Parentage parentage(iter->second.parents());
2463 if (daqProvenanceHelper_) {
2464 retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()),
2465 daqProvenanceHelper_->mapParentageID(parentage.id()));
2466 } else {
2467 retValue.emplace(info.branchID(), parentage.id());
2468 }
2469 }
2470 return retValue;
2471 }
2472
2473 class DummyProvenanceReader : public ProvenanceReaderBase {
2474 public:
2475 DummyProvenanceReader();
2476 ~DummyProvenanceReader() override {}
2477
2478 private:
2479 std::set<ProductProvenance> readProvenance(unsigned int) const override;
2480 void readProvenanceAsync(WaitingTaskHolder task,
2481 ModuleCallingContext const* moduleCallingContext,
2482 unsigned int transitionIndex,
2483 std::atomic<const std::set<ProductProvenance>*>& writeTo) const override;
2484 };
2485
2486 DummyProvenanceReader::DummyProvenanceReader() : ProvenanceReaderBase() {}
2487
2488 std::set<ProductProvenance> DummyProvenanceReader::readProvenance(unsigned int) const {
2489
2490 return std::set<ProductProvenance>{};
2491 }
2492 void DummyProvenanceReader::readProvenanceAsync(WaitingTaskHolder task,
2493 ModuleCallingContext const* moduleCallingContext,
2494 unsigned int transitionIndex,
2495 std::atomic<const std::set<ProductProvenance>*>& writeTo) const {
2496 if (nullptr == writeTo.load()) {
2497 auto emptyProv = std::make_unique<const std::set<ProductProvenance>>();
2498 const std::set<ProductProvenance>* expected = nullptr;
2499 if (writeTo.compare_exchange_strong(expected, emptyProv.get())) {
2500 emptyProv.release();
2501 }
2502 }
2503 }
2504
2505 std::unique_ptr<ProvenanceReaderBase> MakeDummyProvenanceReader::makeReader(RootTree&,
2506 DaqProvenanceHelper const*) const {
2507 return std::make_unique<DummyProvenanceReader>();
2508 }
2509
2510 std::unique_ptr<ProvenanceReaderBase> MakeOldProvenanceReader::makeReader(
2511 RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2512 return std::make_unique<OldProvenanceReader>(&rootTree, *entryDescriptionMap_, daqProvenanceHelper);
2513 }
2514
2515 std::unique_ptr<ProvenanceReaderBase> MakeFullProvenanceReader::makeReader(
2516 RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2517 return std::make_unique<FullProvenanceReader>(&rootTree, daqProvenanceHelper);
2518 }
2519
2520 std::unique_ptr<ProvenanceReaderBase> MakeReducedProvenanceReader::makeReader(
2521 RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2522 return std::make_unique<ReducedProvenanceReader>(&rootTree, parentageIDLookup_, daqProvenanceHelper);
2523 }
2524 }