File indexing completed on 2025-06-08 22:25:16
0001
0002
0003
0004 #include "RootFile.h"
0005 #include "DuplicateChecker.h"
0006 #include "InputFile.h"
0007 #include "ProvenanceAdaptor.h"
0008 #include "RunHelper.h"
0009 #include "RootDelayedReaderBase.h"
0010
0011 #include "DataFormats/Common/interface/setIsMergeable.h"
0012 #include "DataFormats/Common/interface/ThinnedAssociation.h"
0013 #include "DataFormats/Provenance/interface/ProductDescription.h"
0014 #include "DataFormats/Provenance/interface/BranchIDListHelper.h"
0015 #include "DataFormats/Provenance/interface/BranchType.h"
0016 #include "DataFormats/Provenance/interface/EventEntryInfo.h"
0017 #include "DataFormats/Provenance/interface/ParameterSetBlob.h"
0018 #include "DataFormats/Provenance/interface/ParentageRegistry.h"
0019 #include "DataFormats/Provenance/interface/ProcessHistoryID.h"
0020 #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
0021 #include "DataFormats/Provenance/interface/ProductRegistry.h"
0022 #include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h"
0023 #include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h"
0024 #include "DataFormats/Provenance/interface/StoredProductProvenance.h"
0025 #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h"
0026 #include "DataFormats/Provenance/interface/RunID.h"
0027 #include "FWCore/Common/interface/ProcessBlockHelper.h"
0028 #include "FWCore/Framework/interface/FileBlock.h"
0029 #include "FWCore/Framework/interface/EventPrincipal.h"
0030 #include "FWCore/Framework/interface/ProductSelector.h"
0031 #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h"
0032 #include "FWCore/Framework/interface/MergeableRunProductMetadata.h"
0033 #include "FWCore/Framework/interface/ProcessBlockPrincipal.h"
0034 #include "FWCore/Framework/interface/RunPrincipal.h"
0035 #include "FWCore/Framework/interface/SharedResourcesAcquirer.h"
0036 #include "FWCore/Framework/interface/SharedResourcesRegistry.h"
0037 #include "FWCore/Framework/interface/DelayedReader.h"
0038 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0039 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0040 #include "FWCore/ParameterSet/interface/Registry.h"
0041 #include "FWCore/Sources/interface/EventSkipperByID.h"
0042 #include "FWCore/Sources/interface/DaqProvenanceHelper.h"
0043 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0044 #include "FWCore/Utilities/interface/Algorithms.h"
0045 #include "FWCore/Utilities/interface/do_nothing_deleter.h"
0046 #include "FWCore/Utilities/interface/EDMException.h"
0047 #include "FWCore/Utilities/interface/FriendlyName.h"
0048 #include "FWCore/Utilities/interface/GlobalIdentifier.h"
0049 #include "FWCore/Utilities/interface/ReleaseVersion.h"
0050 #include "FWCore/Utilities/interface/stemFromPath.h"
0051 #include "FWCore/Utilities/interface/thread_safety_macros.h"
0052 #include "FWCore/Version/interface/GetReleaseVersion.h"
0053 #include "IOPool/Common/interface/getWrapperBasePtr.h"
0054
0055 #include "FWCore/Concurrency/interface/WaitingTaskHolder.h"
0056 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
0057
0058
0059 #include "DataFormats/Provenance/interface/EventAux.h"
0060 #include "DataFormats/Provenance/interface/LuminosityBlockAux.h"
0061 #include "DataFormats/Provenance/interface/RunAux.h"
0062 #include "FWCore/ParameterSet/interface/ParameterSetConverter.h"
0063
0064 #include "Rtypes.h"
0065 #include "TClass.h"
0066 #include "TString.h"
0067 #include "TTree.h"
0068 #include "TTreeCache.h"
0069
0070 #include <algorithm>
0071 #include <cassert>
0072 #include <list>
0073 #include <iostream>
0074
0075 namespace edm {
0076
0077
0078 class MakeDummyProvenanceReader : public MakeProvenanceReader {
0079 public:
0080 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0081 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0082 };
0083 class MakeOldProvenanceReader : public MakeProvenanceReader {
0084 public:
0085 MakeOldProvenanceReader(std::unique_ptr<EntryDescriptionMap>&& entryDescriptionMap)
0086 : MakeProvenanceReader(), entryDescriptionMap_(std::move(entryDescriptionMap)) {}
0087 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0088 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0089
0090 private:
0091 edm::propagate_const<std::unique_ptr<EntryDescriptionMap>> entryDescriptionMap_;
0092 };
0093 class MakeFullProvenanceReader : public MakeProvenanceReader {
0094 public:
0095 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0096 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0097 };
0098 class MakeReducedProvenanceReader : public MakeProvenanceReader {
0099 public:
0100 MakeReducedProvenanceReader(std::vector<ParentageID> const& parentageIDLookup)
0101 : parentageIDLookup_(parentageIDLookup) {}
0102 std::unique_ptr<ProvenanceReaderBase> makeReader(RootTree& eventTree,
0103 DaqProvenanceHelper const* daqProvenanceHelper) const override;
0104
0105 private:
0106 std::vector<ParentageID> const& parentageIDLookup_;
0107 };
0108
0109 namespace {
0110 void checkReleaseVersion(std::vector<ProcessHistory> processHistoryVector, std::string const& fileName) {
0111 std::string releaseVersion = getReleaseVersion();
0112 releaseversion::DecomposedReleaseVersion currentRelease(releaseVersion);
0113 for (auto const& ph : processHistoryVector) {
0114 for (auto const& pc : ph) {
0115 if (releaseversion::isEarlierRelease(currentRelease, pc.releaseVersion())) {
0116 throw Exception(errors::FormatIncompatibility)
0117 << "The release you are using, " << getReleaseVersion() << " , predates\n"
0118 << "a release (" << pc.releaseVersion() << ") used in writing the input file, " << fileName << ".\n"
0119 << "Forward compatibility cannot be supported.\n";
0120 }
0121 }
0122 }
0123 }
0124 }
0125
0126
0127 class RootFileEventFinder : public IndexIntoFile::EventFinder {
0128 public:
0129 explicit RootFileEventFinder(RootTree& eventTree) : eventTree_(eventTree) {}
0130 ~RootFileEventFinder() override {}
0131
0132 EventNumber_t getEventNumberOfEntry(roottree::EntryNumber entry) const override {
0133 roottree::EntryNumber saveEntry = eventTree_.entryNumber();
0134 eventTree_.setEntryNumber(entry);
0135 EventAuxiliary eventAux;
0136 EventAuxiliary* pEvAux = &eventAux;
0137 eventTree_.fillAux<EventAuxiliary>(pEvAux);
0138 eventTree_.setEntryNumber(saveEntry);
0139 return eventAux.event();
0140 }
0141
0142 private:
0143 RootTree& eventTree_;
0144 };
0145
0146
0147 RootFile::RootFile(FileOptions&& fileOptions,
0148 InputType inputType,
0149 ProcessingOptions&& processingOptions,
0150 TTreeOptions&& ttreeOptions,
0151 ProductChoices&& productChoices,
0152 CrossFileInfo&& crossFileInfo,
0153 unsigned int nStreams,
0154 ProcessHistoryRegistry& processHistoryRegistry,
0155 std::vector<ProcessHistoryID>& orderedProcessHistoryIDs)
0156 : file_(fileOptions.fileName),
0157 logicalFile_(fileOptions.logicalFileName),
0158 processHistoryRegistry_(&processHistoryRegistry),
0159 filePtr_(fileOptions.filePtr),
0160 eventSkipperByID_(processingOptions.eventSkipperByID),
0161 fileFormatVersion_(),
0162 fid_(),
0163 indexIntoFileSharedPtr_(new IndexIntoFile),
0164 indexIntoFile_(*indexIntoFileSharedPtr_),
0165 indexIntoFileBegin_(indexIntoFile_.begin(
0166 processingOptions.noRunLumiSort ? IndexIntoFile::entryOrder
0167 : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder
0168 : IndexIntoFile::numericalOrder))),
0169 indexIntoFileEnd_(indexIntoFileBegin_),
0170 indexIntoFileIter_(indexIntoFileBegin_),
0171 storedMergeableRunProductMetadata_((inputType == InputType::Primary) ? new StoredMergeableRunProductMetadata
0172 : nullptr),
0173 eventProcessHistoryIDs_(),
0174 eventProcessHistoryIter_(eventProcessHistoryIDs_.begin()),
0175 savedRunAuxiliary_(),
0176 skipAnyEvents_(processingOptions.skipAnyEvents),
0177 noRunLumiSort_(processingOptions.noRunLumiSort),
0178 noEventSort_(processingOptions.noEventSort),
0179 enforceGUIDInFileName_(fileOptions.enforceGUIDInFileName),
0180 whyNotFastClonable_(0),
0181 hasNewlyDroppedBranch_(),
0182 branchListIndexesUnchanged_(false),
0183 eventAuxCache_(),
0184 eventTree_(fileOptions.filePtr, InEvent, nStreams, ttreeOptions, roottree::defaultLearningEntries, inputType),
0185 lumiTree_(fileOptions.filePtr,
0186 InLumi,
0187 1,
0188 ttreeOptions.usingDefaultNonEventOptions(),
0189 roottree::defaultNonEventLearningEntries,
0190 inputType),
0191 runTree_(fileOptions.filePtr,
0192 InRun,
0193 1,
0194 ttreeOptions.usingDefaultNonEventOptions(),
0195 roottree::defaultNonEventLearningEntries,
0196 inputType),
0197 treePointers_(),
0198 lastEventEntryNumberRead_(IndexIntoFile::invalidEntry),
0199 productRegistry_(),
0200 branchIDLists_(),
0201 branchIDListHelper_(crossFileInfo.branchIDListHelper),
0202 processBlockHelper_(crossFileInfo.processBlockHelper),
0203 fileThinnedAssociationsHelper_(),
0204 thinnedAssociationsHelper_(crossFileInfo.thinnedAssociationsHelper),
0205 processingMode_(processingOptions.processingMode),
0206 runHelper_(crossFileInfo.runHelper),
0207 newBranchToOldBranch_(),
0208 eventHistoryTree_(nullptr),
0209 eventToProcessBlockIndexesBranch_(
0210 inputType == InputType::Primary
0211 ? eventTree_.tree()->GetBranch(poolNames::eventToProcessBlockIndexesBranchName().c_str())
0212 : nullptr),
0213 history_(),
0214 productDependencies_(new ProductDependencies),
0215 duplicateChecker_(crossFileInfo.duplicateChecker),
0216 provenanceAdaptor_(),
0217 provenanceReaderMaker_(),
0218 eventProductProvenanceRetrievers_(),
0219 parentageIDLookup_(),
0220 daqProvenanceHelper_(),
0221 edProductClass_(TypeWithDict::byName("edm::WrapperBase").getClass()),
0222 inputType_(inputType) {
0223 hasNewlyDroppedBranch_.fill(false);
0224
0225 treePointers_.resize(3);
0226 treePointers_[InEvent] = &eventTree_;
0227 treePointers_[InLumi] = &lumiTree_;
0228 treePointers_[InRun] = &runTree_;
0229
0230
0231
0232 std::unique_ptr<TTree> metaDataTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::metaDataTreeName().c_str())));
0233 if (nullptr == metaDataTree.get()) {
0234 throw Exception(errors::FileReadError)
0235 << "Could not find tree " << poolNames::metaDataTreeName() << " in the input file.\n";
0236 }
0237
0238
0239
0240
0241 FileFormatVersion* fftPtr = &fileFormatVersion_;
0242 if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) {
0243 TBranch* fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str());
0244 fft->SetAddress(&fftPtr);
0245 roottree::getEntry(fft, 0);
0246 metaDataTree->SetBranchAddress(poolNames::fileFormatVersionBranchName().c_str(), &fftPtr);
0247 }
0248
0249 FileID* fidPtr = &fid_;
0250 if (metaDataTree->FindBranch(poolNames::fileIdentifierBranchName().c_str()) != nullptr) {
0251 metaDataTree->SetBranchAddress(poolNames::fileIdentifierBranchName().c_str(), &fidPtr);
0252 }
0253
0254 IndexIntoFile* iifPtr = &indexIntoFile_;
0255 if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) {
0256 metaDataTree->SetBranchAddress(poolNames::indexIntoFileBranchName().c_str(), &iifPtr);
0257 }
0258
0259 storedProcessBlockHelper_ = std::make_unique<StoredProcessBlockHelper>();
0260 StoredProcessBlockHelper& storedProcessBlockHelper = *storedProcessBlockHelper_;
0261 StoredProcessBlockHelper* pStoredProcessBlockHelper = storedProcessBlockHelper_.get();
0262 if (inputType == InputType::Primary) {
0263 if (metaDataTree->FindBranch(poolNames::processBlockHelperBranchName().c_str()) != nullptr) {
0264 metaDataTree->SetBranchAddress(poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper);
0265 }
0266 }
0267
0268 StoredMergeableRunProductMetadata* smrc = nullptr;
0269 if (inputType == InputType::Primary) {
0270 smrc = &*storedMergeableRunProductMetadata_;
0271 if (metaDataTree->FindBranch(poolNames::mergeableRunProductMetadataBranchName().c_str()) != nullptr) {
0272 metaDataTree->SetBranchAddress(poolNames::mergeableRunProductMetadataBranchName().c_str(), &smrc);
0273 }
0274 }
0275
0276
0277
0278 ProductRegistry inputProdDescReg;
0279 ProductRegistry* ppReg = &inputProdDescReg;
0280 metaDataTree->SetBranchAddress(poolNames::productDescriptionBranchName().c_str(), (&ppReg));
0281
0282 using PsetMap = std::map<ParameterSetID, ParameterSetBlob>;
0283 PsetMap psetMap;
0284 PsetMap* psetMapPtr = &psetMap;
0285 if (metaDataTree->FindBranch(poolNames::parameterSetMapBranchName().c_str()) != nullptr) {
0286
0287 assert(!fileFormatVersion().parameterSetsTree());
0288 metaDataTree->SetBranchAddress(poolNames::parameterSetMapBranchName().c_str(), &psetMapPtr);
0289 } else {
0290 assert(fileFormatVersion().parameterSetsTree());
0291
0292 std::unique_ptr<TTree> psetTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::parameterSetsTreeName().c_str())));
0293 if (nullptr == psetTree.get()) {
0294 throw Exception(errors::FileReadError)
0295 << "Could not find tree " << poolNames::parameterSetsTreeName() << " in the input file.\n";
0296 }
0297
0298 using IdToBlobs = std::pair<ParameterSetID, ParameterSetBlob>;
0299 IdToBlobs idToBlob;
0300 IdToBlobs* pIdToBlob = &idToBlob;
0301 psetTree->SetBranchAddress(poolNames::idToParameterSetBlobsBranchName().c_str(), &pIdToBlob);
0302
0303 std::unique_ptr<TTreeCache> psetTreeCache =
0304 roottree::trainCache(psetTree.get(), *filePtr_, roottree::defaultNonEventCacheSize, "*");
0305 psetTreeCache->SetEnablePrefetching(false);
0306 auto guard = filePtr_->setCacheReadTemporarily(psetTreeCache.get(), psetTree.get());
0307 for (Long64_t i = 0; i != psetTree->GetEntries(); ++i) {
0308 psetTree->GetEntry(i);
0309 psetMap.insert(idToBlob);
0310 }
0311 }
0312
0313
0314 ProcessHistoryRegistry::collection_type pHistMap;
0315 ProcessHistoryRegistry::collection_type* pHistMapPtr = &pHistMap;
0316 if (metaDataTree->FindBranch(poolNames::processHistoryMapBranchName().c_str()) != nullptr) {
0317 metaDataTree->SetBranchAddress(poolNames::processHistoryMapBranchName().c_str(), &pHistMapPtr);
0318 }
0319
0320 ProcessHistoryRegistry::vector_type pHistVector;
0321 ProcessHistoryRegistry::vector_type* pHistVectorPtr = &pHistVector;
0322 if (metaDataTree->FindBranch(poolNames::processHistoryBranchName().c_str()) != nullptr) {
0323 metaDataTree->SetBranchAddress(poolNames::processHistoryBranchName().c_str(), &pHistVectorPtr);
0324 }
0325
0326
0327 ProcessConfigurationVector processConfigurations;
0328 ProcessConfigurationVector* procConfigVectorPtr = &processConfigurations;
0329 if (metaDataTree->FindBranch(poolNames::processConfigurationBranchName().c_str()) != nullptr) {
0330 metaDataTree->SetBranchAddress(poolNames::processConfigurationBranchName().c_str(), &procConfigVectorPtr);
0331 }
0332
0333 auto branchIDListsAPtr = std::make_unique<BranchIDLists>();
0334 BranchIDLists* branchIDListsPtr = branchIDListsAPtr.get();
0335 if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) != nullptr) {
0336 metaDataTree->SetBranchAddress(poolNames::branchIDListBranchName().c_str(), &branchIDListsPtr);
0337 }
0338
0339 ThinnedAssociationsHelper* thinnedAssociationsHelperPtr;
0340 if (inputType != InputType::SecondarySource) {
0341 fileThinnedAssociationsHelper_ =
0342 std::make_unique<ThinnedAssociationsHelper>();
0343 thinnedAssociationsHelperPtr = fileThinnedAssociationsHelper_.get();
0344 if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) != nullptr) {
0345 metaDataTree->SetBranchAddress(poolNames::thinnedAssociationsHelperBranchName().c_str(),
0346 &thinnedAssociationsHelperPtr);
0347 }
0348 }
0349
0350 ProductDependencies* productDependenciesBuffer = productDependencies_.get();
0351 if (metaDataTree->FindBranch(poolNames::productDependenciesBranchName().c_str()) != nullptr) {
0352 metaDataTree->SetBranchAddress(poolNames::productDependenciesBranchName().c_str(), &productDependenciesBuffer);
0353 }
0354
0355
0356 std::vector<EventProcessHistoryID>* eventHistoryIDsPtr = &eventProcessHistoryIDs_;
0357 if (metaDataTree->FindBranch(poolNames::eventHistoryBranchName().c_str()) != nullptr) {
0358 metaDataTree->SetBranchAddress(poolNames::eventHistoryBranchName().c_str(), &eventHistoryIDsPtr);
0359 }
0360
0361 if (metaDataTree->FindBranch(poolNames::moduleDescriptionMapBranchName().c_str()) != nullptr) {
0362 if (metaDataTree->GetBranch(poolNames::moduleDescriptionMapBranchName().c_str())->GetSplitLevel() != 0) {
0363 metaDataTree->SetBranchStatus((poolNames::moduleDescriptionMapBranchName() + ".*").c_str(), false);
0364 } else {
0365 metaDataTree->SetBranchStatus(poolNames::moduleDescriptionMapBranchName().c_str(), false);
0366 }
0367 }
0368
0369
0370 roottree::getEntry(metaDataTree.get(), 0);
0371
0372 eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin();
0373
0374
0375 readEventHistoryTree();
0376
0377 ParameterSetConverter::ParameterSetIdConverter psetIdConverter;
0378 if (!fileFormatVersion().triggerPathsTracked()) {
0379 ParameterSetConverter converter(psetMap, psetIdConverter, fileFormatVersion().parameterSetsByReference());
0380 } else {
0381
0382 pset::Registry& psetRegistry = *pset::Registry::instance();
0383 try {
0384 for (auto const& psetEntry : psetMap) {
0385 ParameterSet pset(psetEntry.second.pset());
0386 pset.setID(psetEntry.first);
0387
0388 if (inputType != InputType::SecondarySource) {
0389 psetRegistry.insertMapped(pset);
0390 }
0391 }
0392 } catch (edm::Exception const& iExcept) {
0393 if (iExcept.categoryCode() == edm::errors::Configuration) {
0394 edm::Exception exception(edm::errors::FormatIncompatibility);
0395 exception << iExcept.message();
0396 exception.addContext("Creating ParameterSets from file");
0397 throw exception;
0398 } else {
0399 throw;
0400 }
0401 }
0402 }
0403 std::shared_ptr<BranchIDLists> mutableBranchIDLists;
0404 if (!fileFormatVersion().splitProductIDs()) {
0405
0406
0407 provenanceAdaptor_ = std::make_unique<ProvenanceAdaptor>(
0408 inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, true);
0409
0410 mutableBranchIDLists = provenanceAdaptor_->releaseBranchIDLists();
0411 } else {
0412 if (!fileFormatVersion().triggerPathsTracked()) {
0413
0414
0415 provenanceAdaptor_ = std::make_unique<ProvenanceAdaptor>(
0416 inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, false);
0417 }
0418
0419 if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) == nullptr) {
0420 throw Exception(errors::EventCorruption) << "Failed to find branchIDLists branch in metaData tree.\n";
0421 }
0422 mutableBranchIDLists.reset(branchIDListsAPtr.release());
0423 }
0424
0425 if (fileFormatVersion().hasThinnedAssociations()) {
0426 if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) == nullptr) {
0427 throw Exception(errors::EventCorruption)
0428 << "Failed to find thinnedAssociationsHelper branch in metaData tree.\n";
0429 }
0430 }
0431
0432 if (!fileOptions.bypassVersionCheck) {
0433 checkReleaseVersion(pHistVector, file());
0434 }
0435
0436 if (productChoices.labelRawDataLikeMC) {
0437 std::string const rawData("FEDRawDataCollection");
0438 std::string const source("source");
0439 ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
0440 BranchKey finder(rawData, source, "", "");
0441 ProductRegistry::ProductList::iterator it = pList.lower_bound(finder);
0442 if (it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source) {
0443
0444
0445
0446 it->second.init();
0447
0448 daqProvenanceHelper_ = std::make_unique<DaqProvenanceHelper>(it->second.unwrappedTypeID());
0449
0450 ProductDescription const& newBD = daqProvenanceHelper_->productDescription();
0451
0452 daqProvenanceHelper_->saveInfo(it->second, newBD);
0453
0454 newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), it->second.branchName()));
0455
0456 pList.erase(it);
0457
0458 it = pList.lower_bound(finder);
0459 assert(!(it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source));
0460
0461 inputProdDescReg.copyProduct(newBD);
0462
0463 daqProvenanceHelper_->fixMetaData(processConfigurations, pHistVector);
0464 daqProvenanceHelper_->fixMetaData(*mutableBranchIDLists);
0465 daqProvenanceHelper_->fixMetaData(*productDependencies_);
0466 }
0467 branchIDLists_ = std::move(mutableBranchIDLists);
0468 }
0469
0470 for (auto const& history : pHistVector) {
0471 processHistoryRegistry.registerProcessHistory(history);
0472 }
0473
0474 eventTree_.trainCache(BranchTypeToAuxiliaryBranchName(InEvent).c_str());
0475
0476
0477
0478 if (inputType == InputType::Primary) {
0479 branchListIndexesUnchanged_ = branchIDListHelper_->updateFromInput(*branchIDLists_);
0480 }
0481
0482 validateFile(inputType, processingOptions.usingGoToEvent, orderedProcessHistoryIDs);
0483
0484
0485
0486
0487 provenanceReaderMaker_ = std::unique_ptr<MakeProvenanceReader>(makeProvenanceReaderMaker(inputType).release());
0488
0489
0490 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
0491 whyNotFastClonable_ += FileBlock::EventsOrLumisSelectedByID;
0492 }
0493
0494 initializeDuplicateChecker(crossFileInfo.indexesIntoFiles, crossFileInfo.currentIndexIntoFile);
0495 indexIntoFileIter_ = indexIntoFileBegin_ = indexIntoFile_.begin(
0496 processingOptions.noRunLumiSort
0497 ? IndexIntoFile::entryOrder
0498 : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder));
0499 indexIntoFileEnd_ = indexIntoFile_.end(
0500 processingOptions.noRunLumiSort
0501 ? IndexIntoFile::entryOrder
0502 : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder));
0503 runHelper_->setForcedRunOffset(indexIntoFileBegin_ == indexIntoFileEnd_ ? 1 : indexIntoFileBegin_.run());
0504 eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin();
0505
0506 makeProcessBlockRootTrees(fileOptions.filePtr, ttreeOptions, inputType, storedProcessBlockHelper);
0507
0508 setPresenceInProductRegistry(inputProdDescReg, storedProcessBlockHelper);
0509
0510 auto newReg = std::make_unique<ProductRegistry>();
0511
0512
0513 {
0514 ProductRegistry::ProductList const& prodList = inputProdDescReg.productList();
0515 for (auto const& product : prodList) {
0516 ProductDescription const& prod = product.second;
0517 std::string newFriendlyName = friendlyname::friendlyName(prod.className());
0518 if (newFriendlyName == prod.friendlyClassName()) {
0519 newReg->copyProduct(prod);
0520 } else {
0521 if (fileFormatVersion().splitProductIDs()) {
0522 throw Exception(errors::UnimplementedFeature)
0523 << "Cannot change friendly class name algorithm without more development work\n"
0524 << "to update BranchIDLists and ThinnedAssociationsHelper. Contact the framework group.\n";
0525 }
0526 ProductDescription newBD(prod);
0527 newBD.updateFriendlyClassName();
0528 newReg->copyProduct(newBD);
0529 newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), prod.branchName()));
0530 }
0531 }
0532
0533 dropOnInputAndReorder(*newReg,
0534 productChoices.productSelectorRules,
0535 productChoices.dropDescendantsOfDroppedProducts,
0536 inputType,
0537 storedProcessBlockHelper,
0538 crossFileInfo.processBlockHelper);
0539
0540 if (inputType == InputType::SecondaryFile) {
0541 crossFileInfo.thinnedAssociationsHelper->updateFromSecondaryInput(*fileThinnedAssociationsHelper_,
0542 *productChoices.associationsFromSecondary);
0543 } else if (inputType == InputType::Primary) {
0544 crossFileInfo.processBlockHelper->initializeFromPrimaryInput(storedProcessBlockHelper);
0545 crossFileInfo.thinnedAssociationsHelper->updateFromPrimaryInput(*fileThinnedAssociationsHelper_);
0546 }
0547
0548 if (inputType == InputType::Primary) {
0549 for (auto& product : newReg->productListUpdator()) {
0550 setIsMergeable(product.second);
0551 }
0552 }
0553
0554 for (auto& product : newReg->productListUpdator()) {
0555 product.second.setOnDemand(true);
0556 }
0557
0558 for (auto& processBlockTree : processBlockTrees_) {
0559 treePointers_.push_back(processBlockTree.get());
0560 }
0561
0562
0563 newReg->setFrozen(inputType != InputType::Primary);
0564 productRegistry_.reset(newReg.release());
0565 }
0566
0567
0568 ProductRegistry::ProductList const& prodList = productRegistry()->productList();
0569
0570 {
0571 std::vector<size_t> nBranches(treePointers_.size(), 0);
0572 for (auto const& product : prodList) {
0573 if (product.second.branchType() == InProcess) {
0574 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
0575 auto it = std::find(processes.begin(), processes.end(), product.second.processName());
0576 if (it != processes.end()) {
0577 auto index = std::distance(processes.begin(), it);
0578 ++nBranches[numberOfRunLumiEventProductTrees + index];
0579 }
0580 } else {
0581 ++nBranches[product.second.branchType()];
0582 }
0583 }
0584
0585 int i = 0;
0586 for (auto& t : treePointers_) {
0587 t->numberOfBranchesToAdd(nBranches[i]);
0588 ++i;
0589 }
0590 }
0591 for (auto const& product : prodList) {
0592 ProductDescription const& prod = product.second;
0593 if (prod.branchType() == InProcess) {
0594 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
0595 auto it = std::find(processes.begin(), processes.end(), prod.processName());
0596 if (it != processes.end()) {
0597 auto index = std::distance(processes.begin(), it);
0598 treePointers_[numberOfRunLumiEventProductTrees + index]->addBranch(prod,
0599 newBranchToOldBranch(prod.branchName()));
0600 }
0601 } else {
0602 treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName()));
0603 }
0604 }
0605
0606
0607 setIfFastClonable(processingOptions.remainingEvents, processingOptions.remainingLumis);
0608
0609
0610 indexIntoFile_.doneFileInitialization();
0611
0612
0613 eventTree_.resetTraining();
0614
0615
0616 runTree_.trainCache("*");
0617 lumiTree_.trainCache("*");
0618 for (auto& processBlockTree : processBlockTrees_) {
0619 processBlockTree->trainCache("*");
0620 }
0621 }
0622
0623 RootFile::~RootFile() {}
0624
0625 void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) {
0626
0627
0628 std::unique_ptr<TTree> entryDescriptionTree(
0629 dynamic_cast<TTree*>(filePtr_->Get(poolNames::entryDescriptionTreeName().c_str())));
0630 if (nullptr == entryDescriptionTree.get()) {
0631 throw Exception(errors::FileReadError)
0632 << "Could not find tree " << poolNames::entryDescriptionTreeName() << " in the input file.\n";
0633 }
0634
0635 EntryDescriptionID idBuffer;
0636 EntryDescriptionID* pidBuffer = &idBuffer;
0637 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), &pidBuffer);
0638
0639 EventEntryDescription entryDescriptionBuffer;
0640 EventEntryDescription* pEntryDescriptionBuffer = &entryDescriptionBuffer;
0641 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), &pEntryDescriptionBuffer);
0642
0643
0644 ParentageRegistry& registry = *ParentageRegistry::instance();
0645
0646 for (Long64_t i = 0, numEntries = entryDescriptionTree->GetEntries(); i < numEntries; ++i) {
0647 roottree::getEntry(entryDescriptionTree.get(), i);
0648 if (idBuffer != entryDescriptionBuffer.id()) {
0649 throw Exception(errors::EventCorruption) << "Corruption of EntryDescription tree detected.\n";
0650 }
0651 entryDescriptionMap.insert(std::make_pair(entryDescriptionBuffer.id(), entryDescriptionBuffer));
0652 Parentage parents;
0653 parents.setParents(entryDescriptionBuffer.parents());
0654 if (daqProvenanceHelper_) {
0655 ParentageID const oldID = parents.id();
0656 daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate());
0657 ParentageID newID = parents.id();
0658 if (newID != oldID) {
0659 daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID);
0660 }
0661 }
0662
0663 if (inputType != InputType::SecondarySource) {
0664 registry.insertMapped(parents);
0665 }
0666 }
0667 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), nullptr);
0668 entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), nullptr);
0669 }
0670
0671 void RootFile::readParentageTree(InputType inputType) {
0672
0673
0674 std::unique_ptr<TTree> parentageTree(dynamic_cast<TTree*>(filePtr_->Get(poolNames::parentageTreeName().c_str())));
0675 if (nullptr == parentageTree.get()) {
0676 throw Exception(errors::FileReadError)
0677 << "Could not find tree " << poolNames::parentageTreeName() << " in the input file.\n";
0678 }
0679
0680 Parentage parents;
0681 Parentage* pParentageBuffer = &parents;
0682 parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), &pParentageBuffer);
0683
0684 ParentageRegistry& registry = *ParentageRegistry::instance();
0685
0686 parentageIDLookup_.reserve(parentageTree->GetEntries());
0687 for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) {
0688 roottree::getEntry(parentageTree.get(), i);
0689 if (daqProvenanceHelper_) {
0690 ParentageID const oldID = parents.id();
0691 daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate());
0692 ParentageID newID = parents.id();
0693 if (newID != oldID) {
0694 daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID);
0695 }
0696 }
0697
0698 if (inputType != InputType::SecondarySource) {
0699 registry.insertMapped(parents);
0700 }
0701 parentageIDLookup_.push_back(parents.id());
0702 }
0703 parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), nullptr);
0704 }
0705
0706 void RootFile::setIfFastClonable(int remainingEvents, int remainingLumis) {
0707 if (fileFormatVersion().noMetaDataTrees() and !fileFormatVersion().storedProductProvenanceUsed()) {
0708
0709 whyNotFastClonable_ += FileBlock::FileTooOld;
0710 return;
0711 }
0712 if (!fileFormatVersion().splitProductIDs()) {
0713 whyNotFastClonable_ += FileBlock::FileTooOld;
0714 return;
0715 }
0716 if (processingMode_ != InputSource::RunsLumisAndEvents) {
0717 whyNotFastClonable_ += FileBlock::NotProcessingEvents;
0718 return;
0719 }
0720
0721 IndexIntoFile::IndexIntoFileItr it = indexIntoFileBegin_;
0722 while (it != indexIntoFileEnd_ && it.getEntryType() != IndexIntoFile::kEvent) {
0723 ++it;
0724 }
0725 if (it == indexIntoFileEnd_) {
0726 whyNotFastClonable_ += FileBlock::NoEventsInFile;
0727 return;
0728 }
0729
0730
0731 IndexIntoFile::SortOrder sortOrder =
0732 (noRunLumiSort_ ? IndexIntoFile::entryOrder
0733 : (noEventSort_ ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder));
0734 if (!indexIntoFile_.iterationWillBeInEntryOrder(sortOrder)) {
0735 whyNotFastClonable_ += (noEventSort_ ? FileBlock::RunOrLumiNotContiguous : FileBlock::EventsToBeSorted);
0736 }
0737 if (skipAnyEvents_) {
0738 whyNotFastClonable_ += FileBlock::InitialEventsSkipped;
0739 }
0740 if (remainingEvents >= 0 && eventTree_.entries() > remainingEvents) {
0741 whyNotFastClonable_ += FileBlock::MaxEventsTooSmall;
0742 }
0743 if (remainingLumis >= 0 && lumiTree_.entries() > remainingLumis) {
0744 whyNotFastClonable_ += FileBlock::MaxLumisTooSmall;
0745 }
0746 if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) {
0747 whyNotFastClonable_ += FileBlock::DuplicateEventsRemoved;
0748 }
0749 }
0750
0751 std::shared_ptr<FileBlock> RootFile::createFileBlock() {
0752 std::vector<TTree*> processBlockTrees;
0753 std::vector<std::string> processesWithProcessBlockTrees;
0754 processBlockTrees.reserve(processBlockTrees_.size());
0755 processesWithProcessBlockTrees.reserve(processBlockTrees_.size());
0756 for (auto& processBlockTree : processBlockTrees_) {
0757 processBlockTrees.push_back(processBlockTree->tree());
0758 processesWithProcessBlockTrees.push_back(processBlockTree->processName());
0759 }
0760 return std::make_shared<FileBlock>(fileFormatVersion(),
0761 eventTree_.tree(),
0762 eventTree_.metaTree(),
0763 lumiTree_.tree(),
0764 lumiTree_.metaTree(),
0765 runTree_.tree(),
0766 runTree_.metaTree(),
0767 std::move(processBlockTrees),
0768 std::move(processesWithProcessBlockTrees),
0769 whyNotFastClonable(),
0770 hasNewlyDroppedBranch(),
0771 file_,
0772 branchListIndexesUnchanged(),
0773 modifiedIDs(),
0774 productDependencies());
0775 }
0776
0777 void RootFile::updateFileBlock(FileBlock& fileBlock) {
0778 std::vector<TTree*> processBlockTrees;
0779 std::vector<std::string> processesWithProcessBlockTrees;
0780 processBlockTrees.reserve(processBlockTrees_.size());
0781 processesWithProcessBlockTrees.reserve(processBlockTrees_.size());
0782 for (auto& processBlockTree : processBlockTrees_) {
0783 processBlockTrees.push_back(processBlockTree->tree());
0784 processesWithProcessBlockTrees.push_back(processBlockTree->processName());
0785 }
0786 fileBlock.updateTTreePointers(eventTree_.tree(),
0787 eventTree_.metaTree(),
0788 lumiTree_.tree(),
0789 lumiTree_.metaTree(),
0790 runTree_.tree(),
0791 runTree_.metaTree(),
0792 std::move(processBlockTrees),
0793 std::move(processesWithProcessBlockTrees));
0794 }
0795
0796 std::string const& RootFile::newBranchToOldBranch(std::string const& newBranch) const {
0797 std::map<std::string, std::string>::const_iterator it = newBranchToOldBranch_.find(newBranch);
0798 if (it != newBranchToOldBranch_.end()) {
0799 return it->second;
0800 }
0801 return newBranch;
0802 }
0803
0804 IndexIntoFile::IndexIntoFileItr RootFile::indexIntoFileIter() const { return indexIntoFileIter_; }
0805
0806 void RootFile::setPosition(IndexIntoFile::IndexIntoFileItr const& position) {
0807 indexIntoFileIter_.copyPosition(position);
0808 }
0809
0810 void RootFile::initAssociationsFromSecondary(std::vector<BranchID> const& associationsFromSecondary) {
0811 thinnedAssociationsHelper_->initAssociationsFromSecondary(associationsFromSecondary,
0812 *fileThinnedAssociationsHelper_);
0813 }
0814
0815 bool RootFile::skipThisEntry() {
0816 if (indexIntoFileIter_ == indexIntoFileEnd_) {
0817 return false;
0818 }
0819
0820 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
0821
0822 if (eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.lumi(), 0U)) {
0823 return true;
0824 }
0825
0826
0827 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent) {
0828 auto eventAux = fillEventAuxiliary(indexIntoFileIter_.entry());
0829 if (eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.lumi(), eventAux.id().event())) {
0830 return true;
0831 }
0832 }
0833
0834
0835 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun && eventSkipperByID_->skippingLumis()) {
0836
0837 if (indexIntoFileIter_.peekAheadAtLumi() == IndexIntoFile::invalidLumi) {
0838 return true;
0839 }
0840
0841 do {
0842 if (!eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.peekAheadAtLumi(), 0U)) {
0843 return false;
0844 }
0845 } while (indexIntoFileIter_.skipLumiInRun());
0846 return true;
0847 }
0848 }
0849 return false;
0850 }
0851
0852 bool RootFile::isDuplicateEvent() {
0853 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent);
0854 if (duplicateChecker_.get() == nullptr) {
0855 return false;
0856 }
0857 auto const eventAux = fillEventAuxiliary(indexIntoFileIter_.entry());
0858 return duplicateChecker_->isDuplicateAndCheckActive(indexIntoFileIter_.processHistoryIDIndex(),
0859 indexIntoFileIter_.run(),
0860 indexIntoFileIter_.lumi(),
0861 eventAux.id().event(),
0862 file_);
0863 }
0864
0865 bool RootFile::containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const {
0866 return indexIntoFile_.containsItem(run, lumi, event);
0867 }
0868
0869 IndexIntoFile::EntryType RootFile::getNextItemType(RunNumber_t& run,
0870 LuminosityBlockNumber_t& lumi,
0871 EventNumber_t& event) {
0872
0873 while (skipThisEntry()) {
0874 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun) {
0875 indexIntoFileIter_.advanceToNextRun();
0876 } else if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi) {
0877 indexIntoFileIter_.advanceToNextLumiOrRun();
0878 } else {
0879 ++indexIntoFileIter_;
0880 }
0881 }
0882
0883 IndexIntoFile::EntryType entryType = indexIntoFileIter_.getEntryType();
0884 if (entryType == IndexIntoFile::kEnd) {
0885 return IndexIntoFile::kEnd;
0886 }
0887 if (entryType == IndexIntoFile::kRun) {
0888 run = indexIntoFileIter_.run();
0889 runHelper_->checkForNewRun(run, indexIntoFileIter_.peekAheadAtLumi());
0890 return IndexIntoFile::kRun;
0891 } else if (processingMode_ == InputSource::Runs) {
0892 indexIntoFileIter_.advanceToNextRun();
0893 return getNextItemType(run, lumi, event);
0894 }
0895 if (entryType == IndexIntoFile::kLumi) {
0896 run = indexIntoFileIter_.run();
0897 lumi = indexIntoFileIter_.lumi();
0898 return IndexIntoFile::kLumi;
0899 } else if (processingMode_ == InputSource::RunsAndLumis) {
0900 indexIntoFileIter_.advanceToNextLumiOrRun();
0901 return getNextItemType(run, lumi, event);
0902 }
0903 if (isDuplicateEvent()) {
0904 ++indexIntoFileIter_;
0905 return getNextItemType(run, lumi, event);
0906 }
0907 run = indexIntoFileIter_.run();
0908 lumi = indexIntoFileIter_.lumi();
0909 auto eventAux = fillEventAuxiliary(indexIntoFileIter_.entry());
0910 event = eventAux.event();
0911 return IndexIntoFile::kEvent;
0912 }
0913
0914 bool RootFile::wasLastEventJustRead() const {
0915 IndexIntoFile::IndexIntoFileItr itr(indexIntoFileIter_);
0916 itr.advanceToEvent();
0917 return itr.getEntryType() == IndexIntoFile::kEnd;
0918 }
0919
0920 bool RootFile::wasFirstEventJustRead() const {
0921 IndexIntoFile::IndexIntoFileItr itr(indexIntoFileIter_);
0922 int phIndex;
0923 RunNumber_t run;
0924 LuminosityBlockNumber_t lumi;
0925 IndexIntoFile::EntryNumber_t eventEntry;
0926 itr.skipEventBackward(phIndex, run, lumi, eventEntry);
0927 itr.skipEventBackward(phIndex, run, lumi, eventEntry);
0928 return eventEntry == IndexIntoFile::invalidEntry;
0929 }
0930
0931 namespace {
0932 struct RunItem {
0933 RunItem(ProcessHistoryID const& phid, RunNumber_t const& run) : phid_(phid), run_(run) {}
0934 ProcessHistoryID phid_;
0935 RunNumber_t run_;
0936 };
0937 struct RunItemSortByRun {
0938 bool operator()(RunItem const& a, RunItem const& b) const { return a.run_ < b.run_; }
0939 };
0940 struct RunItemSortByRunPhid {
0941 bool operator()(RunItem const& a, RunItem const& b) const {
0942 return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.phid_ < b.phid_);
0943 }
0944 };
0945 struct LumiItem {
0946 LumiItem(ProcessHistoryID const& phid,
0947 RunNumber_t const& run,
0948 LuminosityBlockNumber_t const& lumi,
0949 IndexIntoFile::EntryNumber_t const& entry)
0950 : phid_(phid),
0951 run_(run),
0952 lumi_(lumi),
0953 firstEventEntry_(entry),
0954 lastEventEntry_(entry == IndexIntoFile::invalidEntry ? IndexIntoFile::invalidEntry : entry + 1) {}
0955 ProcessHistoryID phid_;
0956 RunNumber_t run_;
0957 LuminosityBlockNumber_t lumi_;
0958 IndexIntoFile::EntryNumber_t firstEventEntry_;
0959 IndexIntoFile::EntryNumber_t lastEventEntry_;
0960 };
0961 struct LumiItemSortByRunLumi {
0962 bool operator()(LumiItem const& a, LumiItem const& b) const {
0963 return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.lumi_ < b.lumi_);
0964 }
0965 };
0966 struct LumiItemSortByRunLumiPhid {
0967 bool operator()(LumiItem const& a, LumiItem const& b) const {
0968 if (a.run_ < b.run_)
0969 return true;
0970 if (b.run_ < a.run_)
0971 return false;
0972 if (a.lumi_ < b.lumi_)
0973 return true;
0974 if (b.lumi_ < a.lumi_)
0975 return false;
0976 return a.phid_ < b.phid_;
0977 }
0978 };
0979 }
0980
0981 void RootFile::fillIndexIntoFile() {
0982
0983
0984
0985
0986
0987
0988
0989 assert(!fileFormatVersion().hasIndexIntoFile());
0990
0991 typedef std::list<LumiItem> LumiList;
0992 LumiList lumis;
0993
0994 typedef std::set<LuminosityBlockID> RunLumiSet;
0995 RunLumiSet runLumiSet;
0996
0997 typedef std::list<RunItem> RunList;
0998 RunList runs;
0999
1000 typedef std::set<RunNumber_t> RunSet;
1001 RunSet runSet;
1002
1003 typedef std::set<RunItem, RunItemSortByRunPhid> RunItemSet;
1004 RunItemSet runItemSet;
1005
1006 typedef std::map<RunNumber_t, ProcessHistoryID> PHIDMap;
1007 PHIDMap phidMap;
1008
1009 RunNumber_t prevRun = 0;
1010 LuminosityBlockNumber_t prevLumi = 0;
1011 ProcessHistoryID prevPhid;
1012 bool iFirst = true;
1013
1014 indexIntoFile_.unsortedEventNumbers().clear();
1015 indexIntoFile_.unsortedEventNumbers().reserve(eventTree_.entries());
1016
1017
1018 EventSelectionIDVector eventSelectionIDs;
1019 BranchListIndexes branchListIndexes;
1020 while (eventTree_.next()) {
1021 bool newRun = false;
1022 bool newLumi = false;
1023 auto evtAux = fillThisEventAuxiliary();
1024 fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes);
1025
1026
1027
1028
1029 indexIntoFile_.unsortedEventNumbers().push_back(evtAux.event());
1030
1031 ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(evtAux.processHistoryID());
1032
1033 if (iFirst || prevPhid != reducedPHID || prevRun != evtAux.run()) {
1034 iFirst = false;
1035 newRun = newLumi = true;
1036 } else if (prevLumi != evtAux.luminosityBlock()) {
1037 newLumi = true;
1038 }
1039 prevPhid = reducedPHID;
1040 prevRun = evtAux.run();
1041 prevLumi = evtAux.luminosityBlock();
1042 if (newLumi) {
1043 lumis.emplace_back(
1044 reducedPHID, evtAux.run(), evtAux.luminosityBlock(), eventTree_.entryNumber());
1045 runLumiSet.insert(LuminosityBlockID(evtAux.run(), evtAux.luminosityBlock()));
1046 } else {
1047 LumiItem& currentLumi = lumis.back();
1048 assert(currentLumi.lastEventEntry_ == eventTree_.entryNumber());
1049 ++currentLumi.lastEventEntry_;
1050 }
1051 if (newRun) {
1052
1053 RunItem item(reducedPHID, evtAux.run());
1054 if (runItemSet.insert(item).second) {
1055 runs.push_back(std::move(item));
1056 runSet.insert(evtAux.run());
1057 phidMap.insert(std::make_pair(evtAux.run(), reducedPHID));
1058 }
1059 }
1060 }
1061
1062 eventTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1063 lastEventEntryNumberRead_ = IndexIntoFile::invalidEntry;
1064
1065
1066
1067 typedef std::map<RunNumber_t, IndexIntoFile::EntryNumber_t> RunMap;
1068 RunMap runMap;
1069
1070 typedef std::vector<RunItem> RunVector;
1071 RunVector emptyRuns;
1072
1073 if (runTree_.isValid()) {
1074 while (runTree_.next()) {
1075
1076
1077 std::shared_ptr<RunAuxiliary> runAux = fillRunAuxiliary();
1078 ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(runAux->processHistoryID());
1079
1080 if (runSet.insert(runAux->run()).second) {
1081
1082 emptyRuns.emplace_back(reducedPHID, runAux->run());
1083 }
1084 runMap.insert(std::make_pair(runAux->run(), runTree_.entryNumber()));
1085 phidMap.insert(std::make_pair(runAux->run(), reducedPHID));
1086 }
1087
1088 runTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1089 }
1090
1091
1092 RunItemSortByRun runItemSortByRun;
1093 stable_sort_all(emptyRuns, runItemSortByRun);
1094
1095 RunList::iterator itRuns = runs.begin(), endRuns = runs.end();
1096 for (auto const& emptyRun : emptyRuns) {
1097 for (; itRuns != endRuns; ++itRuns) {
1098 if (runItemSortByRun(emptyRun, *itRuns)) {
1099 break;
1100 }
1101 }
1102 runs.insert(itRuns, emptyRun);
1103 }
1104
1105
1106
1107 typedef std::vector<LumiItem> LumiVector;
1108 LumiVector emptyLumis;
1109
1110 typedef std::map<LuminosityBlockID, IndexIntoFile::EntryNumber_t> RunLumiMap;
1111 RunLumiMap runLumiMap;
1112
1113 if (lumiTree_.isValid()) {
1114 while (lumiTree_.next()) {
1115
1116 std::shared_ptr<LuminosityBlockAuxiliary> lumiAux = fillLumiAuxiliary();
1117 LuminosityBlockID lumiID = LuminosityBlockID(lumiAux->run(), lumiAux->luminosityBlock());
1118 if (runLumiSet.insert(lumiID).second) {
1119
1120
1121
1122
1123 PHIDMap::const_iterator iPhidMap = phidMap.find(lumiAux->run());
1124 assert(iPhidMap != phidMap.end());
1125 emptyLumis.emplace_back(
1126 iPhidMap->second, lumiAux->run(), lumiAux->luminosityBlock(), IndexIntoFile::invalidEntry);
1127 }
1128 runLumiMap.insert(std::make_pair(lumiID, lumiTree_.entryNumber()));
1129 }
1130
1131 lumiTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1132 }
1133
1134
1135 LumiItemSortByRunLumi lumiItemSortByRunLumi;
1136 stable_sort_all(emptyLumis, lumiItemSortByRunLumi);
1137
1138 LumiList::iterator itLumis = lumis.begin(), endLumis = lumis.end();
1139 for (auto const& emptyLumi : emptyLumis) {
1140 for (; itLumis != endLumis; ++itLumis) {
1141 if (lumiItemSortByRunLumi(emptyLumi, *itLumis)) {
1142 break;
1143 }
1144 }
1145 lumis.insert(itLumis, emptyLumi);
1146 }
1147
1148
1149
1150 typedef std::map<RunItem, int, RunItemSortByRunPhid> RunCountMap;
1151 RunCountMap runCountMap;
1152 std::vector<ProcessHistoryID>& phids = indexIntoFile_.setProcessHistoryIDs();
1153 assert(phids.empty());
1154 std::vector<IndexIntoFile::RunOrLumiEntry>& entries = indexIntoFile_.setRunOrLumiEntries();
1155 assert(entries.empty());
1156 int rcount = 0;
1157 for (auto& run : runs) {
1158 RunCountMap::const_iterator countMapItem = runCountMap.find(run);
1159 if (countMapItem == runCountMap.end()) {
1160 countMapItem = runCountMap.insert(std::make_pair(run, rcount)).first;
1161 assert(countMapItem != runCountMap.end());
1162 ++rcount;
1163 }
1164 std::vector<ProcessHistoryID>::const_iterator phidItem = find_in_all(phids, run.phid_);
1165 if (phidItem == phids.end()) {
1166 phids.push_back(run.phid_);
1167 phidItem = phids.end() - 1;
1168 }
1169 entries.emplace_back(countMapItem->second,
1170 IndexIntoFile::invalidEntry,
1171 runMap[run.run_],
1172 phidItem - phids.begin(),
1173 run.run_,
1174 0U,
1175 IndexIntoFile::invalidEntry,
1176 IndexIntoFile::invalidEntry);
1177 }
1178
1179
1180 typedef std::map<LumiItem, int, LumiItemSortByRunLumiPhid> LumiCountMap;
1181 LumiCountMap lumiCountMap;
1182 int lcount = 0;
1183 for (auto& lumi : lumis) {
1184 RunCountMap::const_iterator runCountMapItem = runCountMap.find(RunItem(lumi.phid_, lumi.run_));
1185 assert(runCountMapItem != runCountMap.end());
1186 LumiCountMap::const_iterator countMapItem = lumiCountMap.find(lumi);
1187 if (countMapItem == lumiCountMap.end()) {
1188 countMapItem = lumiCountMap.insert(std::make_pair(lumi, lcount)).first;
1189 assert(countMapItem != lumiCountMap.end());
1190 ++lcount;
1191 }
1192 std::vector<ProcessHistoryID>::const_iterator phidItem = find_in_all(phids, lumi.phid_);
1193 assert(phidItem != phids.end());
1194 entries.emplace_back(runCountMapItem->second,
1195 countMapItem->second,
1196 runLumiMap[LuminosityBlockID(lumi.run_, lumi.lumi_)],
1197 phidItem - phids.begin(),
1198 lumi.run_,
1199 lumi.lumi_,
1200 lumi.firstEventEntry_,
1201 lumi.lastEventEntry_);
1202 }
1203 stable_sort_all(entries);
1204 }
1205
1206 void RootFile::validateFile(InputType inputType,
1207 bool usingGoToEvent,
1208 std::vector<ProcessHistoryID>& orderedProcessHistoryIDs) {
1209
1210 if (!fid_.isValid()) {
1211 fid_ = FileID(createGlobalIdentifier());
1212 }
1213 if (!eventTree_.isValid()) {
1214 throw Exception(errors::EventCorruption) << "'Events' tree is corrupted or not present\n"
1215 << "in the input file.\n";
1216 }
1217 if (enforceGUIDInFileName_) {
1218 auto guidFromName = stemFromPath(file_);
1219 if (guidFromName != fid_.fid()) {
1220 throw edm::Exception(edm::errors::FileNameInconsistentWithGUID)
1221 << "GUID " << guidFromName << " extracted from file name " << file_
1222 << " is inconsistent with the GUID read from the file " << fid_.fid();
1223 }
1224 }
1225
1226 if (fileFormatVersion().hasIndexIntoFile()) {
1227 if (runTree().entries() > 0) {
1228 assert(!indexIntoFile_.empty());
1229 }
1230 if (!fileFormatVersion().useReducedProcessHistoryID()) {
1231 if (daqProvenanceHelper_) {
1232 std::vector<ProcessHistoryID>& phidVec = indexIntoFile_.setProcessHistoryIDs();
1233 for (auto& phid : phidVec) {
1234 phid = daqProvenanceHelper_->mapProcessHistoryID(phid);
1235 }
1236 }
1237 indexIntoFile_.reduceProcessHistoryIDs(*processHistoryRegistry_);
1238 }
1239 } else {
1240 assert(indexIntoFile_.empty());
1241 fillIndexIntoFile();
1242 }
1243
1244 indexIntoFile_.fixIndexes(orderedProcessHistoryIDs);
1245 indexIntoFile_.setNumberOfEvents(eventTree_.entries());
1246 indexIntoFile_.setEventFinder(
1247 std::shared_ptr<IndexIntoFile::EventFinder>(std::make_shared<RootFileEventFinder>(eventTree_)));
1248
1249
1250 bool needEventNumbers = false;
1251 bool needIndexesForDuplicateChecker =
1252 duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled();
1253 if (inputType != InputType::Primary || needIndexesForDuplicateChecker || usingGoToEvent) {
1254 needEventNumbers = true;
1255 }
1256 bool needEventEntries = false;
1257 if (inputType != InputType::Primary || !noEventSort_) {
1258
1259 needEventEntries = true;
1260 }
1261 indexIntoFile_.fillEventNumbersOrEntries(needEventNumbers, needEventEntries);
1262 }
1263
1264 void RootFile::reportOpened(std::string const& inputType) {
1265
1266 std::string const label = "source";
1267 std::string moduleName = "PoolSource";
1268 filePtr_->inputFileOpened(logicalFile_, inputType, moduleName, label, fid_.fid(), eventTree_.branchNames());
1269 }
1270
1271 void RootFile::close() {
1272
1273 eventHistoryTree_ = nullptr;
1274 for (auto& treePointer : treePointers_) {
1275 treePointer->close();
1276 treePointer = nullptr;
1277 }
1278 filePtr_->Close();
1279 filePtr_ = nullptr;
1280 }
1281
1282 EventAuxiliary const& RootFile::fillThisEventAuxiliary() {
1283 if (lastEventEntryNumberRead_ == eventTree_.entryNumber()) {
1284
1285 return eventAuxCache_;
1286 }
1287 if (fileFormatVersion().newAuxiliary()) {
1288 EventAuxiliary* pEvAux = &eventAuxCache_;
1289 eventTree_.fillAux<EventAuxiliary>(pEvAux);
1290 } else {
1291
1292 EventAux eventAux;
1293 EventAux* pEvAux = &eventAux;
1294 eventTree_.fillAux<EventAux>(pEvAux);
1295 conversion(eventAux, eventAuxCache_);
1296 }
1297 lastEventEntryNumberRead_ = eventTree_.entryNumber();
1298 return eventAuxCache_;
1299 }
1300
1301 EventAuxiliary RootFile::fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry) {
1302 eventTree_.setEntryNumber(entry);
1303 return fillThisEventAuxiliary();
1304 }
1305
1306 void RootFile::fillEventToProcessBlockIndexes() {
1307 TBranch* eventToProcessBlockIndexesBranch = get_underlying_safe(eventToProcessBlockIndexesBranch_);
1308 if (eventToProcessBlockIndexesBranch == nullptr) {
1309 if (processBlockHelper_.get() == nullptr) {
1310 eventToProcessBlockIndexes_.setIndex(0);
1311 } else {
1312 eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset());
1313 }
1314 } else {
1315 if (processBlockHelper_->cacheIndexVectorsPerFile().back() == 1u) {
1316 eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset());
1317 } else {
1318 EventToProcessBlockIndexes* pEventToProcessBlockIndexes = &eventToProcessBlockIndexes_;
1319 eventTree_.fillBranchEntry(eventToProcessBlockIndexesBranch, pEventToProcessBlockIndexes);
1320 unsigned int updatedIndex = eventToProcessBlockIndexes_.index() + processBlockHelper_->outerOffset();
1321 eventToProcessBlockIndexes_.setIndex(updatedIndex);
1322 }
1323 }
1324 }
1325
1326 bool RootFile::fillEventHistory(EventAuxiliary& evtAux,
1327 EventSelectionIDVector& eventSelectionIDs,
1328 BranchListIndexes& branchListIndexes,
1329 bool assertOnFailure) {
1330
1331
1332
1333 if (fileFormatVersion().eventHistoryBranch()) {
1334
1335 EventID id(evtAux.id().run(), 0, evtAux.id().event());
1336 if (eventProcessHistoryIter_->eventID() != id) {
1337 EventProcessHistoryID target(id, ProcessHistoryID());
1338 eventProcessHistoryIter_ = lower_bound_all(eventProcessHistoryIDs_, target);
1339 assert(eventProcessHistoryIter_->eventID() == id);
1340 }
1341 evtAux.setProcessHistoryID(eventProcessHistoryIter_->processHistoryID());
1342 ++eventProcessHistoryIter_;
1343 } else if (fileFormatVersion().eventHistoryTree()) {
1344
1345 History* pHistory = history_.get();
1346 TBranch* eventHistoryBranch = eventHistoryTree_->GetBranch(poolNames::eventHistoryBranchName().c_str());
1347 if (!eventHistoryBranch) {
1348 throw Exception(errors::EventCorruption) << "Failed to find history branch in event history tree.\n";
1349 }
1350 eventHistoryBranch->SetAddress(&pHistory);
1351 roottree::getEntry(eventHistoryTree_, eventTree_.entryNumber());
1352 eventHistoryBranch->SetAddress(nullptr);
1353 evtAux.setProcessHistoryID(history_->processHistoryID());
1354 eventSelectionIDs.swap(history_->eventSelectionIDs());
1355 branchListIndexes.swap(history_->branchListIndexes());
1356 } else if (fileFormatVersion().noMetaDataTrees()) {
1357
1358 EventSelectionIDVector* pESV = &eventSelectionIDs;
1359 TBranch* eventSelectionIDBranch = eventTree_.tree()->GetBranch(poolNames::eventSelectionsBranchName().c_str());
1360 assert(eventSelectionIDBranch != nullptr);
1361 eventTree_.fillBranchEntry(eventSelectionIDBranch, pESV);
1362 BranchListIndexes* pBLI = &branchListIndexes;
1363 TBranch* branchListIndexesBranch = eventTree_.tree()->GetBranch(poolNames::branchListIndexesBranchName().c_str());
1364 assert(branchListIndexesBranch != nullptr);
1365 eventTree_.fillBranchEntry(branchListIndexesBranch, pBLI);
1366 }
1367 if (provenanceAdaptor_) {
1368 evtAux.setProcessHistoryID(provenanceAdaptor_->convertID(evtAux.processHistoryID()));
1369 for (auto& esID : eventSelectionIDs) {
1370 esID = provenanceAdaptor_->convertID(esID);
1371 }
1372 }
1373 if (daqProvenanceHelper_) {
1374 evtAux.setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(evtAux.processHistoryID()));
1375 }
1376 if (!fileFormatVersion().splitProductIDs()) {
1377
1378 provenanceAdaptor_->branchListIndexes(branchListIndexes);
1379 }
1380 if (branchIDListHelper_) {
1381 return branchIDListHelper_->fixBranchListIndexes(branchListIndexes, assertOnFailure);
1382 }
1383 return true;
1384 }
1385
1386 std::shared_ptr<LuminosityBlockAuxiliary> RootFile::fillLumiAuxiliary() {
1387 auto lumiAuxiliary = std::make_shared<LuminosityBlockAuxiliary>();
1388 if (fileFormatVersion().newAuxiliary()) {
1389 LuminosityBlockAuxiliary* pLumiAux = lumiAuxiliary.get();
1390 lumiTree_.fillAux<LuminosityBlockAuxiliary>(pLumiAux);
1391 } else {
1392 LuminosityBlockAux lumiAux;
1393 LuminosityBlockAux* pLumiAux = &lumiAux;
1394 lumiTree_.fillAux<LuminosityBlockAux>(pLumiAux);
1395 conversion(lumiAux, *lumiAuxiliary);
1396 }
1397 if (provenanceAdaptor_) {
1398 lumiAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(lumiAuxiliary->processHistoryID()));
1399 }
1400 if (daqProvenanceHelper_) {
1401 lumiAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(lumiAuxiliary->processHistoryID()));
1402 }
1403 if (lumiAuxiliary->luminosityBlock() == 0 && !fileFormatVersion().runsAndLumis()) {
1404 lumiAuxiliary->id() = LuminosityBlockID(RunNumber_t(1), LuminosityBlockNumber_t(1));
1405 }
1406 return lumiAuxiliary;
1407 }
1408
1409 std::shared_ptr<RunAuxiliary> RootFile::fillRunAuxiliary() {
1410 auto runAuxiliary = std::make_shared<RunAuxiliary>();
1411 if (fileFormatVersion().newAuxiliary()) {
1412 RunAuxiliary* pRunAux = runAuxiliary.get();
1413 runTree_.fillAux<RunAuxiliary>(pRunAux);
1414 } else {
1415 RunAux runAux;
1416 RunAux* pRunAux = &runAux;
1417 runTree_.fillAux<RunAux>(pRunAux);
1418 conversion(runAux, *runAuxiliary);
1419 }
1420 if (provenanceAdaptor_) {
1421 runAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(runAuxiliary->processHistoryID()));
1422 }
1423 if (daqProvenanceHelper_) {
1424 runAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(runAuxiliary->processHistoryID()));
1425 }
1426 return runAuxiliary;
1427 }
1428
1429 bool RootFile::skipEvents(int& offset) {
1430 while (offset > 0 && indexIntoFileIter_ != indexIntoFileEnd_) {
1431 int phIndexOfSkippedEvent = IndexIntoFile::invalidIndex;
1432 RunNumber_t runOfSkippedEvent = IndexIntoFile::invalidRun;
1433 LuminosityBlockNumber_t lumiOfSkippedEvent = IndexIntoFile::invalidLumi;
1434 IndexIntoFile::EntryNumber_t skippedEventEntry = IndexIntoFile::invalidEntry;
1435
1436 indexIntoFileIter_.skipEventForward(
1437 phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, skippedEventEntry);
1438
1439
1440 if (skippedEventEntry == IndexIntoFile::invalidEntry)
1441 break;
1442
1443 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
1444 auto const evtAux = fillEventAuxiliary(skippedEventEntry);
1445 if (eventSkipperByID_->skipIt(runOfSkippedEvent, lumiOfSkippedEvent, evtAux.id().event())) {
1446 continue;
1447 }
1448 }
1449 if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) {
1450 auto const evtAux = fillEventAuxiliary(skippedEventEntry);
1451 if (duplicateChecker_->isDuplicateAndCheckActive(
1452 phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, evtAux.id().event(), file_)) {
1453 continue;
1454 }
1455 }
1456 --offset;
1457 }
1458
1459 while (offset < 0) {
1460 if (duplicateChecker_) {
1461 duplicateChecker_->disable();
1462 }
1463
1464 int phIndexOfEvent = IndexIntoFile::invalidIndex;
1465 RunNumber_t runOfEvent = IndexIntoFile::invalidRun;
1466 LuminosityBlockNumber_t lumiOfEvent = IndexIntoFile::invalidLumi;
1467 IndexIntoFile::EntryNumber_t eventEntry = IndexIntoFile::invalidEntry;
1468
1469 indexIntoFileIter_.skipEventBackward(phIndexOfEvent, runOfEvent, lumiOfEvent, eventEntry);
1470
1471 if (eventEntry == IndexIntoFile::invalidEntry)
1472 break;
1473
1474 if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) {
1475 auto const evtAux = fillEventAuxiliary(eventEntry);
1476 if (eventSkipperByID_->skipIt(runOfEvent, lumiOfEvent, evtAux.id().event())) {
1477 continue;
1478 }
1479 }
1480 ++offset;
1481 }
1482 return (indexIntoFileIter_ == indexIntoFileEnd_);
1483 }
1484
1485 bool RootFile::goToEvent(EventID const& eventID) {
1486 indexIntoFile_.fillEventNumbers();
1487
1488 if (duplicateChecker_) {
1489 duplicateChecker_->disable();
1490 }
1491
1492 IndexIntoFile::SortOrder sortOrder = IndexIntoFile::numericalOrder;
1493 if (noEventSort_)
1494 sortOrder = IndexIntoFile::firstAppearanceOrder;
1495 if (noRunLumiSort_) {
1496 sortOrder = IndexIntoFile::entryOrder;
1497 }
1498
1499 IndexIntoFile::IndexIntoFileItr iter =
1500 indexIntoFile_.findPosition(sortOrder, eventID.run(), eventID.luminosityBlock(), eventID.event());
1501
1502 if (iter == indexIntoFile_.end(sortOrder)) {
1503 return false;
1504 }
1505 indexIntoFileIter_ = iter;
1506 return true;
1507 }
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521 bool RootFile::readEvent(EventPrincipal& principal, bool readAllProducts) {
1522 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1523 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent);
1524
1525 auto [found, succeeded] = readCurrentEvent(principal, false, readAllProducts);
1526 auto const& evtAux = principal.aux();
1527
1528 runHelper_->checkRunConsistency(evtAux.run(), indexIntoFileIter_.run());
1529 runHelper_->checkLumiConsistency(evtAux.luminosityBlock(), indexIntoFileIter_.lumi());
1530
1531 ++indexIntoFileIter_;
1532 return succeeded;
1533 }
1534
1535
1536 std::tuple<bool, bool> RootFile::readCurrentEvent(EventPrincipal& principal,
1537 bool assertOnFailure,
1538 bool readAllProducts) {
1539 bool found = true;
1540 bool succeeded = true;
1541 if (!eventTree_.current()) {
1542 found = false;
1543 return {found, succeeded};
1544 }
1545 auto evtAux = fillThisEventAuxiliary();
1546 if (!fileFormatVersion().lumiInEventID()) {
1547
1548 const_cast<EventID&>(evtAux.id()).setLuminosityBlockNumber(evtAux.oldLuminosityBlock());
1549 evtAux.resetObsoleteInfo();
1550 }
1551 fillEventToProcessBlockIndexes();
1552 EventSelectionIDVector eventSelectionIDs;
1553 BranchListIndexes branchListIndexes;
1554 if (!fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes, assertOnFailure)) {
1555 succeeded = false;
1556 }
1557 runHelper_->overrideRunNumber(evtAux.id(), evtAux.isRealData());
1558
1559
1560 eventTree_.insertEntryForIndex(principal.transitionIndex());
1561 auto provRetriever = makeProductProvenanceRetriever(principal.streamID().value());
1562 if (readAllProducts) {
1563 eventTree_.rootDelayedReader()->readAllProductsNow(&principal);
1564 provRetriever->unsafe_fillProvenance();
1565 }
1566 auto history = processHistoryRegistry_->getMapped(evtAux.processHistoryID());
1567 principal.fillEventPrincipal(evtAux,
1568 history,
1569 std::move(eventSelectionIDs),
1570 std::move(branchListIndexes),
1571 eventToProcessBlockIndexes_,
1572 *provRetriever,
1573 eventTree_.resetAndGetRootDelayedReader());
1574
1575
1576
1577 ProcessHistoryID idToCheck = (daqProvenanceHelper_ && fileFormatVersion().useReducedProcessHistoryID()
1578 ? *daqProvenanceHelper_->oldProcessHistoryID()
1579 : evtAux.processHistoryID());
1580 ProcessHistoryID const& reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(idToCheck);
1581 assert(reducedPHID == indexIntoFile_.processHistoryID(indexIntoFileIter_.processHistoryIDIndex()));
1582
1583
1584 filePtr_->eventReadFromFile();
1585 return {found, succeeded};
1586 }
1587
1588 void RootFile::setAtEventEntry(IndexIntoFile::EntryNumber_t entry) { eventTree_.setEntryNumber(entry); }
1589
1590 std::shared_ptr<RunAuxiliary> RootFile::readRunAuxiliary_() {
1591 if (runHelper_->fakeNewRun()) {
1592 auto runAuxiliary = std::make_shared<RunAuxiliary>(*savedRunAuxiliary());
1593 runHelper_->overrideRunNumber(runAuxiliary->id());
1594 return runAuxiliary;
1595 }
1596 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1597 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun);
1598
1599
1600 if (!runTree_.isValid()) {
1601
1602
1603 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisRun();
1604 assert(eventEntry != IndexIntoFile::invalidEntry);
1605 assert(eventTree_.current(eventEntry));
1606 auto const evtAux = fillEventAuxiliary(eventEntry);
1607
1608 RunID run = RunID(indexIntoFileIter_.run());
1609 runHelper_->overrideRunNumber(run);
1610 savedRunAuxiliary_ = std::make_shared<RunAuxiliary>(run.run(), evtAux.time(), Timestamp::invalidTimestamp());
1611 return savedRunAuxiliary();
1612 }
1613
1614 runTree_.setEntryNumber(indexIntoFileIter_.entry());
1615 std::shared_ptr<RunAuxiliary> runAuxiliary = fillRunAuxiliary();
1616 assert(runAuxiliary->run() == indexIntoFileIter_.run());
1617 runHelper_->overrideRunNumber(runAuxiliary->id());
1618 filePtr_->reportInputRunNumber(runAuxiliary->run());
1619
1620 if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1621 runAuxiliary->setEndTime(Timestamp::invalidTimestamp());
1622 }
1623
1624
1625
1626
1627 if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp() ||
1628 !fileFormatVersion().processHistorySameWithinRun()) {
1629 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisRun();
1630
1631 if (eventEntry != IndexIntoFile::invalidEntry) {
1632 assert(eventTree_.current(eventEntry));
1633 auto evtAux = fillEventAuxiliary(eventEntry);
1634
1635
1636 if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1637 runAuxiliary->setBeginTime(evtAux.time());
1638 }
1639
1640
1641
1642
1643
1644 if (!fileFormatVersion().processHistorySameWithinRun()) {
1645 EventSelectionIDVector eventSelectionIDs;
1646 BranchListIndexes branchListIndexes;
1647 fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes);
1648 runAuxiliary->setProcessHistoryID(evtAux.processHistoryID());
1649 }
1650 }
1651 }
1652 savedRunAuxiliary_ = runAuxiliary;
1653 return runAuxiliary;
1654 }
1655
1656 void RootFile::fillProcessBlockHelper_() {
1657 assert(inputType_ == InputType::Primary);
1658 std::vector<unsigned int> nEntries;
1659 nEntries.reserve(processBlockTrees_.size());
1660 for (auto const& processBlockTree : processBlockTrees_) {
1661 nEntries.push_back(processBlockTree->entries());
1662 }
1663 processBlockHelper_->fillFromPrimaryInput(*storedProcessBlockHelper_, nEntries);
1664 storedProcessBlockHelper_ =
1665 std::make_unique<StoredProcessBlockHelper>();
1666 }
1667
1668 bool RootFile::initializeFirstProcessBlockEntry() {
1669 if (processBlockTrees_[currentProcessBlockTree_]->entryNumber() == IndexIntoFile::invalidEntry) {
1670 processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0);
1671 assert(processBlockTrees_[currentProcessBlockTree_]->current());
1672 return true;
1673 }
1674 return false;
1675 }
1676
1677 bool RootFile::endOfProcessBlocksReached() const { return currentProcessBlockTree_ >= processBlockTrees_.size(); }
1678
1679 bool RootFile::nextProcessBlock_(ProcessBlockPrincipal&) {
1680 assert(inputType_ == InputType::Primary);
1681 if (endOfProcessBlocksReached()) {
1682 return false;
1683 }
1684 if (initializeFirstProcessBlockEntry()) {
1685 return true;
1686 }
1687
1688
1689
1690 assert(processBlockTrees_[currentProcessBlockTree_]->current());
1691
1692 if (processBlockTrees_[currentProcessBlockTree_]->nextWithCache()) {
1693 return true;
1694 }
1695
1696 ++currentProcessBlockTree_;
1697 if (endOfProcessBlocksReached()) {
1698 return false;
1699 }
1700
1701
1702 processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0);
1703 assert(processBlockTrees_[currentProcessBlockTree_]->current());
1704 return true;
1705 }
1706
1707 void RootFile::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) {
1708 assert(inputType_ == InputType::Primary);
1709 RootTree* rootTree = processBlockTrees_[currentProcessBlockTree_].get();
1710 rootTree->insertEntryForIndex(0);
1711 assert(!rootTree->processName().empty());
1712 processBlockPrincipal.fillProcessBlockPrincipal(rootTree->processName(), rootTree->resetAndGetRootDelayedReader());
1713 }
1714
1715 bool RootFile::readRun_(RunPrincipal& runPrincipal) {
1716 bool shouldProcessRun = indexIntoFileIter_.shouldProcessRun();
1717
1718 MergeableRunProductMetadata* mergeableRunProductMetadata = nullptr;
1719 if (shouldProcessRun) {
1720 if (inputType_ == InputType::Primary) {
1721 mergeableRunProductMetadata = runPrincipal.mergeableRunProductMetadata();
1722 RootTree::EntryNumber const& entryNumber = runTree_.entryNumber();
1723 assert(entryNumber >= 0);
1724 mergeableRunProductMetadata->readRun(
1725 entryNumber, *storedMergeableRunProductMetadata_, IndexIntoFileItrHolder(indexIntoFileIter_));
1726 }
1727 }
1728
1729 if (!runHelper_->fakeNewRun()) {
1730 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1731 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun);
1732 ++indexIntoFileIter_;
1733 }
1734
1735 if (!runTree_.isValid()) {
1736 return shouldProcessRun;
1737 }
1738
1739 if (shouldProcessRun) {
1740
1741 runTree_.insertEntryForIndex(0);
1742 runPrincipal.fillRunPrincipal(*processHistoryRegistry_, runTree_.resetAndGetRootDelayedReader());
1743
1744 runPrincipal.readAllFromSourceAndMergeImmediately(mergeableRunProductMetadata);
1745 runPrincipal.setShouldWriteRun(RunPrincipal::kYes);
1746 } else {
1747 runPrincipal.fillRunPrincipal(*processHistoryRegistry_, nullptr);
1748 if (runPrincipal.shouldWriteRun() != RunPrincipal::kYes) {
1749 runPrincipal.setShouldWriteRun(RunPrincipal::kNo);
1750 }
1751 }
1752 return shouldProcessRun;
1753 }
1754
1755 std::shared_ptr<LuminosityBlockAuxiliary> RootFile::readLuminosityBlockAuxiliary_() {
1756 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1757 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi);
1758
1759 if (!lumiTree_.isValid()) {
1760 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisLumi();
1761 assert(eventEntry != IndexIntoFile::invalidEntry);
1762 assert(eventTree_.current(eventEntry));
1763 auto const evtAux = fillEventAuxiliary(eventEntry);
1764
1765 LuminosityBlockID lumi = LuminosityBlockID(indexIntoFileIter_.run(), indexIntoFileIter_.lumi());
1766 runHelper_->overrideRunNumber(lumi);
1767 return std::make_shared<LuminosityBlockAuxiliary>(
1768 lumi.run(), lumi.luminosityBlock(), evtAux.time(), Timestamp::invalidTimestamp());
1769 }
1770
1771 lumiTree_.setEntryNumber(indexIntoFileIter_.entry());
1772 std::shared_ptr<LuminosityBlockAuxiliary> lumiAuxiliary = fillLumiAuxiliary();
1773 assert(lumiAuxiliary->run() == indexIntoFileIter_.run());
1774 assert(lumiAuxiliary->luminosityBlock() == indexIntoFileIter_.lumi());
1775 runHelper_->overrideRunNumber(lumiAuxiliary->id());
1776 filePtr_->reportInputLumiSection(lumiAuxiliary->run(), lumiAuxiliary->luminosityBlock());
1777 if (lumiAuxiliary->beginTime() == Timestamp::invalidTimestamp()) {
1778 IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisLumi();
1779 if (eventEntry != IndexIntoFile::invalidEntry) {
1780 assert(eventTree_.current(eventEntry));
1781 auto const evtAux = fillEventAuxiliary(eventEntry);
1782
1783 lumiAuxiliary->setBeginTime(evtAux.time());
1784 }
1785 lumiAuxiliary->setEndTime(Timestamp::invalidTimestamp());
1786 }
1787 if (!fileFormatVersion().processHistorySameWithinRun() && savedRunAuxiliary_) {
1788 lumiAuxiliary->setProcessHistoryID(savedRunAuxiliary_->processHistoryID());
1789 }
1790 return lumiAuxiliary;
1791 }
1792
1793 bool RootFile::readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) {
1794 bool shouldProcessLumi = indexIntoFileIter_.shouldProcessLumi();
1795 assert(indexIntoFileIter_ != indexIntoFileEnd_);
1796 assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi);
1797
1798 if (!lumiTree_.isValid()) {
1799 ++indexIntoFileIter_;
1800 return shouldProcessLumi;
1801 }
1802
1803 if (shouldProcessLumi) {
1804 lumiTree_.setEntryNumber(indexIntoFileIter_.entry());
1805
1806 lumiTree_.insertEntryForIndex(0);
1807 auto history = processHistoryRegistry_->getMapped(lumiPrincipal.aux().processHistoryID());
1808 lumiPrincipal.fillLuminosityBlockPrincipal(history, lumiTree_.resetAndGetRootDelayedReader());
1809
1810 lumiPrincipal.readAllFromSourceAndMergeImmediately();
1811 lumiPrincipal.setShouldWriteLumi(LuminosityBlockPrincipal::kYes);
1812 } else {
1813 auto history = processHistoryRegistry_->getMapped(lumiPrincipal.aux().processHistoryID());
1814 lumiPrincipal.fillLuminosityBlockPrincipal(history, nullptr);
1815 if (lumiPrincipal.shouldWriteLumi() != LuminosityBlockPrincipal::kYes) {
1816 lumiPrincipal.setShouldWriteLumi(LuminosityBlockPrincipal::kNo);
1817 }
1818 }
1819 ++indexIntoFileIter_;
1820 return shouldProcessLumi;
1821 }
1822
1823 bool RootFile::setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) {
1824 indexIntoFileIter_ = indexIntoFile_.findEventPosition(run, lumi, event);
1825 if (indexIntoFileIter_ == indexIntoFileEnd_)
1826 return false;
1827 eventTree_.setEntryNumber(indexIntoFileIter_.entry());
1828 return true;
1829 }
1830
1831 bool RootFile::setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) {
1832 indexIntoFileIter_ = indexIntoFile_.findLumiPosition(run, lumi);
1833 if (indexIntoFileIter_ == indexIntoFileEnd_)
1834 return false;
1835 lumiTree_.setEntryNumber(indexIntoFileIter_.entry());
1836 return true;
1837 }
1838
1839 bool RootFile::setEntryAtRun(RunNumber_t run) {
1840 indexIntoFileIter_ = indexIntoFile_.findRunPosition(run);
1841 if (indexIntoFileIter_ == indexIntoFileEnd_)
1842 return false;
1843 runTree_.setEntryNumber(indexIntoFileIter_.entry());
1844 return true;
1845 }
1846
1847 bool RootFile::setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) {
1848 if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent) {
1849 ++indexIntoFileIter_;
1850 }
1851 indexIntoFileIter_.advanceToEvent();
1852 if (indexIntoFileIter_.getEntryType() != IndexIntoFile::kEvent)
1853 return false;
1854 if (run != indexIntoFileIter_.run())
1855 return false;
1856 if (lumi != indexIntoFileIter_.lumi())
1857 return false;
1858
1859
1860 fillEventAuxiliary(indexIntoFileIter_.entry());
1861 return true;
1862 }
1863
1864 void RootFile::readEventHistoryTree() {
1865
1866 if (fileFormatVersion().eventHistoryTree()) {
1867 history_ = std::make_unique<History>();
1868 eventHistoryTree_ = dynamic_cast<TTree*>(filePtr_->Get(poolNames::eventHistoryTreeName().c_str()));
1869 if (!eventHistoryTree_) {
1870 throw Exception(errors::EventCorruption) << "Failed to find the event history tree.\n";
1871 }
1872 }
1873 }
1874
1875 void RootFile::initializeDuplicateChecker(
1876 std::vector<std::shared_ptr<IndexIntoFile>> const& indexesIntoFiles,
1877 std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile) {
1878 if (duplicateChecker_ && !duplicateChecker_->checkDisabled()) {
1879 if (eventTree_.next()) {
1880 auto const evtAux = fillThisEventAuxiliary();
1881
1882 duplicateChecker_->inputFileOpened(evtAux.isRealData(), indexIntoFile_, indexesIntoFiles, currentIndexIntoFile);
1883 }
1884 eventTree_.setEntryNumber(IndexIntoFile::invalidEntry);
1885 }
1886 }
1887
1888 void RootFile::setPresenceInProductRegistry(ProductRegistry& inputProdDescReg,
1889 StoredProcessBlockHelper const& storedProcessBlockHelper) {
1890
1891
1892
1893
1894 ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator();
1895 for (auto& product : pList) {
1896 ProductDescription& prod = product.second;
1897
1898
1899
1900
1901 prod.initBranchName();
1902 if (prod.branchType() == InProcess) {
1903 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
1904 auto it = std::find(processes.begin(), processes.end(), prod.processName());
1905 if (it != processes.end()) {
1906 auto index = std::distance(processes.begin(), it);
1907 processBlockTrees_[index]->setPresence(prod, newBranchToOldBranch(prod.branchName()));
1908 } else {
1909
1910
1911 prod.setDropped(true);
1912 }
1913 } else {
1914 treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName()));
1915 }
1916 if (prod.present()) {
1917 prod.initFromDictionary();
1918 }
1919 }
1920 }
1921
1922 void RootFile::markBranchToBeDropped(bool dropDescendants,
1923 ProductDescription const& branch,
1924 std::set<BranchID>& branchesToDrop,
1925 std::map<BranchID, BranchID> const& droppedToKeptAlias) const {
1926 if (dropDescendants) {
1927 productDependencies_->appendToDescendants(branch, branchesToDrop, droppedToKeptAlias);
1928 } else {
1929 branchesToDrop.insert(branch.branchID());
1930 }
1931 }
1932
1933 void RootFile::dropOnInputAndReorder(ProductRegistry& reg,
1934 ProductSelectorRules const& rules,
1935 bool dropDescendants,
1936 InputType inputType,
1937 StoredProcessBlockHelper& storedProcessBlockHelper,
1938 ProcessBlockHelper const* processBlockHelper) {
1939 ProductRegistry::ProductList& prodList = reg.productListUpdator();
1940
1941
1942
1943 std::map<BranchID, BranchID> droppedToKeptAlias;
1944 for (auto const& product : prodList) {
1945 ProductDescription const& prod = product.second;
1946 if (prod.branchID() != prod.originalBranchID() && prod.present()) {
1947 droppedToKeptAlias[prod.originalBranchID()] = prod.branchID();
1948 }
1949 }
1950
1951
1952
1953 ProductSelector productSelector;
1954 productSelector.initialize(rules, reg.allProductDescriptions());
1955
1956
1957
1958 std::set<BranchID> branchesToDrop;
1959 std::vector<ProductDescription const*> associationDescriptions;
1960 for (auto const& product : prodList) {
1961 ProductDescription const& prod = product.second;
1962 if (inputType != InputType::Primary && prod.branchType() == InProcess) {
1963 markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1964 } else if (prod.unwrappedType() == typeid(ThinnedAssociation) && prod.present()) {
1965
1966 if (inputType != InputType::SecondarySource) {
1967 associationDescriptions.push_back(&prod);
1968 } else {
1969 markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1970 }
1971 } else if (!productSelector.selected(prod)) {
1972 markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias);
1973 }
1974 }
1975
1976 if (inputType != InputType::SecondarySource) {
1977
1978
1979
1980
1981
1982
1983
1984 std::set<BranchID> keptProductsInEvent;
1985 for (auto const& product : prodList) {
1986 ProductDescription const& prod = product.second;
1987 if (branchesToDrop.find(prod.branchID()) == branchesToDrop.end() && prod.present() &&
1988 prod.branchType() == InEvent) {
1989 keptProductsInEvent.insert(prod.branchID());
1990 }
1991 }
1992
1993
1994 std::map<BranchID, bool> keepAssociation;
1995 fileThinnedAssociationsHelper_->selectAssociationProducts(
1996 associationDescriptions, keptProductsInEvent, keepAssociation);
1997
1998 for (auto association : associationDescriptions) {
1999 if (!keepAssociation[association->branchID()]) {
2000 markBranchToBeDropped(dropDescendants, *association, branchesToDrop, droppedToKeptAlias);
2001 }
2002 }
2003
2004
2005 auto temp = std::make_unique<ThinnedAssociationsHelper>();
2006 for (auto const& associationBranches : fileThinnedAssociationsHelper_->data()) {
2007 auto iter = keepAssociation.find(associationBranches.association());
2008 if (iter != keepAssociation.end() && iter->second) {
2009 temp->addAssociation(associationBranches);
2010 }
2011 }
2012
2013 fileThinnedAssociationsHelper_ = std::unique_ptr<ThinnedAssociationsHelper>(temp.release());
2014 }
2015
2016
2017 std::set<std::string> processesWithKeptProcessBlockProducts;
2018 std::set<BranchID>::const_iterator branchesToDropEnd = branchesToDrop.end();
2019 for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
2020 ProductDescription const& prod = it->second;
2021 bool drop = branchesToDrop.find(prod.branchID()) != branchesToDropEnd;
2022 if (drop) {
2023 if (!prod.dropped()) {
2024 if (productSelector.selected(prod) && prod.unwrappedType() != typeid(ThinnedAssociation) &&
2025 prod.branchType() != InProcess) {
2026 LogWarning("RootFile") << "Branch '" << prod.branchName() << "' is being dropped from the input\n"
2027 << "of file '" << file_ << "' because it is dependent on a branch\n"
2028 << "that was explicitly dropped.\n";
2029 }
2030 if (prod.branchType() == InProcess) {
2031 std::vector<std::string> const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts();
2032 auto it = std::find(processes.begin(), processes.end(), prod.processName());
2033 assert(it != processes.end());
2034 auto index = std::distance(processes.begin(), it);
2035 processBlockTrees_[index]->dropBranch(newBranchToOldBranch(prod.branchName()));
2036 } else {
2037 treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName()));
2038 }
2039 hasNewlyDroppedBranch_[prod.branchType()] = true;
2040 }
2041 ProductRegistry::ProductList::iterator icopy = it;
2042 ++it;
2043 prodList.erase(icopy);
2044 } else {
2045 if (prod.branchType() == InProcess && prod.present()) {
2046 processesWithKeptProcessBlockProducts.insert(prod.processName());
2047 }
2048 ++it;
2049 }
2050 }
2051
2052 dropProcessesAndReorder(storedProcessBlockHelper, processesWithKeptProcessBlockProducts, processBlockHelper);
2053
2054
2055 if (inputType == InputType::SecondaryFile) {
2056 TString tString;
2057 for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) {
2058 ProductDescription const& prod = it->second;
2059 if (prod.present() and prod.branchType() != InEvent and prod.branchType() != InProcess) {
2060 TClass* cp = prod.wrappedType().getClass();
2061 void* p = cp->New();
2062 int offset = cp->GetBaseClassOffset(edProductClass_);
2063 std::unique_ptr<WrapperBase> edp = getWrapperBasePtr(p, offset);
2064 if (edp->isMergeable()) {
2065 treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName()));
2066 ProductRegistry::ProductList::iterator icopy = it;
2067 ++it;
2068 prodList.erase(icopy);
2069 } else {
2070 ++it;
2071 }
2072 } else
2073 ++it;
2074 }
2075 }
2076 }
2077
2078 void RootFile::dropProcessesAndReorder(StoredProcessBlockHelper& storedProcessBlockHelper,
2079 std::set<std::string> const& processesWithKeptProcessBlockProducts,
2080 ProcessBlockHelper const* processBlockHelper) {
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093 if (processBlockTrees_.empty()) {
2094 return;
2095 }
2096
2097 std::vector<unsigned int> nEntries;
2098 nEntries.reserve(processBlockTrees_.size());
2099 for (auto const& processBlockTree : processBlockTrees_) {
2100 nEntries.push_back(processBlockTree->entries());
2101 }
2102
2103 bool firstInputFile = !processBlockHelper->initializedFromInput();
2104 bool isModified = false;
2105 std::vector<unsigned int> finalIndexToStoredIndex;
2106
2107 if (firstInputFile) {
2108 isModified = processBlockHelper->firstFileDropProcessesAndReorderStored(
2109 storedProcessBlockHelper, processesWithKeptProcessBlockProducts, nEntries, finalIndexToStoredIndex);
2110 } else {
2111 isModified =
2112 processBlockHelper->dropProcessesAndReorderStored(storedProcessBlockHelper,
2113 processesWithKeptProcessBlockProducts,
2114 nEntries,
2115 finalIndexToStoredIndex,
2116 processBlockHelper->processesWithProcessBlockProducts());
2117 }
2118
2119
2120
2121
2122 if (isModified) {
2123 std::vector<edm::propagate_const<std::unique_ptr<RootTree>>> newProcessBlockTrees;
2124 unsigned int nFinalProducts = storedProcessBlockHelper.processesWithProcessBlockProducts().size();
2125 for (unsigned int j = 0; j < nFinalProducts; ++j) {
2126 unsigned int iStored = finalIndexToStoredIndex[j];
2127 newProcessBlockTrees.push_back(std::move(processBlockTrees_[iStored]));
2128 }
2129 processBlockTrees_.swap(newProcessBlockTrees);
2130 }
2131 }
2132
2133 void RootFile::setSignals(
2134 signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* preEventReadSource,
2135 signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)> const* postEventReadSource) {
2136 eventTree_.setSignals(preEventReadSource, postEventReadSource);
2137 }
2138
2139 void RootFile::makeProcessBlockRootTrees(std::shared_ptr<InputFile> filePtr,
2140 TTreeOptions const& options,
2141 InputType inputType,
2142 StoredProcessBlockHelper const& storedProcessBlockHelper) {
2143
2144
2145
2146
2147
2148
2149 for (auto const& process : storedProcessBlockHelper.processesWithProcessBlockProducts()) {
2150 processBlockTrees_.emplace_back(std::make_unique<RootTree>(filePtr,
2151 InProcess,
2152 process,
2153 1,
2154 options.usingDefaultNonEventOptions(),
2155 roottree::defaultNonEventLearningEntries,
2156 inputType));
2157 }
2158 }
2159
2160 std::unique_ptr<MakeProvenanceReader> RootFile::makeProvenanceReaderMaker(InputType inputType) {
2161 if (fileFormatVersion_.storedProductProvenanceUsed()) {
2162 readParentageTree(inputType);
2163 return std::make_unique<MakeReducedProvenanceReader>(parentageIDLookup_);
2164 } else if (fileFormatVersion_.splitProductIDs()) {
2165 readParentageTree(inputType);
2166 return std::make_unique<MakeFullProvenanceReader>();
2167 } else if (fileFormatVersion_.perEventProductIDs()) {
2168 auto entryDescriptionMap = std::make_unique<EntryDescriptionMap>();
2169 readEntryDescriptionTree(*entryDescriptionMap, inputType);
2170 return std::make_unique<MakeOldProvenanceReader>(std::move(entryDescriptionMap));
2171 } else {
2172 return std::make_unique<MakeDummyProvenanceReader>();
2173 }
2174 }
2175
2176 std::shared_ptr<ProductProvenanceRetriever> RootFile::makeProductProvenanceRetriever(unsigned int iStreamID) {
2177 if (eventProductProvenanceRetrievers_.size() <= iStreamID) {
2178 eventProductProvenanceRetrievers_.resize(iStreamID + 1);
2179 }
2180 if (!eventProductProvenanceRetrievers_[iStreamID]) {
2181
2182 eventProductProvenanceRetrievers_[iStreamID] = std::make_shared<ProductProvenanceRetriever>(
2183 iStreamID, provenanceReaderMaker_->makeReader(eventTree_, daqProvenanceHelper_.get()));
2184 }
2185 eventProductProvenanceRetrievers_[iStreamID]->reset();
2186 return eventProductProvenanceRetriever(iStreamID);
2187 }
2188
2189 class ReducedProvenanceReader : public ProvenanceReaderBase {
2190 public:
2191 ReducedProvenanceReader(RootTree* iRootTree,
2192 std::vector<ParentageID> const& iParentageIDLookup,
2193 DaqProvenanceHelper const* daqProvenanceHelper);
2194
2195 std::set<ProductProvenance> readProvenance(unsigned int) const override;
2196
2197 void unsafe_fillProvenance(unsigned int) const override;
2198
2199 private:
2200 void readProvenanceAsync(WaitingTaskHolder task,
2201 ModuleCallingContext const* moduleCallingContext,
2202 unsigned int transitionIndex,
2203 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept override;
2204
2205 edm::propagate_const<RootTree*> rootTree_;
2206 edm::propagate_const<TBranch*> provBranch_;
2207 StoredProductProvenanceVector provVector_;
2208 StoredProductProvenanceVector const* pProvVector_;
2209 std::vector<ParentageID> const& parentageIDLookup_;
2210 DaqProvenanceHelper const* daqProvenanceHelper_;
2211 std::shared_ptr<std::recursive_mutex> mutex_;
2212 SharedResourcesAcquirer acquirer_;
2213 };
2214
2215 ReducedProvenanceReader::ReducedProvenanceReader(RootTree* iRootTree,
2216 std::vector<ParentageID> const& iParentageIDLookup,
2217 DaqProvenanceHelper const* daqProvenanceHelper)
2218 : ProvenanceReaderBase(),
2219 rootTree_(iRootTree),
2220 pProvVector_(&provVector_),
2221 parentageIDLookup_(iParentageIDLookup),
2222 daqProvenanceHelper_(daqProvenanceHelper),
2223 mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2224 acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {
2225 provBranch_ =
2226 rootTree_->tree()->GetBranch(BranchTypeToProductProvenanceBranchName(rootTree_->branchType()).c_str());
2227 }
2228
2229 namespace {
2230 using SignalType = signalslot::Signal<void(StreamContext const&, ModuleCallingContext const&)>;
2231 template <typename R>
2232 void readProvenanceAsyncImpl(R const* iThis,
2233 SerialTaskQueueChain& chain,
2234 WaitingTaskHolder task,
2235 unsigned int transitionIndex,
2236 std::atomic<const std::set<ProductProvenance>*>& writeTo,
2237 ModuleCallingContext const* iContext,
2238 SignalType const* pre,
2239 SignalType const* post) {
2240 if (nullptr == writeTo.load()) {
2241
2242 WaitingTaskHolder taskHolder{task};
2243 auto pWriteTo = &writeTo;
2244
2245 auto serviceToken = ServiceRegistry::instance().presentToken();
2246
2247 chain.push(
2248 *taskHolder.group(),
2249 [holder = std::move(taskHolder),
2250 pWriteTo,
2251 iThis,
2252 transitionIndex,
2253 iContext,
2254 pre,
2255 post,
2256 serviceToken]() mutable {
2257 if (nullptr == pWriteTo->load()) {
2258 ServiceRegistry::Operate operate(serviceToken);
2259 std::unique_ptr<const std::set<ProductProvenance>> prov;
2260 try {
2261 if (pre) {
2262 pre->emit(*(iContext->getStreamContext()), *iContext);
2263 }
2264 prov = std::make_unique<const std::set<ProductProvenance>>(iThis->readProvenance(transitionIndex));
2265 if (post) {
2266 post->emit(*(iContext->getStreamContext()), *iContext);
2267 }
2268
2269 } catch (...) {
2270 if (post) {
2271 post->emit(*(iContext->getStreamContext()), *iContext);
2272 }
2273
2274 holder.doneWaiting(std::current_exception());
2275 return;
2276 }
2277 const std::set<ProductProvenance>* expected = nullptr;
2278
2279 if (pWriteTo->compare_exchange_strong(expected, prov.get())) {
2280 prov.release();
2281 }
2282 }
2283 holder.doneWaiting(std::exception_ptr());
2284 });
2285 }
2286 }
2287 }
2288
2289 void ReducedProvenanceReader::readProvenanceAsync(
2290 WaitingTaskHolder task,
2291 ModuleCallingContext const* moduleCallingContext,
2292 unsigned int transitionIndex,
2293 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept {
2294 readProvenanceAsyncImpl(this,
2295 acquirer_.serialQueueChain(),
2296 task,
2297 transitionIndex,
2298 writeTo,
2299 moduleCallingContext,
2300 rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2301 rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2302 }
2303
2304
2305 void ReducedProvenanceReader::unsafe_fillProvenance(unsigned int transitionIndex) const {
2306 ReducedProvenanceReader* me = const_cast<ReducedProvenanceReader*>(this);
2307 me->rootTree_->fillBranchEntry(
2308 me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_);
2309 }
2310
2311 std::set<ProductProvenance> ReducedProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2312 if (provVector_.empty()) {
2313 std::lock_guard<std::recursive_mutex> guard(*mutex_);
2314 ReducedProvenanceReader* me = const_cast<ReducedProvenanceReader*>(this);
2315 me->rootTree_->fillBranchEntry(
2316 me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_);
2317 }
2318 std::set<ProductProvenance> retValue;
2319 if (daqProvenanceHelper_) {
2320 for (auto const& prov : provVector_) {
2321 BranchID bid(prov.branchID_);
2322 retValue.emplace(daqProvenanceHelper_->mapBranchID(BranchID(prov.branchID_)),
2323 daqProvenanceHelper_->mapParentageID(parentageIDLookup_[prov.parentageIDIndex_]));
2324 }
2325 } else {
2326 for (auto const& prov : provVector_) {
2327 if (prov.parentageIDIndex_ >= parentageIDLookup_.size()) {
2328 throw edm::Exception(errors::LogicError)
2329 << "ReducedProvenanceReader::ReadProvenance\n"
2330 << "The parentage ID index value " << prov.parentageIDIndex_
2331 << " is out of bounds. The maximum value is " << parentageIDLookup_.size() - 1 << ".\n"
2332 << "This should never happen.\n"
2333 << "Please report this to the framework developers.";
2334 }
2335 retValue.emplace(BranchID(prov.branchID_), parentageIDLookup_[prov.parentageIDIndex_]);
2336 }
2337 }
2338
2339 const_cast<ReducedProvenanceReader*>(this)->provVector_.clear();
2340
2341 return retValue;
2342 }
2343
2344 class FullProvenanceReader : public ProvenanceReaderBase {
2345 public:
2346 explicit FullProvenanceReader(RootTree* rootTree, DaqProvenanceHelper const* daqProvenanceHelper);
2347 ~FullProvenanceReader() override {}
2348 std::set<ProductProvenance> readProvenance(unsigned int transitionIndex) const override;
2349
2350 private:
2351 void readProvenanceAsync(WaitingTaskHolder task,
2352 ModuleCallingContext const* moduleCallingContext,
2353 unsigned int transitionIndex,
2354 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept override;
2355
2356 RootTree* rootTree_;
2357 ProductProvenanceVector infoVector_;
2358
2359 CMS_SA_ALLOW mutable ProductProvenanceVector* pInfoVector_;
2360 DaqProvenanceHelper const* daqProvenanceHelper_;
2361 std::shared_ptr<std::recursive_mutex> mutex_;
2362 SharedResourcesAcquirer acquirer_;
2363 };
2364
2365 FullProvenanceReader::FullProvenanceReader(RootTree* rootTree, DaqProvenanceHelper const* daqProvenanceHelper)
2366 : ProvenanceReaderBase(),
2367 rootTree_(rootTree),
2368 infoVector_(),
2369 pInfoVector_(&infoVector_),
2370 daqProvenanceHelper_(daqProvenanceHelper),
2371 mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2372 acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {}
2373
2374 void FullProvenanceReader::readProvenanceAsync(
2375 WaitingTaskHolder task,
2376 ModuleCallingContext const* moduleCallingContext,
2377 unsigned int transitionIndex,
2378 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept {
2379 readProvenanceAsyncImpl(this,
2380 acquirer_.serialQueueChain(),
2381 task,
2382 transitionIndex,
2383 writeTo,
2384 moduleCallingContext,
2385 rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2386 rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2387 }
2388
2389 std::set<ProductProvenance> FullProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2390 {
2391 std::lock_guard<std::recursive_mutex> guard(*mutex_);
2392 rootTree_->fillBranchEntryMeta(
2393 rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex), pInfoVector_);
2394 }
2395 std::set<ProductProvenance> retValue;
2396 if (daqProvenanceHelper_) {
2397 for (auto const& info : infoVector_) {
2398 retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()),
2399 daqProvenanceHelper_->mapParentageID(info.parentageID()));
2400 }
2401 } else {
2402 for (auto const& info : infoVector_) {
2403 retValue.emplace(info);
2404 }
2405 }
2406 return retValue;
2407 }
2408
2409 class OldProvenanceReader : public ProvenanceReaderBase {
2410 public:
2411 explicit OldProvenanceReader(RootTree* rootTree,
2412 EntryDescriptionMap const& theMap,
2413 DaqProvenanceHelper const* daqProvenanceHelper);
2414 ~OldProvenanceReader() override {}
2415 std::set<ProductProvenance> readProvenance(unsigned int transitionIndex) const override;
2416
2417 private:
2418 void readProvenanceAsync(WaitingTaskHolder task,
2419 ModuleCallingContext const* moduleCallingContext,
2420 unsigned int transitionIndex,
2421 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept override;
2422
2423 edm::propagate_const<RootTree*> rootTree_;
2424 std::vector<EventEntryInfo> infoVector_;
2425
2426 CMS_SA_ALLOW mutable std::vector<EventEntryInfo>* pInfoVector_;
2427 EntryDescriptionMap const& entryDescriptionMap_;
2428 DaqProvenanceHelper const* daqProvenanceHelper_;
2429 std::shared_ptr<std::recursive_mutex> mutex_;
2430 SharedResourcesAcquirer acquirer_;
2431 };
2432
2433 OldProvenanceReader::OldProvenanceReader(RootTree* rootTree,
2434 EntryDescriptionMap const& theMap,
2435 DaqProvenanceHelper const* daqProvenanceHelper)
2436 : ProvenanceReaderBase(),
2437 rootTree_(rootTree),
2438 infoVector_(),
2439 pInfoVector_(&infoVector_),
2440 entryDescriptionMap_(theMap),
2441 daqProvenanceHelper_(daqProvenanceHelper),
2442 mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second),
2443 acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {}
2444
2445 void OldProvenanceReader::readProvenanceAsync(
2446 WaitingTaskHolder task,
2447 ModuleCallingContext const* moduleCallingContext,
2448 unsigned int transitionIndex,
2449 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept {
2450 readProvenanceAsyncImpl(this,
2451 acquirer_.serialQueueChain(),
2452 task,
2453 transitionIndex,
2454 writeTo,
2455 moduleCallingContext,
2456 rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(),
2457 rootTree_->rootDelayedReader()->postEventReadFromSourceSignal());
2458 }
2459
2460 std::set<ProductProvenance> OldProvenanceReader::readProvenance(unsigned int transitionIndex) const {
2461 {
2462 std::lock_guard<std::recursive_mutex> guard(*mutex_);
2463 rootTree_->branchEntryInfoBranch()->SetAddress(&pInfoVector_);
2464 roottree::getEntry(rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex));
2465 }
2466 std::set<ProductProvenance> retValue;
2467 for (auto const& info : infoVector_) {
2468 EntryDescriptionMap::const_iterator iter = entryDescriptionMap_.find(info.entryDescriptionID());
2469 assert(iter != entryDescriptionMap_.end());
2470 Parentage parentage(iter->second.parents());
2471 if (daqProvenanceHelper_) {
2472 retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()),
2473 daqProvenanceHelper_->mapParentageID(parentage.id()));
2474 } else {
2475 retValue.emplace(info.branchID(), parentage.id());
2476 }
2477 }
2478 return retValue;
2479 }
2480
2481 class DummyProvenanceReader : public ProvenanceReaderBase {
2482 public:
2483 DummyProvenanceReader();
2484 ~DummyProvenanceReader() override {}
2485
2486 private:
2487 std::set<ProductProvenance> readProvenance(unsigned int) const override;
2488 void readProvenanceAsync(WaitingTaskHolder task,
2489 ModuleCallingContext const* moduleCallingContext,
2490 unsigned int transitionIndex,
2491 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept override;
2492 };
2493
2494 DummyProvenanceReader::DummyProvenanceReader() : ProvenanceReaderBase() {}
2495
2496 std::set<ProductProvenance> DummyProvenanceReader::readProvenance(unsigned int) const {
2497
2498 return std::set<ProductProvenance>{};
2499 }
2500 void DummyProvenanceReader::readProvenanceAsync(
2501 WaitingTaskHolder task,
2502 ModuleCallingContext const* moduleCallingContext,
2503 unsigned int transitionIndex,
2504 std::atomic<const std::set<ProductProvenance>*>& writeTo) const noexcept {
2505 if (nullptr == writeTo.load()) {
2506 auto emptyProv = std::make_unique<const std::set<ProductProvenance>>();
2507 const std::set<ProductProvenance>* expected = nullptr;
2508 if (writeTo.compare_exchange_strong(expected, emptyProv.get())) {
2509 emptyProv.release();
2510 }
2511 }
2512 }
2513
2514 std::unique_ptr<ProvenanceReaderBase> MakeDummyProvenanceReader::makeReader(RootTree&,
2515 DaqProvenanceHelper const*) const {
2516 return std::make_unique<DummyProvenanceReader>();
2517 }
2518
2519 std::unique_ptr<ProvenanceReaderBase> MakeOldProvenanceReader::makeReader(
2520 RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2521 return std::make_unique<OldProvenanceReader>(&rootTree, *entryDescriptionMap_, daqProvenanceHelper);
2522 }
2523
2524 std::unique_ptr<ProvenanceReaderBase> MakeFullProvenanceReader::makeReader(
2525 RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2526 return std::make_unique<FullProvenanceReader>(&rootTree, daqProvenanceHelper);
2527 }
2528
2529 std::unique_ptr<ProvenanceReaderBase> MakeReducedProvenanceReader::makeReader(
2530 RootTree& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const {
2531 return std::make_unique<ReducedProvenanceReader>(&rootTree, parentageIDLookup_, daqProvenanceHelper);
2532 }
2533 }