File indexing completed on 2024-04-06 12:11:38
0001 #include <boost/regex.hpp>
0002
0003 #include "TFile.h"
0004 #include "TEveTreeTools.h"
0005 #include "TError.h"
0006 #include "TMath.h"
0007 #include "TEnv.h"
0008
0009 #include "DataFormats/FWLite/interface/Handle.h"
0010 #include "DataFormats/Common/interface/TriggerResults.h"
0011 #include "DataFormats/Provenance/interface/ProcessConfiguration.h"
0012 #include "DataFormats/Provenance/interface/ProcessHistory.h"
0013 #include "DataFormats/Provenance/interface/ReleaseVersion.h"
0014 #include "DataFormats/Provenance/interface/ParameterSetBlob.h"
0015 #include "DataFormats/Provenance/interface/ParameterSetID.h"
0016 #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
0017
0018 #include "FWCore/Utilities/interface/WrappedClassName.h"
0019 #include "FWCore/Common/interface/TriggerNames.h"
0020 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0021 #include "FWCore/ParameterSet/interface/Registry.h"
0022
0023 #include "Fireworks/Core/interface/FWEventItem.h"
0024 #include "Fireworks/Core/interface/FWFileEntry.h"
0025 #include "Fireworks/Core/interface/FWEventItemsManager.h"
0026 #include "Fireworks/Core/interface/fwLog.h"
0027 #include "Fireworks/Core/interface/fwPaths.h"
0028
0029 #include "Fireworks/Core/interface/FWGUIManager.h"
0030
0031 #include "Fireworks/Core/src/FWTTreeCache.h"
0032
0033 #include <functional>
0034
0035 FWFileEntry::FWFileEntry(const std::string& name, bool checkVersion, bool checkGT)
0036 : m_name(name),
0037 m_file(nullptr),
0038 m_eventTree(nullptr),
0039 m_event(nullptr),
0040 m_needUpdate(true),
0041 m_globalTag("gt_undef"),
0042 m_globalEventList(nullptr) {
0043 openFile(checkVersion, checkGT);
0044 }
0045
0046 FWFileEntry::~FWFileEntry() {
0047 for (std::list<Filter*>::iterator i = m_filterEntries.begin(); i != m_filterEntries.end(); ++i)
0048 delete (*i)->m_eventList;
0049
0050 delete m_globalEventList;
0051 }
0052
0053 void FWFileEntry::openFile(bool checkVersion, bool checkGlobalTag) {
0054 gErrorIgnoreLevel = 3000;
0055
0056 TFile* newFile = TFile::Open(m_name.c_str());
0057
0058 if (newFile == nullptr || newFile->IsZombie() || !newFile->Get("Events")) {
0059
0060
0061 throw std::runtime_error("Invalid file. Ignored.");
0062 }
0063
0064 m_file = newFile;
0065
0066 gErrorIgnoreLevel = -1;
0067
0068
0069 typedef std::vector<edm::ProcessHistory> provList;
0070
0071 TTree* metaData = dynamic_cast<TTree*>(m_file->Get("MetaData"));
0072 TBranch* b = metaData->GetBranch("ProcessHistory");
0073 edm::ProcessHistoryVector phv_;
0074 edm::ProcessHistoryVector* pPhv = &phv_;
0075 metaData->SetBranchAddress("ProcessHistory", &pPhv);
0076
0077 b->GetEntry(0);
0078
0079 typedef std::map<edm::ParameterSetID, edm::ParameterSetBlob> ParameterSetMap;
0080 ParameterSetMap psm_;
0081 TTree* psetTree = dynamic_cast<TTree*>(newFile->Get("ParameterSets"));
0082 typedef std::pair<edm::ParameterSetID, edm::ParameterSetBlob> IdToBlobs;
0083 IdToBlobs idToBlob;
0084 IdToBlobs* pIdToBlob = &idToBlob;
0085 psetTree->SetBranchAddress("IdToParameterSetsBlobs", &pIdToBlob);
0086 for (long long i = 0; i != psetTree->GetEntries(); ++i) {
0087 psetTree->GetEntry(i);
0088 psm_.insert(idToBlob);
0089 }
0090
0091 edm::pset::Registry& psetRegistry = *edm::pset::Registry::instance();
0092 for (auto const& item : psm_) {
0093 edm::ParameterSet pset(item.second.pset());
0094 pset.setID(item.first);
0095 psetRegistry.insertMapped(pset);
0096 }
0097
0098 const edm::ProcessConfiguration* dd = nullptr;
0099 int latestVersion = 0;
0100 int currentVersionArr[] = {0, 0, 0};
0101 for (auto const& processHistory : phv_) {
0102 for (auto const& processConfiguration : processHistory) {
0103
0104 TString dcv = processConfiguration.releaseVersion();
0105 fireworks::getDecomposedVersion(dcv, currentVersionArr);
0106 int nvv = currentVersionArr[0] * 100 + currentVersionArr[1] * 10 + currentVersionArr[2];
0107 if (nvv > latestVersion) {
0108 latestVersion = nvv;
0109 dd = &processConfiguration;
0110 }
0111 }
0112 }
0113
0114
0115 if (checkGlobalTag) {
0116 std::map<edm::ProcessConfigurationID, unsigned int> simpleIDs;
0117 m_globalTag = "";
0118 for (auto const& ph : phv_) {
0119 for (auto const& pc : ph) {
0120 unsigned int id = simpleIDs[pc.id()];
0121 if (0 == id) {
0122 id = 1;
0123 simpleIDs[pc.id()] = id;
0124 }
0125 ParameterSetMap::const_iterator itFind = psm_.find(pc.parameterSetID());
0126 if (itFind == psm_.end()) {
0127 std::cout << "No ParameterSetID for " << pc.parameterSetID() << std::endl;
0128 fwLog(fwlog::kInfo) << "FWFileEntry::openFile no ParameterSetID for " << pc.parameterSetID() << std::endl;
0129 } else {
0130 edm::ParameterSet processConfig(itFind->second.pset());
0131 std::vector<std::string> sourceStrings, moduleStrings;
0132 std::vector<std::string> sources = processConfig.getParameter<std::vector<std::string>>("@all_essources");
0133 for (auto& itM : sources) {
0134 edm::ParameterSet const& pset = processConfig.getParameterSet(itM);
0135 std::string name(pset.getParameter<std::string>("@module_label"));
0136 if (name.empty()) {
0137 name = pset.getParameter<std::string>("@module_type");
0138 }
0139 if (name != "GlobalTag")
0140 continue;
0141
0142 for (auto const& item : pset.tbl()) {
0143 if (item.first == "globaltag") {
0144 m_globalTag = item.second.getString();
0145 goto gtEnd;
0146 }
0147 }
0148 }
0149 }
0150 }
0151 }
0152
0153 gtEnd:
0154 fwLog(fwlog::kDebug) << "FWFileEntry::openFile detected global tag " << m_globalTag << "\n";
0155 }
0156
0157
0158 if (checkVersion) {
0159 if (latestVersion) {
0160 fwLog(fwlog::kInfo) << "Checking process history. " << m_name.c_str() << " latest process \"" << dd->processName()
0161 << "\", version " << dd->releaseVersion() << std::endl;
0162
0163 b->SetAddress(nullptr);
0164 TString v = dd->releaseVersion();
0165 if (!fireworks::acceptDataFormatsVersion(v)) {
0166 int* di = (fireworks::supportedDataFormatsVersion());
0167 TString msg = Form(
0168 "incompatible data: Process version does not mactch major data formats version. File produced with %s. "
0169 "Data formats version \"CMSSW_%d_%d_%d\".\n",
0170 dd->releaseVersion().c_str(),
0171 di[0],
0172 di[1],
0173 di[2]);
0174 msg += "Use --no-version-check option if you still want to view the file.\n";
0175 throw std::runtime_error(msg.Data());
0176 }
0177 } else {
0178 TString msg = "No process history available\n";
0179 msg += "Use --no-version-check option if you still want to view the file.\n";
0180 throw std::runtime_error(msg.Data());
0181 }
0182 }
0183
0184 m_eventTree = dynamic_cast<TTree*>(m_file->Get("Events"));
0185
0186 if (m_eventTree == nullptr) {
0187 throw std::runtime_error("Cannot find TTree 'Events' in the data file");
0188 }
0189
0190
0191 if (FWTTreeCache::IsLogging())
0192 printf("FWFileEntry::openFile enabling FWTTreeCache for file class '%s'.", m_file->ClassName());
0193
0194 auto tc = new FWTTreeCache(m_eventTree, FWTTreeCache::GetDefaultCacheSize());
0195 m_file->SetCacheRead(tc, m_eventTree);
0196 tc->SetEnablePrefetching(FWTTreeCache::IsPrefetching());
0197 tc->SetLearnEntries(20);
0198 tc->SetLearnPrefill(TTreeCache::kAllBranches);
0199 tc->StartLearningPhase();
0200
0201
0202 m_event = new fwlite::Event(m_file, false, [tc](TBranch const& b) { tc->BranchAccessCallIn(&b); });
0203
0204
0205 FWEventItemsManager* eiMng = (FWEventItemsManager*)FWGUIManager::getGUIManager()->getContext()->eventItemsManager();
0206 eiMng->newItem_.connect(std::bind(&FWFileEntry::NewEventItemCallIn, this, std::placeholders::_1));
0207 eiMng->removingItem_.connect(std::bind(&FWFileEntry::RemovingEventItemCallIn, this, std::placeholders::_1));
0208
0209
0210 if (m_event->size() == 0)
0211 throw std::runtime_error("fwlite::Event size == 0");
0212 }
0213
0214 void FWFileEntry::closeFile() {
0215 if (m_file) {
0216 printf("Reading %lld bytes in %d transactions.\n", m_file->GetBytesRead(), m_file->GetReadCalls());
0217 delete m_file->GetCacheRead(m_eventTree);
0218
0219 m_file->Close();
0220 delete m_file;
0221 }
0222 if (m_event)
0223 delete m_event;
0224 }
0225
0226
0227
0228 bool FWFileEntry::isEventSelected(int tree_entry) {
0229 int idx = m_globalEventList->GetIndex(tree_entry);
0230 return idx >= 0;
0231 }
0232
0233 bool FWFileEntry::hasSelectedEvents() { return m_globalEventList->GetN() > 0; }
0234
0235 int FWFileEntry::firstSelectedEvent() {
0236 if (m_globalEventList->GetN() > 0) {
0237 return m_globalEventList->GetEntry(0);
0238 } else {
0239 return -1;
0240 }
0241 }
0242
0243 int FWFileEntry::lastSelectedEvent() {
0244 if (m_globalEventList->GetN() > 0)
0245 return m_globalEventList->GetEntry(m_globalEventList->GetN() - 1);
0246 else
0247 return -1;
0248 }
0249
0250 int FWFileEntry::nextSelectedEvent(int tree_entry) {
0251
0252
0253
0254
0255 const Long64_t* list = m_globalEventList->GetList();
0256 Long64_t val = tree_entry;
0257 Long64_t idx = TMath::BinarySearch(m_globalEventList->GetN(), list, val);
0258 ++idx;
0259 if (idx >= m_globalEventList->GetN() || idx < 0)
0260 return -1;
0261 return list[idx];
0262 }
0263
0264 int FWFileEntry::previousSelectedEvent(int tree_entry) {
0265
0266
0267
0268
0269 const Long64_t* list = m_globalEventList->GetList();
0270 Long64_t val = tree_entry;
0271 Long64_t idx = TMath::BinarySearch(m_globalEventList->GetN(), list, val);
0272 if (list[idx] == val)
0273 --idx;
0274 if (idx >= 0)
0275 return list[idx];
0276 else
0277 return -1;
0278 }
0279
0280
0281 bool FWFileEntry::hasActiveFilters() {
0282 for (std::list<Filter*>::iterator it = m_filterEntries.begin(); it != m_filterEntries.end(); ++it) {
0283 if ((*it)->m_selector->m_enabled)
0284 return true;
0285 }
0286
0287 return false;
0288 }
0289
0290
0291 void FWFileEntry::updateFilters(const FWEventItemsManager* eiMng, bool globalOR) {
0292 if (!m_needUpdate)
0293 return;
0294
0295 if (m_globalEventList)
0296 m_globalEventList->Reset();
0297 else
0298 m_globalEventList = new FWTEventList;
0299
0300 for (std::list<Filter*>::iterator it = m_filterEntries.begin(); it != m_filterEntries.end(); ++it) {
0301 if ((*it)->m_selector->m_enabled && (*it)->m_needsUpdate) {
0302 runFilter(*it, eiMng);
0303 }
0304
0305
0306 if ((*it)->m_selector->m_enabled) {
0307 if ((*it)->hasSelectedEvents()) {
0308 if (globalOR || m_globalEventList->GetN() == 0) {
0309 m_globalEventList->Add((*it)->m_eventList);
0310 } else {
0311 m_globalEventList->Intersect((*it)->m_eventList);
0312 }
0313 } else if (!globalOR) {
0314 m_globalEventList->Reset();
0315 break;
0316 }
0317 }
0318 }
0319
0320 fwLog(fwlog::kDebug) << "FWFileEntry::updateFilters in [" << m_file->GetName() << "] global selection ["
0321 << m_globalEventList->GetN() << "/" << m_eventTree->GetEntries() << "]" << std::endl;
0322
0323 m_needUpdate = false;
0324 }
0325
0326
0327 void FWFileEntry::runFilter(Filter* filter, const FWEventItemsManager* eiMng) {
0328 if (!filter->m_selector->m_triggerProcess.empty()) {
0329 filterEventsWithCustomParser(filter);
0330 return;
0331 }
0332
0333
0334 std::string interpretedSelection = filter->m_selector->m_expression;
0335
0336 std::vector<std::string> branch_names;
0337
0338 for (FWEventItemsManager::const_iterator i = eiMng->begin(), end = eiMng->end(); i != end; ++i) {
0339 FWEventItem* item = *i;
0340 if (item == nullptr)
0341 continue;
0342
0343 if (!item->hasEvent()) {
0344 item->setEvent(m_event);
0345 item->getPrimaryData();
0346 item->setEvent(nullptr);
0347 }
0348
0349 boost::regex re(std::string("\\$") + (*i)->name());
0350
0351 if (boost::regex_search(interpretedSelection, re)) {
0352 const edm::TypeWithDict elementType(const_cast<TClass*>(item->type()));
0353 const edm::TypeWithDict wrapperType = edm::TypeWithDict::byName(edm::wrappedClassName(elementType.name()));
0354 std::string fullBranchName = m_event->getBranchNameFor(wrapperType.typeInfo(),
0355 item->moduleLabel().c_str(),
0356 item->productInstanceLabel().c_str(),
0357 item->processName().c_str());
0358
0359 interpretedSelection = boost::regex_replace(interpretedSelection, re, fullBranchName + ".obj");
0360
0361 branch_names.push_back(fullBranchName);
0362
0363
0364
0365
0366
0367 }
0368 }
0369
0370 std::size_t found = interpretedSelection.find('$');
0371 if (found != std::string::npos) {
0372 fwLog(fwlog::kError) << "FWFileEntry::RunFilter invalid expression " << interpretedSelection << std::endl;
0373 filter->m_needsUpdate = false;
0374 return;
0375 }
0376
0377 m_file->cd();
0378 m_eventTree->SetEventList(nullptr);
0379
0380 auto prevCache = m_file->GetCacheRead(m_eventTree);
0381
0382 auto interCache = new TTreeCache(m_eventTree, 10 * 1024 * 1024);
0383
0384 m_file->SetCacheRead(interCache, m_eventTree, TFile::kDoNotDisconnect);
0385 interCache->SetEnablePrefetching(FWTTreeCache::IsPrefetching());
0386 for (auto& b : branch_names)
0387 interCache->AddBranch(b.c_str(), true);
0388 interCache->StopLearningPhase();
0389
0390
0391
0392
0393
0394 std::map<TBranch*, void*> prevAddrs;
0395
0396 {
0397 TObjArray* branches = m_eventTree->GetListOfBranches();
0398 std::unique_ptr<TIterator> pIt(branches->MakeIterator());
0399 while (TObject* branchObj = pIt->Next()) {
0400 TBranch* b = dynamic_cast<TBranch*>(branchObj);
0401 if (nullptr != b) {
0402 const char* name = b->GetName();
0403 unsigned int length = strlen(name);
0404 if (length > 1 && name[length - 1] != '.') {
0405
0406 continue;
0407 }
0408 if (nullptr != b->GetAddress()) {
0409 if (prevAddrs.find(b) != prevAddrs.end()) {
0410 fwLog(fwlog::kWarning) << "FWFileEntry::runFilter branch is already in the map!\n";
0411 }
0412 prevAddrs.insert(std::make_pair(b, b->GetAddress()));
0413
0414
0415 b->SetAddress(nullptr);
0416 }
0417 }
0418 }
0419 }
0420
0421 if (filter->m_eventList)
0422 filter->m_eventList->Reset();
0423 else
0424 filter->m_eventList = new FWTEventList;
0425
0426 fwLog(fwlog::kInfo) << "FWFileEntry::runFilter Running filter " << interpretedSelection << "' "
0427 << "for file '" << m_file->GetName() << "'.\n";
0428
0429 TEveSelectorToEventList stoelist(filter->m_eventList, interpretedSelection.c_str());
0430 Long64_t result = m_eventTree->Process(&stoelist);
0431
0432 if (result < 0)
0433 fwLog(fwlog::kWarning) << "FWFileEntry::runFilter in file [" << m_file->GetName() << "] filter ["
0434 << filter->m_selector->m_expression << "] is invalid." << std::endl;
0435 else
0436 fwLog(fwlog::kDebug) << "FWFileEntry::runFilter is file [" << m_file->GetName() << "], filter ["
0437 << filter->m_selector->m_expression << "] has [" << filter->m_eventList->GetN()
0438 << "] events selected" << std::endl;
0439
0440
0441 {
0442 for (auto i : prevAddrs) {
0443
0444 i.first->SetAddress(i.second);
0445 }
0446 }
0447
0448 m_file->SetCacheRead(prevCache, m_eventTree);
0449 delete interCache;
0450
0451 filter->m_needsUpdate = false;
0452 }
0453
0454
0455
0456 bool FWFileEntry::filterEventsWithCustomParser(Filter* filterEntry) {
0457 std::string selection(filterEntry->m_selector->m_expression);
0458
0459 boost::regex re_spaces("\\s+");
0460 selection = boost::regex_replace(selection, re_spaces, "");
0461 if (selection.find("&&") != std::string::npos && selection.find("||") != std::string::npos) {
0462
0463 return false;
0464 }
0465
0466 fwlite::Handle<edm::TriggerResults> hTriggerResults;
0467 edm::TriggerNames const* triggerNames(nullptr);
0468 try {
0469 hTriggerResults.getByLabel(*m_event, "TriggerResults", "", filterEntry->m_selector->m_triggerProcess.c_str());
0470 triggerNames = &(m_event->triggerNames(*hTriggerResults));
0471 } catch (...) {
0472 fwLog(fwlog::kWarning) << " failed to get trigger results with process name "
0473 << filterEntry->m_selector->m_triggerProcess << std::endl;
0474 return false;
0475 }
0476
0477
0478
0479
0480
0481
0482 bool junction_mode = true;
0483 if (selection.find("||") != std::string::npos)
0484 junction_mode = false;
0485
0486 boost::regex re("\\&\\&|\\|\\|");
0487
0488 boost::sregex_token_iterator i(selection.begin(), selection.end(), re, -1);
0489 boost::sregex_token_iterator j;
0490
0491
0492 std::vector<std::pair<unsigned int, bool>> filters;
0493
0494 while (i != j) {
0495 std::string filter = *i++;
0496 bool flag = true;
0497 if (filter[0] == '!') {
0498 flag = false;
0499 filter.erase(filter.begin());
0500 }
0501 unsigned int index = triggerNames->triggerIndex(filter);
0502 if (index == triggerNames->size()) {
0503
0504 return false;
0505 }
0506 filters.push_back(std::make_pair(index, flag));
0507 }
0508 if (filters.empty())
0509 return false;
0510
0511 if (filterEntry->m_eventList)
0512 filterEntry->m_eventList->Reset();
0513 else
0514 filterEntry->m_eventList = new FWTEventList();
0515 FWTEventList* list = filterEntry->m_eventList;
0516
0517
0518 edm::EventID currentEvent = m_event->id();
0519 unsigned int iEvent = 0;
0520
0521 for (m_event->toBegin(); !m_event->atEnd(); ++(*m_event)) {
0522 hTriggerResults.getByLabel(*m_event, "TriggerResults", "", filterEntry->m_selector->m_triggerProcess.c_str());
0523 std::vector<std::pair<unsigned int, bool>>::const_iterator filter = filters.begin();
0524 bool passed = hTriggerResults->accept(filter->first) == filter->second;
0525 while (++filter != filters.end()) {
0526 if (junction_mode)
0527 passed &= hTriggerResults->accept(filter->first) == filter->second;
0528 else
0529 passed |= hTriggerResults->accept(filter->first) == filter->second;
0530 }
0531 if (passed)
0532 list->Enter(iEvent);
0533 ++iEvent;
0534 }
0535 m_event->to(currentEvent);
0536
0537 filterEntry->m_needsUpdate = false;
0538
0539 fwLog(fwlog::kDebug) << "FWFile::filterEventsWithCustomParser file [" << m_file->GetName() << "], filter ["
0540 << filterEntry->m_selector->m_expression << "], selected [" << list->GetN() << "]" << std::endl;
0541
0542 return true;
0543 }
0544
0545
0546
0547 FWTTreeCache* FWFileEntry::fwTreeCache() {
0548 FWTTreeCache* tc = dynamic_cast<FWTTreeCache*>(m_file->GetCacheRead(m_eventTree));
0549 assert(tc != nullptr && "FWFileEntry::treeCache can not access TTreeCache");
0550 return tc;
0551 }
0552
0553 std::string FWFileEntry::getBranchName(const FWEventItem* it) const {
0554 const edm::TypeWithDict elementType(const_cast<TClass*>(it->type()));
0555 const edm::TypeWithDict wrapperType = edm::TypeWithDict::byName(edm::wrappedClassName(elementType.name()));
0556 return m_event->getBranchNameFor(
0557 wrapperType.typeInfo(), it->moduleLabel().c_str(), it->productInstanceLabel().c_str(), it->processName().c_str());
0558 }
0559
0560 void FWFileEntry::NewEventItemCallIn(const FWEventItem* it) {
0561 auto tc = fwTreeCache();
0562
0563 if (FWTTreeCache::IsLogging())
0564 printf("FWFileEntry:NewEventItemCallIn FWEventItem %s, learning=%d\n", getBranchName(it).c_str(), tc->IsLearning());
0565
0566 tc->AddBranchTopLevel(getBranchName(it).c_str());
0567 }
0568
0569 void FWFileEntry::RemovingEventItemCallIn(const FWEventItem* it) {
0570 auto tc = fwTreeCache();
0571
0572 if (FWTTreeCache::IsLogging())
0573 printf("FWFileEntry:RemovingEventItemCallIn FWEventItem %s, learning=%d\n",
0574 getBranchName(it).c_str(),
0575 tc->IsLearning());
0576
0577 tc->DropBranchTopLevel(getBranchName(it).c_str());
0578 }