File indexing completed on 2023-03-17 11:01:48
0001
0002
0003
0004
0005 #include "FWCore/Catalog/interface/InputFileCatalog.h"
0006
0007 #include "FWCore/ServiceRegistry/interface/Service.h"
0008 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
0009
0010 #include "FWCore/Utilities/interface/Exception.h"
0011 #include "FWCore/Utilities/interface/EDMException.h"
0012 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0013
0014 #include <boost/algorithm/string.hpp>
0015
0016 #include <iostream>
0017
0018 namespace edm {
0019
0020 InputFileCatalog::InputFileCatalog(std::vector<std::string> const& fileNames,
0021 std::string const& override,
0022 bool useLFNasPFNifLFNnotFound,
0023 edm::CatalogType catType)
0024 : logicalFileNames_(fileNames), fileNames_(fileNames), fileCatalogItems_(), overrideFileLocator_() {
0025 init(override, useLFNasPFNifLFNnotFound, catType);
0026 }
0027
0028 InputFileCatalog::~InputFileCatalog() {}
0029
0030 std::vector<std::string> InputFileCatalog::fileNames(unsigned iCatalog) const {
0031 std::vector<std::string> tmp;
0032 tmp.reserve(fileCatalogItems_.size());
0033 for (auto const& item : fileCatalogItems_) {
0034 tmp.push_back(item.fileName(iCatalog));
0035 }
0036 return tmp;
0037 }
0038
0039 void InputFileCatalog::init(std::string const& inputOverride,
0040 bool useLFNasPFNifLFNnotFound,
0041 edm::CatalogType catType) {
0042 typedef std::vector<std::string>::iterator iter;
0043
0044 if (!overrideFileLocator_ && !inputOverride.empty()) {
0045 if (catType == edm::CatalogType::TrivialCatalog) {
0046 overrideFileLocator_ =
0047 std::make_unique<FileLocator>(inputOverride);
0048 } else if (catType == edm::CatalogType::RucioCatalog) {
0049
0050 std::vector<std::string> tmps;
0051 boost::algorithm::split(tmps, inputOverride, boost::is_any_of(std::string(",")));
0052 if (tmps.size() != 5) {
0053 cms::Exception ex("FileCatalog");
0054 ex << "Trying to override input file catalog but invalid input override string " << inputOverride
0055 << " (Should be site,subSite,storageSite,volume,protocol)";
0056 ex.addContext("Calling edm::InputFileCatalog::init()");
0057 throw ex;
0058 }
0059
0060 edm::CatalogAttributes inputOverride_struct(tmps[0],
0061 tmps[1],
0062 tmps[2],
0063 tmps[3],
0064 tmps[4]);
0065
0066 overrideFileLocator_ =
0067 std::make_unique<FileLocator>(inputOverride_struct);
0068 }
0069 }
0070
0071 Service<SiteLocalConfig> localconfservice;
0072 if (!localconfservice.isAvailable()) {
0073 cms::Exception ex("FileCatalog");
0074 ex << "edm::SiteLocalConfigService is not available";
0075 ex.addContext("Calling edm::InputFileCatalog::init()");
0076 throw ex;
0077 }
0078
0079 if (catType == edm::CatalogType::TrivialCatalog) {
0080 std::vector<std::string> const& tmp_dataCatalogs = localconfservice->trivialDataCatalogs();
0081 if (!fileLocators_trivalCatalog_.empty())
0082 fileLocators_trivalCatalog_.clear();
0083
0084 for (const auto& catalog : tmp_dataCatalogs) {
0085 try {
0086 fileLocators_trivalCatalog_.push_back(std::make_unique<FileLocator>(catalog));
0087 } catch (cms::Exception const& e) {
0088 edm::LogWarning("InputFileCatalog")
0089 << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
0090 << "Skip this catalog";
0091 }
0092 }
0093 if (fileLocators_trivalCatalog_.empty()) {
0094 cms::Exception ex("FileCatalog");
0095 ex << "Unable to construct any file locator in InputFileCatalog::init";
0096 ex.addContext("Calling edm::InputFileCatalog::init()");
0097 throw ex;
0098 }
0099 } else if (catType == edm::CatalogType::RucioCatalog) {
0100 std::vector<edm::CatalogAttributes> const& tmp_dataCatalogs = localconfservice->dataCatalogs();
0101 if (!fileLocators_.empty())
0102 fileLocators_.clear();
0103
0104 for (const auto& catalog : tmp_dataCatalogs) {
0105 try {
0106 fileLocators_.push_back(std::make_unique<FileLocator>(catalog));
0107 } catch (cms::Exception const& e) {
0108 edm::LogWarning("InputFileCatalog")
0109 << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
0110 << "Skip this catalog";
0111 }
0112 }
0113 if (fileLocators_.empty()) {
0114 cms::Exception ex("FileCatalog");
0115 ex << "Unable to construct any file locator in InputFileCatalog::init";
0116 ex.addContext("Calling edm::InputFileCatalog::init()");
0117 throw ex;
0118 }
0119 } else {
0120 cms::Exception ex("FileCatalog");
0121 ex << "Undefined catalog type";
0122 ex.addContext("Calling edm::InputFileCatalog::init()");
0123 throw ex;
0124 }
0125
0126 for (iter it = fileNames_.begin(), lt = logicalFileNames_.begin(), itEnd = fileNames_.end(); it != itEnd;
0127 ++it, ++lt) {
0128 boost::trim(*it);
0129 std::vector<std::string> pfns;
0130 if (it->empty()) {
0131 cms::Exception ex("FileCatalog");
0132 ex << "An empty string specified in the fileNames parameter for input source";
0133 ex.addContext("Calling edm::InputFileCatalog::init()");
0134 throw ex;
0135 }
0136 if (isPhysical(*it)) {
0137 if (it->back() == ':') {
0138 cms::Exception ex("FileCatalog");
0139 ex << "An empty physical file name specified in the fileNames parameter for input source";
0140 ex.addContext("Calling edm::InputFileCatalog::init()");
0141 throw ex;
0142 }
0143 pfns.push_back(*it);
0144
0145 lt->clear();
0146 } else {
0147 boost::trim(*lt);
0148 findFile(*lt, pfns, useLFNasPFNifLFNnotFound, catType);
0149 }
0150
0151 fileCatalogItems_.push_back(FileCatalogItem(pfns, *lt));
0152 }
0153 }
0154
0155 void InputFileCatalog::findFile(std::string const& lfn,
0156 std::vector<std::string>& pfns,
0157 bool useLFNasPFNifLFNnotFound,
0158 edm::CatalogType catType) {
0159 if (overrideFileLocator_) {
0160 pfns.push_back(overrideFileLocator_->pfn(lfn, catType));
0161 } else {
0162 if (catType == edm::CatalogType::TrivialCatalog) {
0163 for (auto const& locator : fileLocators_trivalCatalog_) {
0164 std::string pfn = locator->pfn(lfn, edm::CatalogType::TrivialCatalog);
0165 if (pfn.empty() && useLFNasPFNifLFNnotFound)
0166 pfns.push_back(lfn);
0167 else
0168 pfns.push_back(pfn);
0169 }
0170 } else if (catType == edm::CatalogType::RucioCatalog) {
0171 for (auto const& locator : fileLocators_) {
0172 std::string pfn = locator->pfn(lfn, edm::CatalogType::RucioCatalog);
0173 if (pfn.empty() && useLFNasPFNifLFNnotFound)
0174 pfns.push_back(lfn);
0175 else
0176 pfns.push_back(pfn);
0177 }
0178 } else {
0179 cms::Exception ex("FileCatalog");
0180 ex << "Undefined catalog type";
0181 ex.addContext("Calling edm::InputFileCatalog::findFile()");
0182 throw ex;
0183 }
0184 }
0185
0186
0187 }
0188
0189 }