Catalog/src/InputFileCatalog.cc

0001 //////////////////////////////////////////////////////////////////////
0002 //
0003 //////////////////////////////////////////////////////////////////////
0004
0005 #include "FWCore/Catalog/interface/InputFileCatalog.h"
0006
0007 #include "FWCore/ServiceRegistry/interface/Service.h"
0008 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
0009
0010 #include "FWCore/Utilities/interface/Exception.h"
0011 #include "FWCore/Utilities/interface/EDMException.h"
0012 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0013
0014 #include <boost/algorithm/string.hpp>
0015
0016 #include <iostream>
0017
0018 namespace edm {
0019
0020   InputFileCatalog::InputFileCatalog(std::vector<std::string> fileNames,
0021                                      std::string const& override,
0022                                      bool useLFNasPFNifLFNnotFound,
0023                                      edm::CatalogType catType)
0024       : fileCatalogItems_(), overrideFileLocator_() {
0025     init(std::move(fileNames), override, useLFNasPFNifLFNnotFound, catType);
0026   }
0027
0028   InputFileCatalog::~InputFileCatalog() {}
0029
0030   std::vector<std::string> InputFileCatalog::fileNames(unsigned iCatalog) const {
0031     std::vector<std::string> tmp;
0032     tmp.reserve(fileCatalogItems_.size());
0033     for (auto const& item : fileCatalogItems_) {
0034       tmp.push_back(item.fileName(iCatalog));
0035     }
0036     return tmp;
0037   }
0038
0039   void InputFileCatalog::init(std::vector<std::string> logicalFileNames,
0040                               std::string const& inputOverride,
0041                               bool useLFNasPFNifLFNnotFound,
0042                               edm::CatalogType catType) {
0043     typedef std::vector<std::string>::iterator iter;
0044
0045     if (!overrideFileLocator_ && !inputOverride.empty()) {
0046       if (catType == edm::CatalogType::TrivialCatalog) {
0047         overrideFileLocator_ =
0048             std::make_unique<FileLocator>(inputOverride);  // propagate_const<T> has no reset() function
0049       } else if (catType == edm::CatalogType::RucioCatalog) {
0050         //now make a struct from input string
0051         std::vector<std::string> tmps;
0052         boost::algorithm::split(tmps, inputOverride, boost::is_any_of(std::string(",")));
0053         if (tmps.size() != 5) {
0054           cms::Exception ex("FileCatalog");
0055           ex << "Trying to override input file catalog but invalid input override string " << inputOverride
0056              << " (Should be site,subSite,storageSite,volume,protocol)";
0057           ex.addContext("Calling edm::InputFileCatalog::init()");
0058           throw ex;
0059         }
0060
0061         edm::CatalogAttributes inputOverride_struct(tmps[0],   //current-site
0062                                                     tmps[1],   //current-subSite
0063                                                     tmps[2],   //desired-data-access-site
0064                                                     tmps[3],   //desired-data-access-volume
0065                                                     tmps[4]);  //desired-data-access-protocol
0066
0067         overrideFileLocator_ =
0068             std::make_unique<FileLocator>(inputOverride_struct);  // propagate_const<T> has no reset() function
0069       }
0070     }
0071
0072     Service<SiteLocalConfig> localconfservice;
0073     if (!localconfservice.isAvailable()) {
0074       cms::Exception ex("FileCatalog");
0075       ex << "edm::SiteLocalConfigService is not available";
0076       ex.addContext("Calling edm::InputFileCatalog::init()");
0077       throw ex;
0078     }
0079
0080     if (catType == edm::CatalogType::TrivialCatalog) {
0081       std::vector<std::string> const& tmp_dataCatalogs = localconfservice->trivialDataCatalogs();
0082       if (!fileLocators_trivalCatalog_.empty())
0083         fileLocators_trivalCatalog_.clear();
0084       //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init).
0085       for (const auto& catalog : tmp_dataCatalogs) {
0086         try {
0087           fileLocators_trivalCatalog_.push_back(std::make_unique<FileLocator>(catalog));
0088         } catch (cms::Exception const& e) {
0089           edm::LogWarning("InputFileCatalog")
0090               << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
0091               << "Skip this catalog";
0092         }
0093       }
0094       if (fileLocators_trivalCatalog_.empty()) {
0095         cms::Exception ex("FileCatalog");
0096         ex << "Unable to construct any file locator in InputFileCatalog::init";
0097         ex.addContext("Calling edm::InputFileCatalog::init()");
0098         throw ex;
0099       }
0100     } else if (catType == edm::CatalogType::RucioCatalog) {
0101       std::vector<edm::CatalogAttributes> const& tmp_dataCatalogs = localconfservice->dataCatalogs();
0102       if (!fileLocators_.empty())
0103         fileLocators_.clear();
0104       //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init).
0105       for (const auto& catalog : tmp_dataCatalogs) {
0106         try {
0107           fileLocators_.push_back(std::make_unique<FileLocator>(catalog));
0108         } catch (cms::Exception const& e) {
0109           edm::LogWarning("InputFileCatalog")
0110               << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
0111               << "Skip this catalog";
0112         }
0113       }
0114       if (fileLocators_.empty()) {
0115         cms::Exception ex("FileCatalog");
0116         ex << "Unable to construct any file locator in InputFileCatalog::init";
0117         ex.addContext("Calling edm::InputFileCatalog::init()");
0118         throw ex;
0119       }
0120     } else {
0121       cms::Exception ex("FileCatalog");
0122       ex << "Undefined catalog type";
0123       ex.addContext("Calling edm::InputFileCatalog::init()");
0124       throw ex;
0125     }
0126
0127     for (auto& lfn : logicalFileNames) {
0128       boost::trim(lfn);
0129       std::vector<std::string> pfns;
0130       if (lfn.empty()) {
0131         cms::Exception ex("FileCatalog");
0132         ex << "An empty string specified in the fileNames parameter for input source";
0133         ex.addContext("Calling edm::InputFileCatalog::init()");
0134         throw ex;
0135       }
0136       if (isPhysical(lfn)) {
0137         if (lfn.back() == ':') {
0138           cms::Exception ex("FileCatalog");
0139           ex << "An empty physical file name specified in the fileNames parameter for input source";
0140           ex.addContext("Calling edm::InputFileCatalog::init()");
0141           throw ex;
0142         }
0143         pfns.push_back(lfn);
0144         // Clear the LFN.
0145         lfn.clear();
0146       } else {
0147         findFile(lfn, pfns, useLFNasPFNifLFNnotFound, catType);
0148       }
0149       lfn.shrink_to_fit();  // try to release memory
0150
0151       fileCatalogItems_.emplace_back(std::move(pfns), std::move(lfn));
0152     }
0153   }
0154
0155   void InputFileCatalog::findFile(std::string const& lfn,
0156                                   std::vector<std::string>& pfns,
0157                                   bool useLFNasPFNifLFNnotFound,
0158                                   edm::CatalogType catType) {
0159     if (overrideFileLocator_) {
0160       pfns.push_back(overrideFileLocator_->pfn(lfn, catType));
0161     } else {
0162       if (catType == edm::CatalogType::TrivialCatalog) {
0163         for (auto const& locator : fileLocators_trivalCatalog_) {
0164           std::string pfn = locator->pfn(lfn, edm::CatalogType::TrivialCatalog);
0165           if (pfn.empty() && useLFNasPFNifLFNnotFound)
0166             pfns.push_back(lfn);
0167           else
0168             pfns.push_back(pfn);
0169         }
0170       } else if (catType == edm::CatalogType::RucioCatalog) {
0171         for (auto const& locator : fileLocators_) {
0172           std::string pfn = locator->pfn(lfn, edm::CatalogType::RucioCatalog);
0173           if (pfn.empty() && useLFNasPFNifLFNnotFound)
0174             pfns.push_back(lfn);
0175           else
0176             pfns.push_back(pfn);
0177         }
0178       } else {
0179         cms::Exception ex("FileCatalog");
0180         ex << "Undefined catalog type";
0181         ex.addContext("Calling edm::InputFileCatalog::findFile()");
0182         throw ex;
0183       }
0184     }
0185
0186     // Empty PFN will be found by caller.
0187   }
0188
0189 }  // namespace edm