Catalog/src/InputFileCatalog.cc

0001 //////////////////////////////////////////////////////////////////////
0002 //
0003 //////////////////////////////////////////////////////////////////////
0004
0005 #include "FWCore/Catalog/interface/InputFileCatalog.h"
0006
0007 #include "FWCore/ServiceRegistry/interface/Service.h"
0008 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
0009
0010 #include "FWCore/Utilities/interface/Exception.h"
0011 #include "FWCore/Utilities/interface/EDMException.h"
0012 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0013
0014 #include <boost/algorithm/string.hpp>
0015
0016 #include <iostream>
0017
0018 namespace edm {
0019
0020   InputFileCatalog::InputFileCatalog(std::vector<std::string> const& fileNames,
0021                                      std::string const& override,
0022                                      bool useLFNasPFNifLFNnotFound,
0023                                      edm::CatalogType catType)
0024       : logicalFileNames_(fileNames), fileNames_(fileNames), fileCatalogItems_(), overrideFileLocator_() {
0025     init(override, useLFNasPFNifLFNnotFound, catType);
0026   }
0027
0028   InputFileCatalog::~InputFileCatalog() {}
0029
0030   std::vector<std::string> InputFileCatalog::fileNames(unsigned iCatalog) const {
0031     std::vector<std::string> tmp;
0032     tmp.reserve(fileCatalogItems_.size());
0033     for (auto const& item : fileCatalogItems_) {
0034       tmp.push_back(item.fileName(iCatalog));
0035     }
0036     return tmp;
0037   }
0038
0039   void InputFileCatalog::init(std::string const& inputOverride,
0040                               bool useLFNasPFNifLFNnotFound,
0041                               edm::CatalogType catType) {
0042     typedef std::vector<std::string>::iterator iter;
0043
0044     if (!overrideFileLocator_ && !inputOverride.empty()) {
0045       if (catType == edm::CatalogType::TrivialCatalog) {
0046         overrideFileLocator_ =
0047             std::make_unique<FileLocator>(inputOverride);  // propagate_const<T> has no reset() function
0048       } else if (catType == edm::CatalogType::RucioCatalog) {
0049         //now make a struct from input string
0050         std::vector<std::string> tmps;
0051         boost::algorithm::split(tmps, inputOverride, boost::is_any_of(std::string(",")));
0052         if (tmps.size() != 5) {
0053           cms::Exception ex("FileCatalog");
0054           ex << "Trying to override input file catalog but invalid input override string " << inputOverride
0055              << " (Should be site,subSite,storageSite,volume,protocol)";
0056           ex.addContext("Calling edm::InputFileCatalog::init()");
0057           throw ex;
0058         }
0059
0060         edm::CatalogAttributes inputOverride_struct(tmps[0],   //current-site
0061                                                     tmps[1],   //current-subSite
0062                                                     tmps[2],   //desired-data-access-site
0063                                                     tmps[3],   //desired-data-access-volume
0064                                                     tmps[4]);  //desired-data-access-protocol
0065
0066         overrideFileLocator_ =
0067             std::make_unique<FileLocator>(inputOverride_struct);  // propagate_const<T> has no reset() function
0068       }
0069     }
0070
0071     Service<SiteLocalConfig> localconfservice;
0072     if (!localconfservice.isAvailable()) {
0073       cms::Exception ex("FileCatalog");
0074       ex << "edm::SiteLocalConfigService is not available";
0075       ex.addContext("Calling edm::InputFileCatalog::init()");
0076       throw ex;
0077     }
0078
0079     if (catType == edm::CatalogType::TrivialCatalog) {
0080       std::vector<std::string> const& tmp_dataCatalogs = localconfservice->trivialDataCatalogs();
0081       if (!fileLocators_trivalCatalog_.empty())
0082         fileLocators_trivalCatalog_.clear();
0083       //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init).
0084       for (const auto& catalog : tmp_dataCatalogs) {
0085         try {
0086           fileLocators_trivalCatalog_.push_back(std::make_unique<FileLocator>(catalog));
0087         } catch (cms::Exception const& e) {
0088           edm::LogWarning("InputFileCatalog")
0089               << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
0090               << "Skip this catalog";
0091         }
0092       }
0093       if (fileLocators_trivalCatalog_.empty()) {
0094         cms::Exception ex("FileCatalog");
0095         ex << "Unable to construct any file locator in InputFileCatalog::init";
0096         ex.addContext("Calling edm::InputFileCatalog::init()");
0097         throw ex;
0098       }
0099     } else if (catType == edm::CatalogType::RucioCatalog) {
0100       std::vector<edm::CatalogAttributes> const& tmp_dataCatalogs = localconfservice->dataCatalogs();
0101       if (!fileLocators_.empty())
0102         fileLocators_.clear();
0103       //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init).
0104       for (const auto& catalog : tmp_dataCatalogs) {
0105         try {
0106           fileLocators_.push_back(std::make_unique<FileLocator>(catalog));
0107         } catch (cms::Exception const& e) {
0108           edm::LogWarning("InputFileCatalog")
0109               << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what()
0110               << "Skip this catalog";
0111         }
0112       }
0113       if (fileLocators_.empty()) {
0114         cms::Exception ex("FileCatalog");
0115         ex << "Unable to construct any file locator in InputFileCatalog::init";
0116         ex.addContext("Calling edm::InputFileCatalog::init()");
0117         throw ex;
0118       }
0119     } else {
0120       cms::Exception ex("FileCatalog");
0121       ex << "Undefined catalog type";
0122       ex.addContext("Calling edm::InputFileCatalog::init()");
0123       throw ex;
0124     }
0125
0126     for (iter it = fileNames_.begin(), lt = logicalFileNames_.begin(), itEnd = fileNames_.end(); it != itEnd;
0127          ++it, ++lt) {
0128       boost::trim(*it);
0129       std::vector<std::string> pfns;
0130       if (it->empty()) {
0131         cms::Exception ex("FileCatalog");
0132         ex << "An empty string specified in the fileNames parameter for input source";
0133         ex.addContext("Calling edm::InputFileCatalog::init()");
0134         throw ex;
0135       }
0136       if (isPhysical(*it)) {
0137         if (it->back() == ':') {
0138           cms::Exception ex("FileCatalog");
0139           ex << "An empty physical file name specified in the fileNames parameter for input source";
0140           ex.addContext("Calling edm::InputFileCatalog::init()");
0141           throw ex;
0142         }
0143         pfns.push_back(*it);
0144         // Clear the LFN.
0145         lt->clear();
0146       } else {
0147         boost::trim(*lt);
0148         findFile(*lt, pfns, useLFNasPFNifLFNnotFound, catType);
0149       }
0150
0151       fileCatalogItems_.push_back(FileCatalogItem(pfns, *lt));
0152     }
0153   }
0154
0155   void InputFileCatalog::findFile(std::string const& lfn,
0156                                   std::vector<std::string>& pfns,
0157                                   bool useLFNasPFNifLFNnotFound,
0158                                   edm::CatalogType catType) {
0159     if (overrideFileLocator_) {
0160       pfns.push_back(overrideFileLocator_->pfn(lfn, catType));
0161     } else {
0162       if (catType == edm::CatalogType::TrivialCatalog) {
0163         for (auto const& locator : fileLocators_trivalCatalog_) {
0164           std::string pfn = locator->pfn(lfn, edm::CatalogType::TrivialCatalog);
0165           if (pfn.empty() && useLFNasPFNifLFNnotFound)
0166             pfns.push_back(lfn);
0167           else
0168             pfns.push_back(pfn);
0169         }
0170       } else if (catType == edm::CatalogType::RucioCatalog) {
0171         for (auto const& locator : fileLocators_) {
0172           std::string pfn = locator->pfn(lfn, edm::CatalogType::RucioCatalog);
0173           if (pfn.empty() && useLFNasPFNifLFNnotFound)
0174             pfns.push_back(lfn);
0175           else
0176             pfns.push_back(pfn);
0177         }
0178       } else {
0179         cms::Exception ex("FileCatalog");
0180         ex << "Undefined catalog type";
0181         ex.addContext("Calling edm::InputFileCatalog::findFile()");
0182         throw ex;
0183       }
0184     }
0185
0186     // Empty PFN will be found by caller.
0187   }
0188
0189 }  // namespace edm