Catalog/test/FileLocator_t.cpp

0001 #include "FWCore/Catalog/interface/FileLocator.h"
0002 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
0003 #include "FWCore/Utilities/interface/Exception.h"
0004 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0005 #include <filesystem>
0006
0007 #include <string>
0008
0009 #define CATCH_CONFIG_MAIN
0010 #include "catch.hpp"
0011
0012 namespace {
0013   class TestSiteLocalConfig : public edm::SiteLocalConfig {
0014   public:
0015     //constructor using trivial data catalogs
0016     TestSiteLocalConfig(std::vector<std::string> catalogs) : m_trivialCatalogs(std::move(catalogs)) {}
0017     //constructor using Rucio data catalogs
0018     TestSiteLocalConfig(std::vector<edm::CatalogAttributes> catalogs) : m_catalogs(std::move(catalogs)) {}
0019     std::vector<std::string> const& trivialDataCatalogs() const final { return m_trivialCatalogs; }
0020     std::vector<edm::CatalogAttributes> const& dataCatalogs() const final { return m_catalogs; }
0021     std::filesystem::path const storageDescriptionPath(const edm::CatalogAttributes& aDataCatalog) const final {
0022       return std::filesystem::path();
0023     }
0024
0025     std::string const lookupCalibConnect(std::string const& input) const final { return std::string(); }
0026     std::string const rfioType(void) const final { return std::string(); }
0027
0028     std::string const* sourceCacheTempDir() const final { return nullptr; }
0029     double const* sourceCacheMinFree() const final { return nullptr; }
0030     std::string const* sourceCacheHint() const final { return nullptr; }
0031     std::string const* sourceCloneCacheHint() const final { return nullptr; }
0032     std::string const* sourceReadHint() const final { return nullptr; }
0033     unsigned int const* sourceTTreeCacheSize() const final { return nullptr; }
0034     unsigned int const* sourceTimeout() const final { return nullptr; }
0035     bool enablePrefetching() const final { return false; }
0036     unsigned int debugLevel() const final { return 0; }
0037     std::vector<std::string> const* sourceNativeProtocols() const final { return nullptr; }
0038     struct addrinfo const* statisticsDestination() const final { return nullptr; }
0039     std::set<std::string> const* statisticsInfo() const final { return nullptr; }
0040     std::string const& siteName(void) const final { return m_emptyString; }
0041     std::string const& subSiteName(void) const final { return m_emptyString; }
0042     bool useLocalConnectString() const final { return false; }
0043     std::string const& localConnectPrefix() const final { return m_emptyString; }
0044     std::string const& localConnectSuffix() const final { return m_emptyString; }
0045
0046   private:
0047     std::vector<std::string> m_trivialCatalogs;
0048     std::vector<edm::CatalogAttributes> m_catalogs;
0049     std::filesystem::path m_storageDescription_path;
0050     std::string m_emptyString;
0051   };
0052 }  // namespace
0053
0054 TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]") {
0055   //catalog for testing "prefix"
0056   edm::CatalogAttributes aCatalog;
0057   aCatalog.site = "T1_US_FNAL";
0058   aCatalog.subSite = "T1_US_FNAL";
0059   aCatalog.storageSite = "T1_US_FNAL";
0060   aCatalog.volume = "American_Federation";
0061   aCatalog.protocol = "XRootD";
0062   std::vector<edm::CatalogAttributes> tmp{aCatalog};
0063   //catalog for testing "rules"
0064   aCatalog.site = "T1_US_FNAL";
0065   aCatalog.subSite = "T1_US_FNAL";
0066   aCatalog.storageSite = "T1_US_FNAL";
0067   aCatalog.volume = "FNAL_dCache_EOS";
0068   aCatalog.protocol = "XRootD";
0069   tmp.push_back(aCatalog);
0070
0071   //create the services
0072   edm::ServiceToken tempToken(
0073       edm::ServiceRegistry::createContaining(std::unique_ptr<edm::SiteLocalConfig>(new TestSiteLocalConfig(tmp))));
0074
0075   std::string CMSSW_BASE(std::getenv("CMSSW_BASE"));
0076   std::string CMSSW_RELEASE_BASE(std::getenv("CMSSW_RELEASE_BASE"));
0077   std::string file_name("/src/FWCore/Catalog/test/storage.json");
0078   std::string full_file_name = std::filesystem::exists((CMSSW_BASE + file_name).c_str())
0079                                    ? CMSSW_BASE + file_name
0080                                    : CMSSW_RELEASE_BASE + file_name;
0081
0082   SECTION("prefix") {
0083     edm::ServiceRegistry::Operate operate(tempToken);
0084     edm::CatalogAttributes tmp_cat;                   //empty catalog
0085     edm::FileLocator fl(tmp_cat, 0, full_file_name);  //use the first catalog provided by site local config
0086     CHECK("root://cmsxrootd.fnal.gov/store/group/bha/bho" ==
0087           fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
0088   }
0089   SECTION("rule") {
0090     edm::ServiceRegistry::Operate operate(tempToken);
0091     edm::CatalogAttributes tmp_cat;                   //empty catalog
0092     edm::FileLocator fl(tmp_cat, 1, full_file_name);  //use the second catalog provided by site local config
0093     const std::array<const char*, 7> lfn = {{"/bha/bho",
0094                                              "bha",
0095                                              "file:bha",
0096                                              "file:/bha/bho",
0097                                              "/castor/cern.ch/cms/bha/bho",
0098                                              "rfio:/castor/cern.ch/cms/bha/bho",
0099                                              "rfio:/bha/bho"}};
0100     CHECK("root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/group/bha/bho" ==
0101           fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
0102     for (auto file : lfn) {
0103       CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
0104     }
0105   }
0106 }
0107
0108 TEST_CASE("FileLocator", "[filelocator]") {
0109   std::string CMSSW_BASE(std::getenv("CMSSW_BASE"));
0110   std::string CMSSW_RELEASE_BASE(std::getenv("CMSSW_RELEASE_BASE"));
0111   std::string file_name("/src/FWCore/Catalog/test/simple_catalog.xml");
0112   std::string full_file_name = std::filesystem::exists((CMSSW_BASE + file_name).c_str())
0113                                    ? CMSSW_BASE + file_name
0114                                    : CMSSW_RELEASE_BASE + file_name;
0115
0116   //create the services
0117   std::vector<std::string> tmp{std::string("trivialcatalog_file:") + full_file_name + "?protocol=xrd"};
0118   edm::ServiceToken tempToken(
0119       edm::ServiceRegistry::createContaining(std::unique_ptr<edm::SiteLocalConfig>(new TestSiteLocalConfig(tmp))));
0120
0121   //make the services available
0122   SECTION("standard") {
0123     edm::ServiceRegistry::Operate operate(tempToken);
0124     edm::FileLocator fl("");
0125
0126     const std::array<const char*, 7> lfn = {{"/bha/bho",
0127                                              "bha",
0128                                              "file:bha",
0129                                              "file:/bha/bho",
0130                                              "/castor/cern.ch/cms/bha/bho",
0131                                              "rfio:/castor/cern.ch/cms/bha/bho",
0132                                              "rfio:/bha/bho"}};
0133
0134     CHECK("/storage/path/store/group/bha/bho" == fl.pfn("/store/group/bha/bho", edm::CatalogType::TrivialCatalog));
0135     for (auto file : lfn) {
0136       CHECK("" == fl.pfn(file, edm::CatalogType::TrivialCatalog));
0137     }
0138   }
0139
0140   SECTION("override") {
0141     edm::ServiceRegistry::Operate operate(tempToken);
0142
0143     std::string override_file_name("/src/FWCore/Catalog/test/override_catalog.xml");
0144     std::string override_full_file_name = std::filesystem::exists((CMSSW_BASE + override_file_name).c_str())
0145                                               ? CMSSW_BASE + override_file_name
0146                                               : CMSSW_RELEASE_BASE + override_file_name;
0147
0148     edm::FileLocator fl(("trivialcatalog_file:" + override_full_file_name + "?protocol=override").c_str());
0149
0150     std::array<const char*, 8> lfn = {{"/store/group/bha/bho",
0151                                        "/bha/bho",
0152                                        "bha",
0153                                        "file:bha",
0154                                        "file:/bha/bho",
0155                                        "/castor/cern.ch/cms/bha/bho",
0156                                        "rfio:/castor/cern.ch/cms/bha/bho",
0157                                        "rfio:/bha/bho"}};
0158
0159     auto const overriden_file =
0160         "/store/unmerged/relval/CMSSW_3_8_0_pre3/RelValZTT/GEN-SIM-DIGI-RAW-HLTDEBUG/START38_V2-v1/0666/"
0161         "80EC0BCD-D279-DF11-B1DB-0030487C90EE.root";
0162
0163     CHECK("/FULL_PATH_TO_THE_FIRST_STEP_ROOT_FILE/80EC0BCD-D279-DF11-B1DB-0030487C90EE.root" ==
0164           fl.pfn(overriden_file, edm::CatalogType::TrivialCatalog));
0165
0166     for (auto f : lfn) {
0167       CHECK("" == fl.pfn(f, edm::CatalogType::TrivialCatalog));
0168     }
0169   }
0170 }