Catalog/test/FileLocator_t.cpp

0001 #include "FWCore/Catalog/interface/FileLocator.h"
0002 #include "FWCore/Catalog/interface/SiteLocalConfig.h"
0003 #include "FWCore/Utilities/interface/Exception.h"
0004 #include "FWCore/ServiceRegistry/interface/ServiceRegistry.h"
0005 #include <filesystem>
0006
0007 #include <string>
0008
0009 #define CATCH_CONFIG_MAIN
0010 #include "catch.hpp"
0011
0012 namespace {
0013   class TestSiteLocalConfig : public edm::SiteLocalConfig {
0014   public:
0015     //constructor using trivial data catalogs
0016     TestSiteLocalConfig(std::vector<std::string> catalogs) : m_trivialCatalogs(std::move(catalogs)) {}
0017     //constructor using Rucio data catalogs
0018     TestSiteLocalConfig(std::vector<edm::CatalogAttributes> catalogs) : m_catalogs(std::move(catalogs)) {}
0019     std::vector<std::string> const& trivialDataCatalogs() const final { return m_trivialCatalogs; }
0020     std::vector<edm::CatalogAttributes> const& dataCatalogs() const final { return m_catalogs; }
0021     std::filesystem::path const storageDescriptionPath(const edm::CatalogAttributes& aDataCatalog) const final {
0022       return std::filesystem::path();
0023     }
0024
0025     std::string const lookupCalibConnect(std::string const& input) const final { return std::string(); }
0026     std::string const rfioType(void) const final { return std::string(); }
0027
0028     std::string const* sourceCacheTempDir() const final { return nullptr; }
0029     double const* sourceCacheMinFree() const final { return nullptr; }
0030     std::string const* sourceCacheHint() const final { return nullptr; }
0031     std::string const* sourceCloneCacheHint() const final { return nullptr; }
0032     std::string const* sourceReadHint() const final { return nullptr; }
0033     unsigned int const* sourceTTreeCacheSize() const final { return nullptr; }
0034     unsigned int const* sourceTimeout() const final { return nullptr; }
0035     bool enablePrefetching() const final { return false; }
0036     unsigned int debugLevel() const final { return 0; }
0037     std::vector<std::string> const* sourceNativeProtocols() const final { return nullptr; }
0038     struct addrinfo const* statisticsDestination() const final { return nullptr; }
0039     std::set<std::string> const* statisticsInfo() const final { return nullptr; }
0040     std::string const& siteName(void) const final { return m_emptyString; }
0041     std::string const& subSiteName(void) const final { return m_emptyString; }
0042     bool useLocalConnectString() const final { return false; }
0043     std::string const& localConnectPrefix() const final { return m_emptyString; }
0044     std::string const& localConnectSuffix() const final { return m_emptyString; }
0045
0046   private:
0047     std::vector<std::string> m_trivialCatalogs;
0048     std::vector<edm::CatalogAttributes> m_catalogs;
0049     std::filesystem::path m_storageDescription_path;
0050     std::string m_emptyString;
0051   };
0052 }  // namespace
0053
0054 TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]") {
0055   //catalog for testing "prefix"
0056   edm::CatalogAttributes aCatalog;
0057   aCatalog.site = "T1_US_FNAL";
0058   aCatalog.subSite = "T1_US_FNAL";
0059   aCatalog.storageSite = "T1_US_FNAL";
0060   aCatalog.volume = "American_Federation";
0061   aCatalog.protocol = "XRootD";
0062   std::vector<edm::CatalogAttributes> tmp{aCatalog};
0063   //catalog for testing "rules"
0064   aCatalog.site = "T1_US_FNAL";
0065   aCatalog.subSite = "T1_US_FNAL";
0066   aCatalog.storageSite = "T1_US_FNAL";
0067   aCatalog.volume = "FNAL_dCache_EOS";
0068   aCatalog.protocol = "XRootD";
0069   tmp.push_back(aCatalog);
0070   //catalog for testing chained "rules"
0071   aCatalog.site = "T1_US_FNAL";
0072   aCatalog.subSite = "T1_US_FNAL";
0073   aCatalog.storageSite = "T1_US_FNAL";
0074   aCatalog.volume = "FNAL_dCache_EOS";
0075   aCatalog.protocol = "root";
0076   tmp.push_back(aCatalog);
0077
0078   //create the services
0079   edm::ServiceToken tempToken(
0080       edm::ServiceRegistry::createContaining(std::unique_ptr<edm::SiteLocalConfig>(new TestSiteLocalConfig(tmp))));
0081
0082   std::string CMSSW_BASE(std::getenv("CMSSW_BASE"));
0083   std::string CMSSW_RELEASE_BASE(std::getenv("CMSSW_RELEASE_BASE"));
0084   std::string file_name("/src/FWCore/Catalog/test/storage.json");
0085   std::string full_file_name = std::filesystem::exists((CMSSW_BASE + file_name).c_str())
0086                                    ? CMSSW_BASE + file_name
0087                                    : CMSSW_RELEASE_BASE + file_name;
0088
0089   SECTION("prefix") {
0090     edm::ServiceRegistry::Operate operate(tempToken);
0091     //empty catalog
0092     edm::CatalogAttributes tmp_cat;
0093     //use the first catalog provided by site local config
0094     edm::FileLocator fl(tmp_cat, 0, full_file_name);
0095     CHECK("root://cmsxrootd.fnal.gov/store/group/bha/bho" ==
0096           fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
0097   }
0098   SECTION("rule") {
0099     edm::ServiceRegistry::Operate operate(tempToken);
0100     //empty catalog
0101     edm::CatalogAttributes tmp_cat;
0102     //use the second catalog provided by site local config
0103     edm::FileLocator fl(tmp_cat, 1, full_file_name);
0104     const std::array<const char*, 7> lfn = {{"/bha/bho",
0105                                              "bha",
0106                                              "file:bha",
0107                                              "file:/bha/bho",
0108                                              "/castor/cern.ch/cms/bha/bho",
0109                                              "rfio:/castor/cern.ch/cms/bha/bho",
0110                                              "rfio:/bha/bho"}};
0111     CHECK("root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/group/bha/bho" ==
0112           fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
0113     for (auto file : lfn) {
0114       CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
0115     }
0116   }
0117   SECTION("chainedrule") {
0118     edm::ServiceRegistry::Operate operate(tempToken);
0119     //empty catalog
0120     edm::CatalogAttributes tmp_cat;
0121     //use the third catalog provided by site local config above
0122     edm::FileLocator fl(tmp_cat, 2, full_file_name);
0123     const std::array<const char*, 7> lfn = {{"/bha/bho",
0124                                              "bha",
0125                                              "file:bha",
0126                                              "file:/bha/bho",
0127                                              "/castor/cern.ch/cms/bha/bho",
0128                                              "rfio:/castor/cern.ch/cms/bha/bho",
0129                                              "rfio:/bha/bho"}};
0130     //one level chain between "root" and "second" protocols (see storage.json)
0131     CHECK("root://host.domain//pnfs/cms/store/group/bha/bho" ==
0132           fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
0133     //two levels chain between "root", "second" and "first" (see storage.json)
0134     CHECK("root://host.domain//pnfs/cms/store/user/AAA/bho" ==
0135           fl.pfn("/store/user/aaa/bho", edm::CatalogType::RucioCatalog));
0136     for (auto file : lfn) {
0137       CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
0138     }
0139   }
0140 }
0141
0142 TEST_CASE("FileLocator", "[filelocator]") {
0143   std::string CMSSW_BASE(std::getenv("CMSSW_BASE"));
0144   std::string CMSSW_RELEASE_BASE(std::getenv("CMSSW_RELEASE_BASE"));
0145   std::string file_name("/src/FWCore/Catalog/test/simple_catalog.xml");
0146   std::string full_file_name = std::filesystem::exists((CMSSW_BASE + file_name).c_str())
0147                                    ? CMSSW_BASE + file_name
0148                                    : CMSSW_RELEASE_BASE + file_name;
0149
0150   //create the services
0151   std::vector<std::string> tmp{std::string("trivialcatalog_file:") + full_file_name + "?protocol=xrd"};
0152   edm::ServiceToken tempToken(
0153       edm::ServiceRegistry::createContaining(std::unique_ptr<edm::SiteLocalConfig>(new TestSiteLocalConfig(tmp))));
0154
0155   //make the services available
0156   SECTION("standard") {
0157     edm::ServiceRegistry::Operate operate(tempToken);
0158     edm::FileLocator fl("");
0159
0160     const std::array<const char*, 7> lfn = {{"/bha/bho",
0161                                              "bha",
0162                                              "file:bha",
0163                                              "file:/bha/bho",
0164                                              "/castor/cern.ch/cms/bha/bho",
0165                                              "rfio:/castor/cern.ch/cms/bha/bho",
0166                                              "rfio:/bha/bho"}};
0167
0168     CHECK("/storage/path/store/group/bha/bho" == fl.pfn("/store/group/bha/bho", edm::CatalogType::TrivialCatalog));
0169     for (auto file : lfn) {
0170       CHECK("" == fl.pfn(file, edm::CatalogType::TrivialCatalog));
0171     }
0172   }
0173
0174   SECTION("override") {
0175     edm::ServiceRegistry::Operate operate(tempToken);
0176
0177     std::string override_file_name("/src/FWCore/Catalog/test/override_catalog.xml");
0178     std::string override_full_file_name = std::filesystem::exists((CMSSW_BASE + override_file_name).c_str())
0179                                               ? CMSSW_BASE + override_file_name
0180                                               : CMSSW_RELEASE_BASE + override_file_name;
0181
0182     edm::FileLocator fl(("trivialcatalog_file:" + override_full_file_name + "?protocol=override").c_str());
0183
0184     std::array<const char*, 8> lfn = {{"/store/group/bha/bho",
0185                                        "/bha/bho",
0186                                        "bha",
0187                                        "file:bha",
0188                                        "file:/bha/bho",
0189                                        "/castor/cern.ch/cms/bha/bho",
0190                                        "rfio:/castor/cern.ch/cms/bha/bho",
0191                                        "rfio:/bha/bho"}};
0192
0193     auto const overriden_file =
0194         "/store/unmerged/relval/CMSSW_3_8_0_pre3/RelValZTT/GEN-SIM-DIGI-RAW-HLTDEBUG/START38_V2-v1/0666/"
0195         "80EC0BCD-D279-DF11-B1DB-0030487C90EE.root";
0196
0197     CHECK("/FULL_PATH_TO_THE_FIRST_STEP_ROOT_FILE/80EC0BCD-D279-DF11-B1DB-0030487C90EE.root" ==
0198           fl.pfn(overriden_file, edm::CatalogType::TrivialCatalog));
0199
0200     for (auto f : lfn) {
0201       CHECK("" == fl.pfn(f, edm::CatalogType::TrivialCatalog));
0202     }
0203   }
0204 }