StorageFactory/src/StorageFactory.cc

0001 #include <memory>
0002
0003 #include "Utilities/StorageFactory/interface/StorageFactory.h"
0004 #include "Utilities/StorageFactory/interface/StorageMakerFactory.h"
0005 #include "Utilities/StorageFactory/interface/StorageAccount.h"
0006 #include "Utilities/StorageFactory/interface/StorageAccountProxy.h"
0007 #include "Utilities/StorageFactory/interface/StorageProxyMaker.h"
0008 #include "Utilities/StorageFactory/interface/LocalCacheFile.h"
0009 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0010 #include "FWCore/PluginManager/interface/PluginManager.h"
0011 #include "FWCore/PluginManager/interface/standard.h"
0012 #include "FWCore/Utilities/interface/Exception.h"
0013
0014 using namespace edm::storage;
0015
0016 StorageFactory StorageFactory::s_instance;
0017
0018 StorageFactory::StorageFactory(void)
0019     : m_cacheHint(CACHE_HINT_AUTO_DETECT),
0020       m_readHint(READ_HINT_AUTO),
0021       m_accounting(false),
0022       m_tempfree(defaultMinTempFree()),
0023       m_temppath(defaultTempDir()),
0024       m_timeout(0U),
0025       m_debugLevel(0U) {
0026   setTempDir(m_temppath, m_tempfree);
0027 }
0028
0029 StorageFactory::~StorageFactory(void) {}
0030
0031 const StorageFactory *StorageFactory::get(void) { return &s_instance; }
0032
0033 StorageFactory *StorageFactory::getToModify(void) { return &s_instance; }
0034
0035 std::string StorageFactory::defaultTempDir() { return ".:$TMPDIR"; }
0036
0037 bool StorageFactory::enableAccounting(bool enabled) {
0038   bool old = m_accounting;
0039   m_accounting = enabled;
0040   return old;
0041 }
0042
0043 bool StorageFactory::accounting(void) const { return m_accounting; }
0044
0045 void StorageFactory::setCacheHint(CacheHint value) { m_cacheHint = value; }
0046
0047 StorageFactory::CacheHint StorageFactory::cacheHint(void) const { return m_cacheHint; }
0048
0049 void StorageFactory::setReadHint(ReadHint value) { m_readHint = value; }
0050
0051 StorageFactory::ReadHint StorageFactory::readHint(void) const { return m_readHint; }
0052
0053 void StorageFactory::setTimeout(unsigned int timeout) { m_timeout = timeout; }
0054
0055 unsigned int StorageFactory::timeout(void) const { return m_timeout; }
0056
0057 void StorageFactory::setDebugLevel(unsigned int level) { m_debugLevel = level; }
0058
0059 unsigned int StorageFactory::debugLevel(void) const { return m_debugLevel; }
0060
0061 void StorageFactory::setTempDir(const std::string &s, double minFreeSpace) {
0062 #if 0
0063   std::cerr /* edm::LogInfo("StorageFactory") */
0064     << "Considering path '" << s
0065     << "', min free space " << minFreeSpace
0066     << "GB for temp dir" << std::endl;
0067 #endif
0068
0069   size_t begin = 0;
0070   std::vector<std::string> dirs;
0071   dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);
0072
0073   while (true) {
0074     size_t end = s.find(':', begin);
0075     if (end == std::string::npos) {
0076       dirs.push_back(s.substr(begin, end));
0077       break;
0078     } else {
0079       dirs.push_back(s.substr(begin, end - begin));
0080       begin = end + 1;
0081     }
0082   }
0083
0084   m_temppath = s;
0085   m_tempfree = minFreeSpace;
0086   std::tie(m_tempdir, m_unusableDirWarnings) = m_lfs.findCachePath(dirs, minFreeSpace);
0087
0088 #if 0
0089   std::cerr /* edm::LogInfo("StorageFactory") */
0090     << "Using '" << m_tempdir << "' for temp dir"
0091     << std::endl;
0092 #endif
0093 }
0094
0095 std::string StorageFactory::tempDir(void) const { return m_tempdir; }
0096
0097 std::string StorageFactory::tempPath(void) const { return m_temppath; }
0098
0099 double StorageFactory::tempMinFree(void) const { return m_tempfree; }
0100
0101 void StorageFactory::setStorageProxyMakers(std::vector<std::unique_ptr<StorageProxyMaker>> makers) {
0102   m_storageProxyMakers_ = std::move(makers);
0103 }
0104
0105 StorageMaker *StorageFactory::getMaker(const std::string &proto) const {
0106   auto itFound = m_makers.find(proto);
0107   if (itFound != m_makers.end()) {
0108     return itFound->second.get();
0109   }
0110   if (!edmplugin::PluginManager::isAvailable()) {
0111     edmplugin::PluginManager::configure(edmplugin::standard::config());
0112   }
0113   std::shared_ptr<StorageMaker> instance{StorageMakerFactory::get()->tryToCreate(proto)};
0114   auto insertResult = m_makers.insert(MakerTable::value_type(proto, instance));
0115   //Can't use instance since it is possible that another thread beat
0116   // us to the insertion so the map contains a different instance.
0117   return insertResult.first->second.get();
0118 }
0119
0120 StorageMaker *StorageFactory::getMaker(const std::string &url, std::string &protocol, std::string &rest) const {
0121   size_t p = url.find(':');
0122   if (p != std::string::npos) {
0123     protocol = url.substr(0, p);
0124     rest = url.substr(p + 1);
0125   } else {
0126     protocol = "file";
0127     rest = url;
0128   }
0129
0130   return getMaker(protocol);
0131 }
0132
0133 std::unique_ptr<Storage> StorageFactory::open(const std::string &url, const int mode /* = IOFlags::OpenRead */) const {
0134   std::string protocol;
0135   std::string rest;
0136   std::unique_ptr<Storage> ret;
0137   std::unique_ptr<StorageAccount::Stamp> stats;
0138   if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0139     if (m_accounting) {
0140       auto token = StorageAccount::tokenForStorageClassName(protocol);
0141       stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::open));
0142     }
0143     try {
0144       ret = maker->open(
0145           protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
0146       if (ret) {
0147         // Inject proxy wrappers at the lowest level, in the order
0148         // specified in the configuration
0149         for (auto const &proxyMaker : m_storageProxyMakers_) {
0150           ret = proxyMaker->wrap(url, std::move(ret));
0151         }
0152
0153         // Wrap the storage to LocalCacheFile if storage backend is
0154         // not already using a local file, and lazy-download is requested
0155         if (auto const useLocalFile = maker->usesLocalFile(); useLocalFile != StorageMaker::UseLocalFile::kNo) {
0156           bool useCacheFile;
0157           std::string path;
0158           if (useLocalFile == StorageMaker::UseLocalFile::kCheckFromPath) {
0159             path = rest;
0160           }
0161           std::tie(ret, useCacheFile) = wrapNonLocalFile(std::move(ret), protocol, path, mode);
0162           if (useCacheFile) {
0163             protocol = "local-cache";
0164           }
0165         }
0166
0167         if (m_accounting)
0168           ret = std::make_unique<StorageAccountProxy>(protocol, std::move(ret));
0169
0170         if (stats)
0171           stats->tick();
0172       }
0173     } catch (cms::Exception &err) {
0174       err.addContext("Calling StorageFactory::open()");
0175       err.addAdditionalInfo(err.message());
0176       err.clearMessage();
0177       err << "Failed to open the file '" << url << "'";
0178       throw;
0179     }
0180   }
0181   return ret;
0182 }
0183
0184 void StorageFactory::stagein(const std::string &url) const {
0185   std::string protocol;
0186   std::string rest;
0187
0188   std::unique_ptr<StorageAccount::Stamp> stats;
0189   if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0190     if (m_accounting) {
0191       auto token = StorageAccount::tokenForStorageClassName(protocol);
0192       stats =
0193           std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::stagein));
0194     }
0195     try {
0196       maker->stagein(protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
0197       if (stats)
0198         stats->tick();
0199     } catch (cms::Exception &err) {
0200       edm::LogWarning("StorageFactory::stagein()") << "Failed to stage in file '" << url << "' because:\n"
0201                                                    << err.explainSelf();
0202     }
0203   }
0204 }
0205
0206 bool StorageFactory::check(const std::string &url, IOOffset *size /* = 0 */) const {
0207   std::string protocol;
0208   std::string rest;
0209
0210   bool ret = false;
0211   std::unique_ptr<StorageAccount::Stamp> stats;
0212   if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0213     if (m_accounting) {
0214       auto token = StorageAccount::tokenForStorageClassName(protocol);
0215       stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::check));
0216     }
0217     try {
0218       ret = maker->check(
0219           protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size);
0220       if (stats)
0221         stats->tick();
0222     } catch (cms::Exception &err) {
0223       edm::LogWarning("StorageFactory::check()")
0224           << "Existence or size check for the file '" << url << "' failed because:\n"
0225           << err.explainSelf();
0226     }
0227   }
0228
0229   return ret;
0230 }
0231
0232 std::tuple<std::unique_ptr<Storage>, bool> StorageFactory::wrapNonLocalFile(std::unique_ptr<Storage> s,
0233                                                                             const std::string &proto,
0234                                                                             const std::string &path,
0235                                                                             const int mode) const {
0236   StorageFactory::CacheHint hint = cacheHint();
0237   bool useCacheFile = false;
0238   if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) {
0239     if (mode & IOFlags::OpenWrite) {
0240       // For now, issue no warning - otherwise, we'd always warn on output files.
0241     } else if (m_tempdir.empty()) {
0242       edm::LogWarning("StorageFactory") << m_unusableDirWarnings;
0243     } else if ((not path.empty()) and m_lfs.isLocalPath(path)) {
0244       // For now, issue no warning - otherwise, we'd always warn on local input files.
0245     } else {
0246       if (accounting()) {
0247         s = std::make_unique<StorageAccountProxy>(proto, std::move(s));
0248       }
0249       s = std::make_unique<LocalCacheFile>(std::move(s), m_tempdir);
0250       useCacheFile = true;
0251     }
0252   }
0253
0254   return {std::move(s), useCacheFile};
0255 }