StorageFactory/src/StorageFactory.cc

0001 #include <memory>
0002
0003 #include "Utilities/StorageFactory/interface/StorageFactory.h"
0004 #include "Utilities/StorageFactory/interface/StorageMakerFactory.h"
0005 #include "Utilities/StorageFactory/interface/StorageAccount.h"
0006 #include "Utilities/StorageFactory/interface/StorageAccountProxy.h"
0007 #include "Utilities/StorageFactory/interface/LocalCacheFile.h"
0008 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0009 #include "FWCore/PluginManager/interface/PluginManager.h"
0010 #include "FWCore/PluginManager/interface/standard.h"
0011 #include "FWCore/Utilities/interface/Exception.h"
0012
0013 using namespace edm::storage;
0014
0015 StorageFactory StorageFactory::s_instance;
0016
0017 StorageFactory::StorageFactory(void)
0018     : m_cacheHint(CACHE_HINT_AUTO_DETECT),
0019       m_readHint(READ_HINT_AUTO),
0020       m_accounting(false),
0021       m_tempfree(4.),  // GB
0022       m_temppath(".:$TMPDIR"),
0023       m_timeout(0U),
0024       m_debugLevel(0U) {
0025   setTempDir(m_temppath, m_tempfree);
0026 }
0027
0028 StorageFactory::~StorageFactory(void) {}
0029
0030 const StorageFactory *StorageFactory::get(void) { return &s_instance; }
0031
0032 StorageFactory *StorageFactory::getToModify(void) { return &s_instance; }
0033
0034 bool StorageFactory::enableAccounting(bool enabled) {
0035   bool old = m_accounting;
0036   m_accounting = enabled;
0037   return old;
0038 }
0039
0040 bool StorageFactory::accounting(void) const { return m_accounting; }
0041
0042 void StorageFactory::setCacheHint(CacheHint value) { m_cacheHint = value; }
0043
0044 StorageFactory::CacheHint StorageFactory::cacheHint(void) const { return m_cacheHint; }
0045
0046 void StorageFactory::setReadHint(ReadHint value) { m_readHint = value; }
0047
0048 StorageFactory::ReadHint StorageFactory::readHint(void) const { return m_readHint; }
0049
0050 void StorageFactory::setTimeout(unsigned int timeout) { m_timeout = timeout; }
0051
0052 unsigned int StorageFactory::timeout(void) const { return m_timeout; }
0053
0054 void StorageFactory::setDebugLevel(unsigned int level) { m_debugLevel = level; }
0055
0056 unsigned int StorageFactory::debugLevel(void) const { return m_debugLevel; }
0057
0058 void StorageFactory::setTempDir(const std::string &s, double minFreeSpace) {
0059 #if 0
0060   std::cerr /* edm::LogInfo("StorageFactory") */
0061     << "Considering path '" << s
0062     << "', min free space " << minFreeSpace
0063     << "GB for temp dir" << std::endl;
0064 #endif
0065
0066   size_t begin = 0;
0067   std::vector<std::string> dirs;
0068   dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);
0069
0070   while (true) {
0071     size_t end = s.find(':', begin);
0072     if (end == std::string::npos) {
0073       dirs.push_back(s.substr(begin, end));
0074       break;
0075     } else {
0076       dirs.push_back(s.substr(begin, end - begin));
0077       begin = end + 1;
0078     }
0079   }
0080
0081   m_temppath = s;
0082   m_tempfree = minFreeSpace;
0083   std::tie(m_tempdir, m_unusableDirWarnings) = m_lfs.findCachePath(dirs, minFreeSpace);
0084
0085 #if 0
0086   std::cerr /* edm::LogInfo("StorageFactory") */
0087     << "Using '" << m_tempdir << "' for temp dir"
0088     << std::endl;
0089 #endif
0090 }
0091
0092 std::string StorageFactory::tempDir(void) const { return m_tempdir; }
0093
0094 std::string StorageFactory::tempPath(void) const { return m_temppath; }
0095
0096 double StorageFactory::tempMinFree(void) const { return m_tempfree; }
0097
0098 StorageMaker *StorageFactory::getMaker(const std::string &proto) const {
0099   auto itFound = m_makers.find(proto);
0100   if (itFound != m_makers.end()) {
0101     return itFound->second.get();
0102   }
0103   if (!edmplugin::PluginManager::isAvailable()) {
0104     edmplugin::PluginManager::configure(edmplugin::standard::config());
0105   }
0106   std::shared_ptr<StorageMaker> instance{StorageMakerFactory::get()->tryToCreate(proto)};
0107   auto insertResult = m_makers.insert(MakerTable::value_type(proto, instance));
0108   //Can't use instance since it is possible that another thread beat
0109   // us to the insertion so the map contains a different instance.
0110   return insertResult.first->second.get();
0111 }
0112
0113 StorageMaker *StorageFactory::getMaker(const std::string &url, std::string &protocol, std::string &rest) const {
0114   size_t p = url.find(':');
0115   if (p != std::string::npos) {
0116     protocol = url.substr(0, p);
0117     rest = url.substr(p + 1);
0118   } else {
0119     protocol = "file";
0120     rest = url;
0121   }
0122
0123   return getMaker(protocol);
0124 }
0125
0126 std::unique_ptr<Storage> StorageFactory::open(const std::string &url, int mode /* = IOFlags::OpenRead */) const {
0127   std::string protocol;
0128   std::string rest;
0129   std::unique_ptr<Storage> ret;
0130   std::unique_ptr<StorageAccount::Stamp> stats;
0131   if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0132     if (m_accounting) {
0133       auto token = StorageAccount::tokenForStorageClassName(protocol);
0134       stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::open));
0135     }
0136     try {
0137       if (auto storage = maker->open(
0138               protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout))) {
0139         if (dynamic_cast<LocalCacheFile *>(storage.get()))
0140           protocol = "local-cache";
0141
0142         if (m_accounting)
0143           ret = std::make_unique<StorageAccountProxy>(protocol, std::move(storage));
0144         else
0145           ret = std::move(storage);
0146
0147         if (stats)
0148           stats->tick();
0149       }
0150     } catch (cms::Exception &err) {
0151       err.addContext("Calling StorageFactory::open()");
0152       err.addAdditionalInfo(err.message());
0153       err.clearMessage();
0154       err << "Failed to open the file '" << url << "'";
0155       throw;
0156     }
0157   }
0158   return ret;
0159 }
0160
0161 void StorageFactory::stagein(const std::string &url) const {
0162   std::string protocol;
0163   std::string rest;
0164
0165   std::unique_ptr<StorageAccount::Stamp> stats;
0166   if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0167     if (m_accounting) {
0168       auto token = StorageAccount::tokenForStorageClassName(protocol);
0169       stats =
0170           std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::stagein));
0171     }
0172     try {
0173       maker->stagein(protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
0174       if (stats)
0175         stats->tick();
0176     } catch (cms::Exception &err) {
0177       edm::LogWarning("StorageFactory::stagein()") << "Failed to stage in file '" << url << "' because:\n"
0178                                                    << err.explainSelf();
0179     }
0180   }
0181 }
0182
0183 bool StorageFactory::check(const std::string &url, IOOffset *size /* = 0 */) const {
0184   std::string protocol;
0185   std::string rest;
0186
0187   bool ret = false;
0188   std::unique_ptr<StorageAccount::Stamp> stats;
0189   if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0190     if (m_accounting) {
0191       auto token = StorageAccount::tokenForStorageClassName(protocol);
0192       stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::check));
0193     }
0194     try {
0195       ret = maker->check(
0196           protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size);
0197       if (stats)
0198         stats->tick();
0199     } catch (cms::Exception &err) {
0200       edm::LogWarning("StorageFactory::check()")
0201           << "Existence or size check for the file '" << url << "' failed because:\n"
0202           << err.explainSelf();
0203     }
0204   }
0205
0206   return ret;
0207 }
0208
0209 std::unique_ptr<Storage> StorageFactory::wrapNonLocalFile(std::unique_ptr<Storage> s,
0210                                                           const std::string &proto,
0211                                                           const std::string &path,
0212                                                           int mode) const {
0213   StorageFactory::CacheHint hint = cacheHint();
0214   if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) {
0215     if (mode & IOFlags::OpenWrite) {
0216       // For now, issue no warning - otherwise, we'd always warn on output files.
0217     } else if (m_tempdir.empty()) {
0218       edm::LogWarning("StorageFactory") << m_unusableDirWarnings;
0219     } else if ((not path.empty()) and m_lfs.isLocalPath(path)) {
0220       // For now, issue no warning - otherwise, we'd always warn on local input files.
0221     } else {
0222       if (accounting()) {
0223         s = std::make_unique<StorageAccountProxy>(proto, std::move(s));
0224       }
0225       s = std::make_unique<LocalCacheFile>(std::move(s), m_tempdir);
0226     }
0227   }
0228
0229   return s;
0230 }