File indexing completed on 2024-04-06 12:31:52
0001 #include <memory>
0002
0003 #include "Utilities/StorageFactory/interface/StorageFactory.h"
0004 #include "Utilities/StorageFactory/interface/StorageMakerFactory.h"
0005 #include "Utilities/StorageFactory/interface/StorageAccount.h"
0006 #include "Utilities/StorageFactory/interface/StorageAccountProxy.h"
0007 #include "Utilities/StorageFactory/interface/LocalCacheFile.h"
0008 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0009 #include "FWCore/PluginManager/interface/PluginManager.h"
0010 #include "FWCore/PluginManager/interface/standard.h"
0011 #include "FWCore/Utilities/interface/Exception.h"
0012
0013 using namespace edm::storage;
0014
0015 StorageFactory StorageFactory::s_instance;
0016
0017 StorageFactory::StorageFactory(void)
0018 : m_cacheHint(CACHE_HINT_AUTO_DETECT),
0019 m_readHint(READ_HINT_AUTO),
0020 m_accounting(false),
0021 m_tempfree(4.),
0022 m_temppath(".:$TMPDIR"),
0023 m_timeout(0U),
0024 m_debugLevel(0U) {
0025 setTempDir(m_temppath, m_tempfree);
0026 }
0027
0028 StorageFactory::~StorageFactory(void) {}
0029
0030 const StorageFactory *StorageFactory::get(void) { return &s_instance; }
0031
0032 StorageFactory *StorageFactory::getToModify(void) { return &s_instance; }
0033
0034 bool StorageFactory::enableAccounting(bool enabled) {
0035 bool old = m_accounting;
0036 m_accounting = enabled;
0037 return old;
0038 }
0039
0040 bool StorageFactory::accounting(void) const { return m_accounting; }
0041
0042 void StorageFactory::setCacheHint(CacheHint value) { m_cacheHint = value; }
0043
0044 StorageFactory::CacheHint StorageFactory::cacheHint(void) const { return m_cacheHint; }
0045
0046 void StorageFactory::setReadHint(ReadHint value) { m_readHint = value; }
0047
0048 StorageFactory::ReadHint StorageFactory::readHint(void) const { return m_readHint; }
0049
0050 void StorageFactory::setTimeout(unsigned int timeout) { m_timeout = timeout; }
0051
0052 unsigned int StorageFactory::timeout(void) const { return m_timeout; }
0053
0054 void StorageFactory::setDebugLevel(unsigned int level) { m_debugLevel = level; }
0055
0056 unsigned int StorageFactory::debugLevel(void) const { return m_debugLevel; }
0057
0058 void StorageFactory::setTempDir(const std::string &s, double minFreeSpace) {
0059 #if 0
0060 std::cerr
0061 << "Considering path '" << s
0062 << "', min free space " << minFreeSpace
0063 << "GB for temp dir" << std::endl;
0064 #endif
0065
0066 size_t begin = 0;
0067 std::vector<std::string> dirs;
0068 dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);
0069
0070 while (true) {
0071 size_t end = s.find(':', begin);
0072 if (end == std::string::npos) {
0073 dirs.push_back(s.substr(begin, end));
0074 break;
0075 } else {
0076 dirs.push_back(s.substr(begin, end - begin));
0077 begin = end + 1;
0078 }
0079 }
0080
0081 m_temppath = s;
0082 m_tempfree = minFreeSpace;
0083 std::tie(m_tempdir, m_unusableDirWarnings) = m_lfs.findCachePath(dirs, minFreeSpace);
0084
0085 #if 0
0086 std::cerr
0087 << "Using '" << m_tempdir << "' for temp dir"
0088 << std::endl;
0089 #endif
0090 }
0091
0092 std::string StorageFactory::tempDir(void) const { return m_tempdir; }
0093
0094 std::string StorageFactory::tempPath(void) const { return m_temppath; }
0095
0096 double StorageFactory::tempMinFree(void) const { return m_tempfree; }
0097
0098 StorageMaker *StorageFactory::getMaker(const std::string &proto) const {
0099 auto itFound = m_makers.find(proto);
0100 if (itFound != m_makers.end()) {
0101 return itFound->second.get();
0102 }
0103 if (!edmplugin::PluginManager::isAvailable()) {
0104 edmplugin::PluginManager::configure(edmplugin::standard::config());
0105 }
0106 std::shared_ptr<StorageMaker> instance{StorageMakerFactory::get()->tryToCreate(proto)};
0107 auto insertResult = m_makers.insert(MakerTable::value_type(proto, instance));
0108
0109
0110 return insertResult.first->second.get();
0111 }
0112
0113 StorageMaker *StorageFactory::getMaker(const std::string &url, std::string &protocol, std::string &rest) const {
0114 size_t p = url.find(':');
0115 if (p != std::string::npos) {
0116 protocol = url.substr(0, p);
0117 rest = url.substr(p + 1);
0118 } else {
0119 protocol = "file";
0120 rest = url;
0121 }
0122
0123 return getMaker(protocol);
0124 }
0125
0126 std::unique_ptr<Storage> StorageFactory::open(const std::string &url, int mode ) const {
0127 std::string protocol;
0128 std::string rest;
0129 std::unique_ptr<Storage> ret;
0130 std::unique_ptr<StorageAccount::Stamp> stats;
0131 if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0132 if (m_accounting) {
0133 auto token = StorageAccount::tokenForStorageClassName(protocol);
0134 stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::open));
0135 }
0136 try {
0137 if (auto storage = maker->open(
0138 protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout))) {
0139 if (dynamic_cast<LocalCacheFile *>(storage.get()))
0140 protocol = "local-cache";
0141
0142 if (m_accounting)
0143 ret = std::make_unique<StorageAccountProxy>(protocol, std::move(storage));
0144 else
0145 ret = std::move(storage);
0146
0147 if (stats)
0148 stats->tick();
0149 }
0150 } catch (cms::Exception &err) {
0151 err.addContext("Calling StorageFactory::open()");
0152 err.addAdditionalInfo(err.message());
0153 err.clearMessage();
0154 err << "Failed to open the file '" << url << "'";
0155 throw;
0156 }
0157 }
0158 return ret;
0159 }
0160
0161 void StorageFactory::stagein(const std::string &url) const {
0162 std::string protocol;
0163 std::string rest;
0164
0165 std::unique_ptr<StorageAccount::Stamp> stats;
0166 if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0167 if (m_accounting) {
0168 auto token = StorageAccount::tokenForStorageClassName(protocol);
0169 stats =
0170 std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::stagein));
0171 }
0172 try {
0173 maker->stagein(protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
0174 if (stats)
0175 stats->tick();
0176 } catch (cms::Exception &err) {
0177 edm::LogWarning("StorageFactory::stagein()") << "Failed to stage in file '" << url << "' because:\n"
0178 << err.explainSelf();
0179 }
0180 }
0181 }
0182
0183 bool StorageFactory::check(const std::string &url, IOOffset *size ) const {
0184 std::string protocol;
0185 std::string rest;
0186
0187 bool ret = false;
0188 std::unique_ptr<StorageAccount::Stamp> stats;
0189 if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0190 if (m_accounting) {
0191 auto token = StorageAccount::tokenForStorageClassName(protocol);
0192 stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::check));
0193 }
0194 try {
0195 ret = maker->check(
0196 protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size);
0197 if (stats)
0198 stats->tick();
0199 } catch (cms::Exception &err) {
0200 edm::LogWarning("StorageFactory::check()")
0201 << "Existence or size check for the file '" << url << "' failed because:\n"
0202 << err.explainSelf();
0203 }
0204 }
0205
0206 return ret;
0207 }
0208
0209 std::unique_ptr<Storage> StorageFactory::wrapNonLocalFile(std::unique_ptr<Storage> s,
0210 const std::string &proto,
0211 const std::string &path,
0212 int mode) const {
0213 StorageFactory::CacheHint hint = cacheHint();
0214 if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) {
0215 if (mode & IOFlags::OpenWrite) {
0216
0217 } else if (m_tempdir.empty()) {
0218 edm::LogWarning("StorageFactory") << m_unusableDirWarnings;
0219 } else if ((not path.empty()) and m_lfs.isLocalPath(path)) {
0220
0221 } else {
0222 if (accounting()) {
0223 s = std::make_unique<StorageAccountProxy>(proto, std::move(s));
0224 }
0225 s = std::make_unique<LocalCacheFile>(std::move(s), m_tempdir);
0226 }
0227 }
0228
0229 return s;
0230 }