File indexing completed on 2025-05-19 07:20:22
0001 #include <memory>
0002
0003 #include "Utilities/StorageFactory/interface/StorageFactory.h"
0004 #include "Utilities/StorageFactory/interface/StorageMakerFactory.h"
0005 #include "Utilities/StorageFactory/interface/StorageAccount.h"
0006 #include "Utilities/StorageFactory/interface/StorageAccountProxy.h"
0007 #include "Utilities/StorageFactory/interface/StorageProxyMaker.h"
0008 #include "Utilities/StorageFactory/interface/LocalCacheFile.h"
0009 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0010 #include "FWCore/PluginManager/interface/PluginManager.h"
0011 #include "FWCore/PluginManager/interface/standard.h"
0012 #include "FWCore/Utilities/interface/Exception.h"
0013
0014 using namespace edm::storage;
0015
0016 StorageFactory StorageFactory::s_instance;
0017
0018 StorageFactory::StorageFactory(void)
0019 : m_cacheHint(CACHE_HINT_AUTO_DETECT),
0020 m_readHint(READ_HINT_AUTO),
0021 m_accounting(false),
0022 m_tempfree(defaultMinTempFree()),
0023 m_temppath(defaultTempDir()),
0024 m_timeout(0U),
0025 m_debugLevel(0U) {
0026 setTempDir(m_temppath, m_tempfree);
0027 }
0028
0029 StorageFactory::~StorageFactory(void) {}
0030
0031 const StorageFactory *StorageFactory::get(void) { return &s_instance; }
0032
0033 StorageFactory *StorageFactory::getToModify(void) { return &s_instance; }
0034
0035 std::string StorageFactory::defaultTempDir() { return ".:$TMPDIR"; }
0036
0037 bool StorageFactory::enableAccounting(bool enabled) {
0038 bool old = m_accounting;
0039 m_accounting = enabled;
0040 return old;
0041 }
0042
0043 bool StorageFactory::accounting(void) const { return m_accounting; }
0044
0045 void StorageFactory::setCacheHint(CacheHint value) { m_cacheHint = value; }
0046
0047 StorageFactory::CacheHint StorageFactory::cacheHint(void) const { return m_cacheHint; }
0048
0049 void StorageFactory::setReadHint(ReadHint value) { m_readHint = value; }
0050
0051 StorageFactory::ReadHint StorageFactory::readHint(void) const { return m_readHint; }
0052
0053 void StorageFactory::setTimeout(unsigned int timeout) { m_timeout = timeout; }
0054
0055 unsigned int StorageFactory::timeout(void) const { return m_timeout; }
0056
0057 void StorageFactory::setDebugLevel(unsigned int level) { m_debugLevel = level; }
0058
0059 unsigned int StorageFactory::debugLevel(void) const { return m_debugLevel; }
0060
0061 void StorageFactory::setTempDir(const std::string &s, double minFreeSpace) {
0062 #if 0
0063 std::cerr
0064 << "Considering path '" << s
0065 << "', min free space " << minFreeSpace
0066 << "GB for temp dir" << std::endl;
0067 #endif
0068
0069 size_t begin = 0;
0070 std::vector<std::string> dirs;
0071 dirs.reserve(std::count(s.begin(), s.end(), ':') + 1);
0072
0073 while (true) {
0074 size_t end = s.find(':', begin);
0075 if (end == std::string::npos) {
0076 dirs.push_back(s.substr(begin, end));
0077 break;
0078 } else {
0079 dirs.push_back(s.substr(begin, end - begin));
0080 begin = end + 1;
0081 }
0082 }
0083
0084 m_temppath = s;
0085 m_tempfree = minFreeSpace;
0086 std::tie(m_tempdir, m_unusableDirWarnings) = m_lfs.findCachePath(dirs, minFreeSpace);
0087
0088 #if 0
0089 std::cerr
0090 << "Using '" << m_tempdir << "' for temp dir"
0091 << std::endl;
0092 #endif
0093 }
0094
0095 std::string StorageFactory::tempDir(void) const { return m_tempdir; }
0096
0097 std::string StorageFactory::tempPath(void) const { return m_temppath; }
0098
0099 double StorageFactory::tempMinFree(void) const { return m_tempfree; }
0100
0101 void StorageFactory::setStorageProxyMakers(std::vector<std::unique_ptr<StorageProxyMaker>> makers) {
0102 m_storageProxyMakers_ = std::move(makers);
0103 }
0104
0105 StorageMaker *StorageFactory::getMaker(const std::string &proto) const {
0106 auto itFound = m_makers.find(proto);
0107 if (itFound != m_makers.end()) {
0108 return itFound->second.get();
0109 }
0110 if (!edmplugin::PluginManager::isAvailable()) {
0111 edmplugin::PluginManager::configure(edmplugin::standard::config());
0112 }
0113 std::shared_ptr<StorageMaker> instance{StorageMakerFactory::get()->tryToCreate(proto)};
0114 auto insertResult = m_makers.insert(MakerTable::value_type(proto, instance));
0115
0116
0117 return insertResult.first->second.get();
0118 }
0119
0120 StorageMaker *StorageFactory::getMaker(const std::string &url, std::string &protocol, std::string &rest) const {
0121 size_t p = url.find(':');
0122 if (p != std::string::npos) {
0123 protocol = url.substr(0, p);
0124 rest = url.substr(p + 1);
0125 } else {
0126 protocol = "file";
0127 rest = url;
0128 }
0129
0130 return getMaker(protocol);
0131 }
0132
0133 std::unique_ptr<Storage> StorageFactory::open(const std::string &url, const int mode ) const {
0134 std::string protocol;
0135 std::string rest;
0136 std::unique_ptr<Storage> ret;
0137 std::unique_ptr<StorageAccount::Stamp> stats;
0138 if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0139 if (m_accounting) {
0140 auto token = StorageAccount::tokenForStorageClassName(protocol);
0141 stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::open));
0142 }
0143 try {
0144 ret = maker->open(
0145 protocol, rest, mode, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
0146 if (ret) {
0147
0148
0149 for (auto const &proxyMaker : m_storageProxyMakers_) {
0150 ret = proxyMaker->wrap(url, std::move(ret));
0151 }
0152
0153
0154
0155 if (auto const useLocalFile = maker->usesLocalFile(); useLocalFile != StorageMaker::UseLocalFile::kNo) {
0156 bool useCacheFile;
0157 std::string path;
0158 if (useLocalFile == StorageMaker::UseLocalFile::kCheckFromPath) {
0159 path = rest;
0160 }
0161 std::tie(ret, useCacheFile) = wrapNonLocalFile(std::move(ret), protocol, path, mode);
0162 if (useCacheFile) {
0163 protocol = "local-cache";
0164 }
0165 }
0166
0167 if (m_accounting)
0168 ret = std::make_unique<StorageAccountProxy>(protocol, std::move(ret));
0169
0170 if (stats)
0171 stats->tick();
0172 }
0173 } catch (cms::Exception &err) {
0174 err.addContext("Calling StorageFactory::open()");
0175 err.addAdditionalInfo(err.message());
0176 err.clearMessage();
0177 err << "Failed to open the file '" << url << "'";
0178 throw;
0179 }
0180 }
0181 return ret;
0182 }
0183
0184 void StorageFactory::stagein(const std::string &url) const {
0185 std::string protocol;
0186 std::string rest;
0187
0188 std::unique_ptr<StorageAccount::Stamp> stats;
0189 if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0190 if (m_accounting) {
0191 auto token = StorageAccount::tokenForStorageClassName(protocol);
0192 stats =
0193 std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::stagein));
0194 }
0195 try {
0196 maker->stagein(protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout));
0197 if (stats)
0198 stats->tick();
0199 } catch (cms::Exception &err) {
0200 edm::LogWarning("StorageFactory::stagein()") << "Failed to stage in file '" << url << "' because:\n"
0201 << err.explainSelf();
0202 }
0203 }
0204 }
0205
0206 bool StorageFactory::check(const std::string &url, IOOffset *size ) const {
0207 std::string protocol;
0208 std::string rest;
0209
0210 bool ret = false;
0211 std::unique_ptr<StorageAccount::Stamp> stats;
0212 if (StorageMaker *maker = getMaker(url, protocol, rest)) {
0213 if (m_accounting) {
0214 auto token = StorageAccount::tokenForStorageClassName(protocol);
0215 stats = std::make_unique<StorageAccount::Stamp>(StorageAccount::counter(token, StorageAccount::Operation::check));
0216 }
0217 try {
0218 ret = maker->check(
0219 protocol, rest, StorageMaker::AuxSettings{}.setDebugLevel(m_debugLevel).setTimeout(m_timeout), size);
0220 if (stats)
0221 stats->tick();
0222 } catch (cms::Exception &err) {
0223 edm::LogWarning("StorageFactory::check()")
0224 << "Existence or size check for the file '" << url << "' failed because:\n"
0225 << err.explainSelf();
0226 }
0227 }
0228
0229 return ret;
0230 }
0231
0232 std::tuple<std::unique_ptr<Storage>, bool> StorageFactory::wrapNonLocalFile(std::unique_ptr<Storage> s,
0233 const std::string &proto,
0234 const std::string &path,
0235 const int mode) const {
0236 StorageFactory::CacheHint hint = cacheHint();
0237 bool useCacheFile = false;
0238 if ((hint == StorageFactory::CACHE_HINT_LAZY_DOWNLOAD) || (mode & IOFlags::OpenWrap)) {
0239 if (mode & IOFlags::OpenWrite) {
0240
0241 } else if (m_tempdir.empty()) {
0242 edm::LogWarning("StorageFactory") << m_unusableDirWarnings;
0243 } else if ((not path.empty()) and m_lfs.isLocalPath(path)) {
0244
0245 } else {
0246 if (accounting()) {
0247 s = std::make_unique<StorageAccountProxy>(proto, std::move(s));
0248 }
0249 s = std::make_unique<LocalCacheFile>(std::move(s), m_tempdir);
0250 useCacheFile = true;
0251 }
0252 }
0253
0254 return {std::move(s), useCacheFile};
0255 }