File indexing completed on 2025-03-23 15:57:26
0001 #include "CondCore/CondDB/interface/ConnectionPool.h"
0002 #include "CondCore/CondDB/interface/IOVProxy.h"
0003 #include "CondCore/CondDB/interface/GTProxy.h"
0004
0005 #include "CondCore/Utilities/interface/Utilities.h"
0006 #include <iostream>
0007
0008 #include <chrono>
0009 #include <memory>
0010
0011
0012
0013 class Timer {
0014 public:
0015 Timer(const std::string& nameIn) : name(nameIn) { reset(); }
0016 void reset() {
0017 start = std::chrono::steady_clock::now();
0018 intervals.clear();
0019 intervalNames.clear();
0020 interval("start");
0021 }
0022
0023 void interval(const std::string& intName) {
0024 intervals.push_back(std::chrono::steady_clock::now());
0025 intervalNames.push_back(intName);
0026 }
0027
0028 void fetchInt(size_t sizeIn) {
0029 fetchTime.push_back(std::chrono::steady_clock::now());
0030 fetchNum.push_back(sizeIn);
0031 }
0032 void deserInt(size_t sizeIn) {
0033 deserTime.push_back(std::chrono::steady_clock::now());
0034 deserNum.push_back(sizeIn);
0035 }
0036
0037 void show(std::ostream& os = std::cout) {
0038 showIntervals(os);
0039 showFetchInfo(os);
0040 showDeserInfo(os);
0041 }
0042 void showIntervals(std::ostream& os = std::cout);
0043 void showFetchInfo(std::ostream& os = std::cout);
0044 void showDeserInfo(std::ostream& os = std::cout);
0045
0046 private:
0047 std::string name;
0048
0049 std::chrono::time_point<std::chrono::steady_clock> start;
0050
0051 std::vector<std::chrono::time_point<std::chrono::steady_clock> > intervals;
0052 std::vector<std::string> intervalNames;
0053
0054 std::vector<std::chrono::time_point<std::chrono::steady_clock> > fetchTime;
0055 std::vector<int> fetchNum;
0056
0057 std::vector<std::chrono::time_point<std::chrono::steady_clock> > deserTime;
0058 std::vector<int> deserNum;
0059 };
0060
0061 void Timer::showIntervals(std::ostream& os) {
0062 os << std::endl;
0063 os << "Serialization type: " << name << std::endl;
0064 for (size_t i = 1; i < intervals.size(); i++) {
0065 os << intervalNames[i] << " : "
0066 << std::chrono::duration<double, std::milli>(intervals[i] - intervals[i - 1]).count() << " msec. " << std::endl;
0067 }
0068 os << "\noverall time elapsed"
0069 << " : " << std::chrono::duration<double, std::milli>(intervals[intervals.size() - 1] - intervals[0]).count()
0070 << " msec. " << std::endl;
0071 os << std::endl;
0072 }
0073
0074 void Timer::showFetchInfo(std::ostream& os) {
0075 os << std::endl;
0076 os << "Serialization type: " << name << std::endl;
0077 if (fetchTime.empty()) {
0078 os << "No fetch info available." << std::endl;
0079 return;
0080 }
0081 int totSize = 0;
0082 for (size_t i = 1; i < fetchTime.size(); i++) {
0083 totSize += fetchNum[i];
0084 auto delta = std::chrono::duration<double, std::milli>(fetchTime[i] - fetchTime[i - 1]).count();
0085 os << fetchNum[i] << " : " << delta << " ms (" << float(fetchNum[i]) / (1024. * float(delta)) << " MB/s)"
0086 << std::endl;
0087 }
0088 auto deltaAll = std::chrono::duration<double, std::milli>(fetchTime[fetchTime.size() - 1] - fetchTime[0]).count();
0089 os << "\noverall time for " << totSize << " bytes : " << deltaAll << " ms ("
0090 << float(totSize) / (1024. * float(deltaAll)) << " MB/s)" << std::endl;
0091 os << std::endl;
0092 }
0093
0094 void Timer::showDeserInfo(std::ostream& os) {
0095 os << std::endl;
0096 os << "Serialization type: " << name << std::endl;
0097 if (deserTime.empty()) {
0098 os << "No deserialization info available." << std::endl;
0099 return;
0100 }
0101 int totSize = 0;
0102 for (size_t i = 1; i < deserTime.size(); i++) {
0103 totSize += deserNum[i];
0104 auto delta = std::chrono::duration<double, std::milli>(deserTime[i] - deserTime[i - 1]).count();
0105 os << deserNum[i] << " : " << delta << " ms (" << float(deserNum[i]) / (1024. * float(delta)) << " MB/s)"
0106 << std::endl;
0107 }
0108 auto deltaAll = std::chrono::duration<double, std::milli>(deserTime[deserTime.size() - 1] - deserTime[0]).count();
0109 os << "\noverall time for " << totSize << " bytes : " << deltaAll << " ms ("
0110 << float(totSize) / (1024. * float(deltaAll)) << " MB/s)" << std::endl;
0111 os << std::endl;
0112 }
0113
0114
0115
0116 namespace cond {
0117
0118 using namespace persistency;
0119
0120 class UntypedPayloadProxy {
0121 public:
0122 explicit UntypedPayloadProxy(Session& session);
0123
0124 UntypedPayloadProxy(const UntypedPayloadProxy& rhs);
0125
0126 UntypedPayloadProxy& operator=(const UntypedPayloadProxy& rhs);
0127
0128 void load(const std::string& tag);
0129
0130 void reset();
0131
0132 TimeType timeType() const;
0133 std::string tag() const;
0134
0135 bool get(cond::Time_t targetTime, bool debug);
0136
0137 size_t numberOfQueries() const;
0138
0139 const std::vector<std::string>& history() const;
0140
0141 private:
0142 struct pimpl {
0143 cond::Iov_t current;
0144 std::vector<std::string> history;
0145 };
0146
0147 Session m_session;
0148 IOVProxy m_iov;
0149 std::shared_ptr<pimpl> m_data;
0150 };
0151
0152 class TestGTLoad : public cond::Utilities {
0153 public:
0154 TestGTLoad();
0155 int execute() override;
0156 };
0157 }
0158
0159 cond::UntypedPayloadProxy::UntypedPayloadProxy(Session& session) : m_session(session) {
0160 m_data = std::make_shared<pimpl>();
0161 m_data->current.clear();
0162 }
0163
0164 cond::UntypedPayloadProxy::UntypedPayloadProxy(const UntypedPayloadProxy& rhs)
0165 : m_session(rhs.m_session), m_iov(rhs.m_iov), m_data(rhs.m_data) {}
0166
0167 cond::UntypedPayloadProxy& cond::UntypedPayloadProxy::operator=(const cond::UntypedPayloadProxy& rhs) {
0168 m_session = rhs.m_session;
0169 m_iov = rhs.m_iov;
0170 m_data = rhs.m_data;
0171 return *this;
0172 }
0173
0174 void cond::UntypedPayloadProxy::load(const std::string& tag) {
0175 m_data->current.clear();
0176 m_session.transaction().start();
0177 m_iov = m_session.readIov(tag);
0178 m_session.transaction().commit();
0179 }
0180
0181 void cond::UntypedPayloadProxy::reset() {
0182 m_iov.reset();
0183 m_data->current.clear();
0184 }
0185
0186 std::string cond::UntypedPayloadProxy::tag() const { return m_iov.tagInfo().name; }
0187
0188 cond::TimeType cond::UntypedPayloadProxy::timeType() const { return m_iov.tagInfo().timeType; }
0189
0190 bool cond::UntypedPayloadProxy::get(cond::Time_t targetTime, bool debug) {
0191 bool loaded = false;
0192 std::stringstream event;
0193
0194
0195 if (targetTime < m_data->current.since || targetTime >= m_data->current.till) {
0196
0197 if (debug)
0198 std::cout << " Searching tag " << m_iov.tagInfo().name << " for a valid payload for time=" << targetTime
0199 << std::endl;
0200 m_session.transaction().start();
0201
0202 auto iovs = m_iov.selectAll();
0203 auto iIov = iovs.find(targetTime);
0204 if (iIov == iovs.end())
0205 cond::throwException(std::string("Tag ") + m_iov.tagInfo().name +
0206 ": No iov available for the target time:" + std::to_string(targetTime),
0207 "UntypedPayloadProxy::get");
0208 m_data->current = *iIov;
0209 event << "For target time " << targetTime << " got a valid since:" << m_data->current.since << " from group ["
0210 << m_iov.loadedGroup().first << " - " << m_iov.loadedGroup().second << "]";
0211
0212 std::string payloadType("");
0213 Binary data;
0214 Binary streamerInfo;
0215 loaded = m_session.fetchPayloadData(m_data->current.payloadId, payloadType, data, streamerInfo);
0216 m_session.transaction().commit();
0217 if (!loaded) {
0218 std::cout << "ERROR: payload with id " << m_data->current.payloadId << " could not be loaded." << std::endl;
0219 } else {
0220 std::stringstream sz;
0221 sz << data.size();
0222 if (debug)
0223 std::cout << "Loaded payload of type \"" << payloadType << "\" (" << sz.str() << " bytes)" << std::endl;
0224 }
0225 } else {
0226 event << "Target time " << targetTime << " is within range for payloads available in cache: ["
0227 << m_data->current.since << " - " << m_data->current.till << "]";
0228 }
0229 m_data->history.push_back(event.str());
0230 return loaded;
0231 }
0232
0233 size_t cond::UntypedPayloadProxy::numberOfQueries() const { return m_iov.numberOfQueries(); }
0234
0235 const std::vector<std::string>& cond::UntypedPayloadProxy::history() const { return m_data->history; }
0236
0237 cond::TestGTLoad::TestGTLoad() : Utilities("conddb_test_gt_load") {
0238 addConnectOption("connect", "c", "database connection string(required)");
0239 addAuthenticationOptions();
0240 addOption<size_t>("iterations", "n", "number of iterations (default=10)");
0241 addOption<Time_t>("start_run", "R", "start for Run iterations (default=150005)");
0242 addOption<Time_t>("step_run", "r", "step for Run iterations (default=1000)");
0243 addOption<Time_t>("start_ts", "T", "start for TS iterations (default=5800013687234232320)");
0244 addOption<Time_t>("step_ts", "t", "step for TS iterations (default=10000000000000)");
0245 addOption<Time_t>("start_lumi", "L", "start for Lumi iterations (default=908900979179966)");
0246 addOption<Time_t>("step_lumi", "l", "step for Lumi iterations (default=10000000000)");
0247 addOption<std::string>("globaltag", "g", "global tag (required)");
0248 addOption<bool>("verbose", "v", "verbose print out (optional)");
0249 }
0250
0251 int cond::TestGTLoad::execute() {
0252 std::string gtag = getOptionValue<std::string>("globaltag");
0253 std::string connect = getOptionValue<std::string>("connect");
0254 bool verbose = hasOptionValue("verbose");
0255 size_t n = 1;
0256 if (hasOptionValue("iterations"))
0257 n = getOptionValue<size_t>("iterations");
0258 Time_t startRun = 150005;
0259 if (hasOptionValue("start_run"))
0260 startRun = getOptionValue<Time_t>("start_run");
0261 Time_t stepRun = 1000;
0262 if (hasOptionValue("step_run"))
0263 stepRun = getOptionValue<Time_t>("step_run");
0264 Time_t startTs = 5800013687234232320;
0265 if (hasOptionValue("start_ts"))
0266 startTs = getOptionValue<Time_t>("start_ts");
0267 Time_t stepTs = 10000000000000;
0268 if (hasOptionValue("step_ts"))
0269 stepTs = getOptionValue<Time_t>("step_ts");
0270 Time_t startLumi = 908900979179966;
0271 if (hasOptionValue("start_lumi"))
0272 startLumi = getOptionValue<Time_t>("start_lumi");
0273 Time_t stepLumi = 10000000000;
0274 if (hasOptionValue("step_lumi"))
0275 stepLumi = getOptionValue<Time_t>("step_lumi");
0276
0277 Timer timex("condDBv1");
0278
0279 ConnectionPool connPool;
0280 if (hasDebug())
0281 connPool.setMessageVerbosity(coral::Debug);
0282 connPool.configure();
0283 Session session = connPool.createSession(connect);
0284 session.transaction().start();
0285
0286 std::cout << "Loading Global Tag " << gtag << std::endl;
0287 GTProxy gt = session.readGlobalTag(gtag);
0288
0289 session.transaction().commit();
0290
0291 std::cout << "Loading " << gt.size() << " tags..." << std::endl;
0292 std::vector<UntypedPayloadProxy> proxies;
0293 std::map<std::string, size_t> requests;
0294 for (const auto& t : gt) {
0295 std::pair<std::string, std::string> tagParams = parseTag(t.tagName());
0296 std::string tagConnStr = connect;
0297 Session tagSession = session;
0298 if (!tagParams.second.empty()) {
0299 tagConnStr = tagParams.second;
0300 tagSession = connPool.createSession(tagConnStr);
0301 }
0302 UntypedPayloadProxy p(tagSession);
0303 try {
0304 p.load(tagParams.first);
0305 proxies.push_back(p);
0306 requests.insert(std::make_pair(tagParams.first, 0));
0307 } catch (const cond::Exception& e) {
0308 std::cout << "ERROR: " << e.what() << std::endl;
0309 }
0310 }
0311 std::cout << proxies.size() << " tags successfully loaded." << std::endl;
0312
0313 std::cout << "Iterating on " << n << " IOV request(s)..." << std::endl;
0314
0315 for (size_t i = 0; i < n; i++) {
0316 Time_t run = startRun + i * stepRun;
0317 Time_t lumi = startLumi + i * stepLumi;
0318 Time_t ts = startTs + i * stepTs;
0319 for (auto p : proxies) {
0320 bool loaded = false;
0321 time::TimeType ttype = p.timeType();
0322 auto r = requests.find(p.tag());
0323 if (r != requests.end()) {
0324 try {
0325 if (ttype == runnumber) {
0326 p.get(run, hasDebug());
0327 r->second++;
0328 } else if (ttype == lumiid) {
0329 p.get(lumi, hasDebug());
0330 r->second++;
0331 } else if (ttype == timestamp) {
0332 p.get(ts, hasDebug());
0333 r->second++;
0334 } else {
0335 std::cout << "WARNING: iov request on tag " << p.tag() << " (timeType=" << time::timeTypeName(p.timeType())
0336 << ") has been skipped." << std::endl;
0337 }
0338 } catch (const cond::Exception& e) {
0339 std::cout << "ERROR:" << e.what() << std::endl;
0340 }
0341 }
0342 }
0343 }
0344
0345 timex.interval("iterations done");
0346 timex.showIntervals();
0347
0348 std::cout << std::endl;
0349 std::cout << "*** End of job." << std::endl;
0350 std::cout << "*** GT: " << gtag << " Tags:" << gt.size() << " Loaded:" << proxies.size() << std::endl;
0351 std::cout << std::endl;
0352 if (verbose) {
0353 for (const auto& p : proxies) {
0354 auto r = requests.find(p.tag());
0355 if (r != requests.end()) {
0356 std::cout << "*** Tag: " << p.tag() << " Requests processed:" << r->second << " Queries:" << p.numberOfQueries()
0357 << std::endl;
0358 if (verbose) {
0359 const std::vector<std::string>& hist = p.history();
0360 for (const auto& e : p.history())
0361 std::cout << " " << e << std::endl;
0362 }
0363 }
0364 }
0365 }
0366
0367 return 0;
0368 }
0369
0370 int main(int argc, char** argv) {
0371 cond::TestGTLoad test;
0372 return test.run(argc, argv);
0373 }
