Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2025-03-23 15:57:26

0001 #include "CondCore/CondDB/interface/ConnectionPool.h"
0002 #include "CondCore/CondDB/interface/IOVProxy.h"
0003 #include "CondCore/CondDB/interface/GTProxy.h"
0004 
0005 #include "CondCore/Utilities/interface/Utilities.h"
0006 #include <iostream>
0007 
0008 #include <chrono>
0009 #include <memory>
0010 
0011 // ================================================================================
0012 
0013 class Timer {
0014 public:
0015   Timer(const std::string& nameIn) : name(nameIn) { reset(); }
0016   void reset() {
0017     start = std::chrono::steady_clock::now();
0018     intervals.clear();
0019     intervalNames.clear();
0020     interval("start");
0021   }
0022 
0023   void interval(const std::string& intName) {
0024     intervals.push_back(std::chrono::steady_clock::now());
0025     intervalNames.push_back(intName);
0026   }
0027 
0028   void fetchInt(size_t sizeIn) {
0029     fetchTime.push_back(std::chrono::steady_clock::now());
0030     fetchNum.push_back(sizeIn);
0031   }
0032   void deserInt(size_t sizeIn) {
0033     deserTime.push_back(std::chrono::steady_clock::now());
0034     deserNum.push_back(sizeIn);
0035   }
0036 
0037   void show(std::ostream& os = std::cout) {
0038     showIntervals(os);
0039     showFetchInfo(os);
0040     showDeserInfo(os);
0041   }
0042   void showIntervals(std::ostream& os = std::cout);
0043   void showFetchInfo(std::ostream& os = std::cout);
0044   void showDeserInfo(std::ostream& os = std::cout);
0045 
0046 private:
0047   std::string name;
0048 
0049   std::chrono::time_point<std::chrono::steady_clock> start;
0050 
0051   std::vector<std::chrono::time_point<std::chrono::steady_clock> > intervals;
0052   std::vector<std::string> intervalNames;
0053 
0054   std::vector<std::chrono::time_point<std::chrono::steady_clock> > fetchTime;
0055   std::vector<int> fetchNum;
0056 
0057   std::vector<std::chrono::time_point<std::chrono::steady_clock> > deserTime;
0058   std::vector<int> deserNum;
0059 };
0060 
0061 void Timer::showIntervals(std::ostream& os) {
0062   os << std::endl;
0063   os << "Serialization type: " << name << std::endl;
0064   for (size_t i = 1; i < intervals.size(); i++) {
0065     os << intervalNames[i] << " : "
0066        << std::chrono::duration<double, std::milli>(intervals[i] - intervals[i - 1]).count() << " msec. " << std::endl;
0067   }
0068   os << "\noverall time elapsed"
0069      << " : " << std::chrono::duration<double, std::milli>(intervals[intervals.size() - 1] - intervals[0]).count()
0070      << " msec. " << std::endl;
0071   os << std::endl;
0072 }
0073 
0074 void Timer::showFetchInfo(std::ostream& os) {
0075   os << std::endl;
0076   os << "Serialization type: " << name << std::endl;
0077   if (fetchTime.empty()) {
0078     os << "No fetch info available." << std::endl;
0079     return;
0080   }
0081   int totSize = 0;
0082   for (size_t i = 1; i < fetchTime.size(); i++) {
0083     totSize += fetchNum[i];
0084     auto delta = std::chrono::duration<double, std::milli>(fetchTime[i] - fetchTime[i - 1]).count();
0085     os << fetchNum[i] << " : " << delta << " ms (" << float(fetchNum[i]) / (1024. * float(delta)) << " MB/s)"
0086        << std::endl;
0087   }
0088   auto deltaAll = std::chrono::duration<double, std::milli>(fetchTime[fetchTime.size() - 1] - fetchTime[0]).count();
0089   os << "\noverall time for " << totSize << " bytes : " << deltaAll << " ms ("
0090      << float(totSize) / (1024. * float(deltaAll)) << " MB/s)" << std::endl;
0091   os << std::endl;
0092 }
0093 
0094 void Timer::showDeserInfo(std::ostream& os) {
0095   os << std::endl;
0096   os << "Serialization type: " << name << std::endl;
0097   if (deserTime.empty()) {
0098     os << "No deserialization info available." << std::endl;
0099     return;
0100   }
0101   int totSize = 0;
0102   for (size_t i = 1; i < deserTime.size(); i++) {
0103     totSize += deserNum[i];
0104     auto delta = std::chrono::duration<double, std::milli>(deserTime[i] - deserTime[i - 1]).count();
0105     os << deserNum[i] << " : " << delta << " ms (" << float(deserNum[i]) / (1024. * float(delta)) << " MB/s)"
0106        << std::endl;
0107   }
0108   auto deltaAll = std::chrono::duration<double, std::milli>(deserTime[deserTime.size() - 1] - deserTime[0]).count();
0109   os << "\noverall time for " << totSize << " bytes : " << deltaAll << " ms ("
0110      << float(totSize) / (1024. * float(deltaAll)) << " MB/s)" << std::endl;
0111   os << std::endl;
0112 }
0113 
0114 // ================================================================================
0115 
0116 namespace cond {
0117 
0118   using namespace persistency;
0119 
0120   class UntypedPayloadProxy {
0121   public:
0122     explicit UntypedPayloadProxy(Session& session);
0123 
0124     UntypedPayloadProxy(const UntypedPayloadProxy& rhs);
0125 
0126     UntypedPayloadProxy& operator=(const UntypedPayloadProxy& rhs);
0127 
0128     void load(const std::string& tag);
0129 
0130     void reset();
0131 
0132     TimeType timeType() const;
0133     std::string tag() const;
0134 
0135     bool get(cond::Time_t targetTime, bool debug);
0136 
0137     size_t numberOfQueries() const;
0138 
0139     const std::vector<std::string>& history() const;
0140 
0141   private:
0142     struct pimpl {
0143       cond::Iov_t current;
0144       std::vector<std::string> history;
0145     };
0146 
0147     Session m_session;
0148     IOVProxy m_iov;
0149     std::shared_ptr<pimpl> m_data;
0150   };
0151 
0152   class TestGTLoad : public cond::Utilities {
0153   public:
0154     TestGTLoad();
0155     int execute() override;
0156   };
0157 }  // namespace cond
0158 
0159 cond::UntypedPayloadProxy::UntypedPayloadProxy(Session& session) : m_session(session) {
0160   m_data = std::make_shared<pimpl>();
0161   m_data->current.clear();
0162 }
0163 
0164 cond::UntypedPayloadProxy::UntypedPayloadProxy(const UntypedPayloadProxy& rhs)
0165     : m_session(rhs.m_session), m_iov(rhs.m_iov), m_data(rhs.m_data) {}
0166 
0167 cond::UntypedPayloadProxy& cond::UntypedPayloadProxy::operator=(const cond::UntypedPayloadProxy& rhs) {
0168   m_session = rhs.m_session;
0169   m_iov = rhs.m_iov;
0170   m_data = rhs.m_data;
0171   return *this;
0172 }
0173 
0174 void cond::UntypedPayloadProxy::load(const std::string& tag) {
0175   m_data->current.clear();
0176   m_session.transaction().start();
0177   m_iov = m_session.readIov(tag);
0178   m_session.transaction().commit();
0179 }
0180 
0181 void cond::UntypedPayloadProxy::reset() {
0182   m_iov.reset();
0183   m_data->current.clear();
0184 }
0185 
0186 std::string cond::UntypedPayloadProxy::tag() const { return m_iov.tagInfo().name; }
0187 
0188 cond::TimeType cond::UntypedPayloadProxy::timeType() const { return m_iov.tagInfo().timeType; }
0189 
0190 bool cond::UntypedPayloadProxy::get(cond::Time_t targetTime, bool debug) {
0191   bool loaded = false;
0192   std::stringstream event;
0193 
0194   //  check if the current iov loaded is the good one...
0195   if (targetTime < m_data->current.since || targetTime >= m_data->current.till) {
0196     // a new payload is required!
0197     if (debug)
0198       std::cout << " Searching tag " << m_iov.tagInfo().name << " for a valid payload for time=" << targetTime
0199                 << std::endl;
0200     m_session.transaction().start();
0201 
0202     auto iovs = m_iov.selectAll();
0203     auto iIov = iovs.find(targetTime);
0204     if (iIov == iovs.end())
0205       cond::throwException(std::string("Tag ") + m_iov.tagInfo().name +
0206                                ": No iov available for the target time:" + std::to_string(targetTime),
0207                            "UntypedPayloadProxy::get");
0208     m_data->current = *iIov;
0209     event << "For target time " << targetTime << " got a valid since:" << m_data->current.since << " from group ["
0210           << m_iov.loadedGroup().first << " - " << m_iov.loadedGroup().second << "]";
0211 
0212     std::string payloadType("");
0213     Binary data;
0214     Binary streamerInfo;
0215     loaded = m_session.fetchPayloadData(m_data->current.payloadId, payloadType, data, streamerInfo);
0216     m_session.transaction().commit();
0217     if (!loaded) {
0218       std::cout << "ERROR: payload with id " << m_data->current.payloadId << " could not be loaded." << std::endl;
0219     } else {
0220       std::stringstream sz;
0221       sz << data.size();
0222       if (debug)
0223         std::cout << "Loaded payload of type \"" << payloadType << "\" (" << sz.str() << " bytes)" << std::endl;
0224     }
0225   } else {
0226     event << "Target time " << targetTime << " is within range for payloads available in cache: ["
0227           << m_data->current.since << " - " << m_data->current.till << "]";
0228   }
0229   m_data->history.push_back(event.str());
0230   return loaded;
0231 }
0232 
0233 size_t cond::UntypedPayloadProxy::numberOfQueries() const { return m_iov.numberOfQueries(); }
0234 
0235 const std::vector<std::string>& cond::UntypedPayloadProxy::history() const { return m_data->history; }
0236 
0237 cond::TestGTLoad::TestGTLoad() : Utilities("conddb_test_gt_load") {
0238   addConnectOption("connect", "c", "database connection string(required)");
0239   addAuthenticationOptions();
0240   addOption<size_t>("iterations", "n", "number of iterations (default=10)");
0241   addOption<Time_t>("start_run", "R", "start for Run iterations (default=150005)");
0242   addOption<Time_t>("step_run", "r", "step for Run iterations (default=1000)");
0243   addOption<Time_t>("start_ts", "T", "start for TS iterations (default=5800013687234232320)");
0244   addOption<Time_t>("step_ts", "t", "step for TS iterations (default=10000000000000)");
0245   addOption<Time_t>("start_lumi", "L", "start for Lumi iterations (default=908900979179966)");
0246   addOption<Time_t>("step_lumi", "l", "step for Lumi iterations (default=10000000000)");
0247   addOption<std::string>("globaltag", "g", "global tag (required)");
0248   addOption<bool>("verbose", "v", "verbose print out (optional)");
0249 }
0250 
0251 int cond::TestGTLoad::execute() {
0252   std::string gtag = getOptionValue<std::string>("globaltag");
0253   std::string connect = getOptionValue<std::string>("connect");
0254   bool verbose = hasOptionValue("verbose");
0255   size_t n = 1;
0256   if (hasOptionValue("iterations"))
0257     n = getOptionValue<size_t>("iterations");
0258   Time_t startRun = 150005;
0259   if (hasOptionValue("start_run"))
0260     startRun = getOptionValue<Time_t>("start_run");
0261   Time_t stepRun = 1000;
0262   if (hasOptionValue("step_run"))
0263     stepRun = getOptionValue<Time_t>("step_run");
0264   Time_t startTs = 5800013687234232320;
0265   if (hasOptionValue("start_ts"))
0266     startTs = getOptionValue<Time_t>("start_ts");
0267   Time_t stepTs = 10000000000000;
0268   if (hasOptionValue("step_ts"))
0269     stepTs = getOptionValue<Time_t>("step_ts");
0270   Time_t startLumi = 908900979179966;
0271   if (hasOptionValue("start_lumi"))
0272     startLumi = getOptionValue<Time_t>("start_lumi");
0273   Time_t stepLumi = 10000000000;
0274   if (hasOptionValue("step_lumi"))
0275     stepLumi = getOptionValue<Time_t>("step_lumi");
0276 
0277   Timer timex("condDBv1");
0278 
0279   ConnectionPool connPool;
0280   if (hasDebug())
0281     connPool.setMessageVerbosity(coral::Debug);
0282   connPool.configure();
0283   Session session = connPool.createSession(connect);
0284   session.transaction().start();
0285 
0286   std::cout << "Loading Global Tag " << gtag << std::endl;
0287   GTProxy gt = session.readGlobalTag(gtag);
0288 
0289   session.transaction().commit();
0290 
0291   std::cout << "Loading " << gt.size() << " tags..." << std::endl;
0292   std::vector<UntypedPayloadProxy> proxies;
0293   std::map<std::string, size_t> requests;
0294   for (const auto& t : gt) {
0295     std::pair<std::string, std::string> tagParams = parseTag(t.tagName());
0296     std::string tagConnStr = connect;
0297     Session tagSession = session;
0298     if (!tagParams.second.empty()) {
0299       tagConnStr = tagParams.second;
0300       tagSession = connPool.createSession(tagConnStr);
0301     }
0302     UntypedPayloadProxy p(tagSession);
0303     try {
0304       p.load(tagParams.first);
0305       proxies.push_back(p);
0306       requests.insert(std::make_pair(tagParams.first, 0));
0307     } catch (const cond::Exception& e) {
0308       std::cout << "ERROR: " << e.what() << std::endl;
0309     }
0310   }
0311   std::cout << proxies.size() << " tags successfully loaded." << std::endl;
0312 
0313   std::cout << "Iterating on " << n << " IOV request(s)..." << std::endl;
0314 
0315   for (size_t i = 0; i < n; i++) {
0316     Time_t run = startRun + i * stepRun;
0317     Time_t lumi = startLumi + i * stepLumi;
0318     Time_t ts = startTs + i * stepTs;
0319     for (auto p : proxies) {
0320       bool loaded = false;
0321       time::TimeType ttype = p.timeType();
0322       auto r = requests.find(p.tag());
0323       if (r != requests.end()) {
0324         try {
0325           if (ttype == runnumber) {
0326             p.get(run, hasDebug());
0327             r->second++;
0328           } else if (ttype == lumiid) {
0329             p.get(lumi, hasDebug());
0330             r->second++;
0331           } else if (ttype == timestamp) {
0332             p.get(ts, hasDebug());
0333             r->second++;
0334           } else {
0335             std::cout << "WARNING: iov request on tag " << p.tag() << " (timeType=" << time::timeTypeName(p.timeType())
0336                       << ") has been skipped." << std::endl;
0337           }
0338         } catch (const cond::Exception& e) {
0339           std::cout << "ERROR:" << e.what() << std::endl;
0340         }
0341       }
0342     }
0343   }
0344 
0345   timex.interval("iterations done");
0346   timex.showIntervals();
0347 
0348   std::cout << std::endl;
0349   std::cout << "*** End of job." << std::endl;
0350   std::cout << "*** GT: " << gtag << " Tags:" << gt.size() << " Loaded:" << proxies.size() << std::endl;
0351   std::cout << std::endl;
0352   if (verbose) {
0353     for (const auto& p : proxies) {
0354       auto r = requests.find(p.tag());
0355       if (r != requests.end()) {
0356         std::cout << "*** Tag: " << p.tag() << " Requests processed:" << r->second << " Queries:" << p.numberOfQueries()
0357                   << std::endl;
0358         if (verbose) {
0359           const std::vector<std::string>& hist = p.history();
0360           for (const auto& e : p.history())
0361             std::cout << "    " << e << std::endl;
0362         }
0363       }
0364     }
0365   }
0366 
0367   return 0;
0368 }
0369 
0370 int main(int argc, char** argv) {
0371   cond::TestGTLoad test;
0372   return test.run(argc, argv);
0373 }