Utilities/src/CondDBTools.cc

0001 #include "CondCore/Utilities/interface/CondDBTools.h"
0002 #include "CondCore/Utilities/interface/CondDBImport.h"
0003 #include "CondCore/CondDB/interface/ConnectionPool.h"
0004 //
0005 #include <memory>
0006 #include <set>
0007
0008 namespace cond {
0009
0010   namespace persistency {
0011
0012     cond::Hash importPayload(Session& sourceSession,
0013                              const cond::Hash& sourcePayloadId,
0014                              Session& destSession,
0015                              bool reserialize) {
0016       if (reserialize) {
0017         std::pair<std::string, std::shared_ptr<void> > readBackPayload = fetch(sourcePayloadId, sourceSession);
0018         return import(sourceSession, sourcePayloadId, readBackPayload.first, readBackPayload.second.get(), destSession);
0019       } else {
0020         std::string payloadType("");
0021         cond::Binary payloadData;
0022         cond::Binary streamerInfoData;
0023         if (!sourceSession.fetchPayloadData(sourcePayloadId, payloadType, payloadData, streamerInfoData)) {
0024           cond::throwException("Payload with hash" + sourcePayloadId + " has not been found in the source database.",
0025                                "importPayload");
0026         }
0027         boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
0028         return destSession.storePayloadData(payloadType, std::make_pair(payloadData, streamerInfoData), now);
0029       }
0030     }
0031
0032     // comparison functor for iov tuples: Time_t only and Time_t,string
0033     struct IOVComp {
0034       bool operator()(const cond::Time_t& x, const std::pair<cond::Time_t, boost::posix_time::ptime>& y) {
0035         return (x < y.first);
0036       }
0037     };
0038
0039     size_t importIovs(const std::string& sourceTag,
0040                       Session& sourceSession,
0041                       const std::string& destTag,
0042                       Session& destSession,
0043                       cond::Time_t begin,
0044                       cond::Time_t end,
0045                       const std::string& description,
0046                       const std::string& editingNote,
0047                       bool override,
0048                       bool reserialize,
0049                       bool forceInsert) {
0050       persistency::TransactionScope ssc(sourceSession.transaction());
0051       ssc.start();
0052       std::cout << "    Loading source iov..." << std::endl;
0053       persistency::IOVProxy p = sourceSession.readIov(sourceTag);
0054       auto iovs = p.selectAll();
0055       if (iovs.size() == 0) {
0056         std::cout << "    Tag contains 0 iovs." << std::endl;
0057         return 0;
0058       } else {
0059         std::cout << "    Iov size:" << iovs.size() << " timeType:" << p.tagInfo().timeType << " payloadObjectType=\""
0060                   << p.tagInfo().payloadType << "\"" << std::endl;
0061       }
0062       if ((*iovs.begin()).since > begin)
0063         begin = (*iovs.begin()).since;
0064       if (end < begin) {
0065         std::cout << "    No Iov in the selected range." << std::endl;
0066         return 0;
0067       }
0068       persistency::IOVEditor editor;
0069       persistency::TransactionScope dsc(destSession.transaction());
0070       dsc.start(false);
0071       bool exists = false;
0072       if (!destSession.existsDatabase()) {
0073         destSession.createDatabase();
0074       } else {
0075         exists = destSession.existsIov(destTag);
0076       }
0077       persistency::IOVProxy dp;
0078       if (exists) {
0079         dp = destSession.readIov(destTag);
0080         editor = destSession.editIov(destTag);
0081         if (!description.empty())
0082           std::cout << "   INFO. Destination Tag " << destTag
0083                     << " already exists. Provided description will be ignored." << std::endl;
0084         if (editor.timeType() != p.tagInfo().timeType)
0085           throwException("TimeType of the destination tag does not match with the source tag timeType.", "importIovs");
0086         if (editor.payloadType() != p.tagInfo().payloadType)
0087           throwException("PayloadType of the destination tag does not match with the source tag payloadType.",
0088                          "importIovs");
0089       } else {
0090         editor = destSession.createIov(
0091             p.tagInfo().payloadType, destTag, p.tagInfo().timeType, p.tagInfo().synchronizationType);
0092         if (description.empty())
0093           editor.setDescription("Created copying tag " + sourceTag + " from " + sourceSession.connectionString());
0094         else
0095           editor.setDescription(description);
0096       }
0097       size_t niovs = 0;
0098       std::set<cond::Hash> pids;
0099       std::set<cond::Time_t> sinces;
0100       auto iiov = iovs.find(begin);
0101       cond::Time_t newSince = begin;
0102       while (iiov != iovs.end()) {
0103         // skip duplicated sinces
0104         if (sinces.find(newSince) != sinces.end()) {
0105           std::cout << "    WARNING. Skipping duplicated since=" << newSince << std::endl;
0106           continue;
0107         }
0108         // make sure that we import the payload _IN_USE_
0109         auto usedIov = p.getInterval(newSince);
0110         cond::Hash ph = importPayload(sourceSession, usedIov.payloadId, destSession, reserialize);
0111         pids.insert(ph);
0112         bool skip = false;
0113         if (exists) {
0114           // don't insert if the same entry is already there...
0115           auto diovs = dp.selectAll();
0116           auto ie = diovs.find(newSince);
0117           if (ie != diovs.end()) {
0118             if (((*ie).since == newSince) && ((*ie).payloadId == usedIov.payloadId)) {
0119               skip = true;
0120             }
0121           }
0122         }
0123         if (!skip) {
0124           editor.insert(newSince, ph);
0125           sinces.insert(newSince);
0126           niovs++;
0127           if (niovs && (niovs % 1000 == 0))
0128             std::cout << "    Total of iov inserted: " << niovs << " payloads: " << pids.size() << std::endl;
0129         }
0130         iiov++;
0131         if (iiov == iovs.end() || (*iiov).since > end) {
0132           break;
0133         } else {
0134           newSince = (*iiov).since;
0135         }
0136       }
0137       if (exists && override) {
0138         std::cout << "    Adding overlying iovs..." << std::endl;
0139         persistency::IOVProxy dp = destSession.readIov(destTag);
0140         auto diovs = dp.selectRange(begin, end);
0141         std::set<cond::Time_t> extraSinces;
0142         for (const auto& iov : diovs) {
0143           auto siov = p.getInterval(iov.since);
0144           if (siov.since != iov.since) {
0145             if (extraSinces.find(iov.since) == extraSinces.end()) {
0146               editor.insert(iov.since, siov.payloadId);
0147               extraSinces.insert(iov.since);
0148               niovs++;
0149               if (niovs && (niovs % 1000 == 0))
0150                 std::cout << "    Total of iov inserted: " << niovs << " payloads: " << pids.size() << std::endl;
0151             }
0152           }
0153         }
0154       }
0155       std::cout << "    Total of iov inserted: " << niovs << " payloads: " << pids.size() << std::endl;
0156       std::cout << "    Flushing changes..." << std::endl;
0157       editor.flush(editingNote, forceInsert);
0158       dsc.commit();
0159       ssc.commit();
0160       return niovs;
0161     }
0162
0163     bool copyIov(Session& session,
0164                  const std::string& sourceTag,
0165                  const std::string& destTag,
0166                  cond::Time_t sourceSince,
0167                  cond::Time_t destSince,
0168                  const std::string& description) {
0169       persistency::TransactionScope ssc(session.transaction());
0170       ssc.start(false);
0171       std::cout << "    Loading source iov..." << std::endl;
0172       persistency::IOVProxy p = session.readIov(sourceTag);
0173       auto iovs = p.selectAll();
0174       if (iovs.size() == 0) {
0175         std::cout << "    Tag contains 0 iovs." << std::endl;
0176         return false;
0177       } else {
0178         std::cout << "    Iov size:" << iovs.size() << " timeType:" << p.tagInfo().timeType << " payloadObjectType=\""
0179                   << p.tagInfo().payloadType << "\"" << std::endl;
0180       }
0181
0182       auto iiov = iovs.find(sourceSince);
0183       if (iiov == iovs.end()) {
0184         std::cout << "ERROR: No Iov valid found for target time " << sourceSince << std::endl;
0185         return false;
0186       }
0187
0188       persistency::IOVEditor editor;
0189       if (session.existsIov(destTag)) {
0190         if (!description.empty())
0191           std::cout << "   INFO. Destination Tag " << destTag
0192                     << " already exists. Provided description will be ignored." << std::endl;
0193         editor = session.editIov(destTag);
0194         if (editor.timeType() != p.tagInfo().timeType)
0195           throwException("TimeType of the destination tag does not match with the source tag timeType.", "importIovs");
0196         if (editor.payloadType() != p.tagInfo().payloadType)
0197           throwException("PayloadType of the destination tag does not match with the source tag payloadType.",
0198                          "importIovs");
0199       } else {
0200         editor =
0201             session.createIov(p.tagInfo().payloadType, destTag, p.tagInfo().timeType, p.tagInfo().synchronizationType);
0202         if (description.empty())
0203           editor.setDescription("Created copying iovs from tag " + sourceTag);
0204         else
0205           editor.setDescription(description);
0206       }
0207
0208       editor.insert(destSince, (*iiov).payloadId);
0209
0210       std::cout << "    Flushing changes..." << std::endl;
0211       editor.flush();
0212       ssc.commit();
0213       return true;
0214     }
0215
0216   }  // namespace persistency
0217 }  // namespace cond