SiStripO2O/plugins/SiStripPayloadHandler.cc

0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/ServiceRegistry/interface/Service.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "Utilities/OpenSSL/interface/openssl_init.h"
0007
0008 #include <iostream>
0009 #include <sstream>
0010 #include <typeinfo>
0011
0012 #include "CondCore/CondDB/interface/ConnectionPool.h"
0013
0014 #include "RelationalAccess/ITransaction.h"
0015 #include "RelationalAccess/ISessionProxy.h"
0016 #include "RelationalAccess/ISchema.h"
0017 #include "RelationalAccess/ITable.h"
0018 #include "RelationalAccess/IQuery.h"
0019 #include "RelationalAccess/ICursor.h"
0020 #include "RelationalAccess/ITableDataEditor.h"
0021 #include "RelationalAccess/TableDescription.h"
0022 #include "CoralBase/Attribute.h"
0023 #include "CoralBase/AttributeList.h"
0024 #include "CoralBase/TimeStamp.h"
0025
0026 #include "OnlineDB/SiStripESSources/interface/SiStripCondObjBuilderFromDb.h"
0027
0028 template <typename SiStripPayload>
0029 class SiStripPayloadHandler : public edm::one::EDAnalyzer<> {
0030 public:
0031   explicit SiStripPayloadHandler(const edm::ParameterSet& iConfig);
0032   ~SiStripPayloadHandler() override;
0033   void analyze(const edm::Event& evt, const edm::EventSetup& evtSetup) override;
0034   void endJob() override;
0035
0036 private:
0037   std::string makeConfigHash();
0038   std::string queryConfigMap(std::string configHash);
0039   void updateConfigMap(std::string configHash, std::string payloadHash);
0040
0041 private:
0042   cond::persistency::ConnectionPool m_connectionPool;
0043   std::string m_configMapDb;
0044   std::string m_cfgMapSchemaName;
0045   std::string m_cfgMapTableName;
0046   std::string m_condDb;
0047   std::string m_cfgMapDbFile;
0048   std::string m_localCondDbFile;
0049   std::string m_targetTag;
0050   cond::Time_t m_since;
0051
0052   std::string p_type;
0053   std::string p_cfgstr;
0054 };
0055
0056 template <typename SiStripPayload>
0057 SiStripPayloadHandler<SiStripPayload>::SiStripPayloadHandler(const edm::ParameterSet& iConfig)
0058     : m_connectionPool(),
0059       m_cfgMapSchemaName(iConfig.getUntrackedParameter<std::string>("cfgMapSchemaName", "CMS_COND_O2O")),
0060       m_cfgMapTableName(iConfig.getUntrackedParameter<std::string>("cfgMapTableName", "STRIP_CONFIG_TO_PAYLOAD_MAP")),
0061       m_condDb(iConfig.getParameter<std::string>("conditionDatabase")),
0062       m_localCondDbFile(iConfig.getParameter<std::string>("condDbFile")),
0063       m_targetTag(iConfig.getParameter<std::string>("targetTag")),
0064       m_since(iConfig.getParameter<uint32_t>("since")),
0065       p_type(cond::demangledName(typeid(SiStripPayload))) {
0066   edm::Service<SiStripCondObjBuilderFromDb> condObjBuilder;
0067   p_cfgstr = condObjBuilder->getConfigString(typeid(SiStripPayload));
0068   if (iConfig.exists("configMapDatabase"))
0069     m_configMapDb = iConfig.getParameter<std::string>("configMapDatabase");
0070   if (iConfig.exists("cfgMapDbFile"))
0071     m_cfgMapDbFile = iConfig.getParameter<std::string>("cfgMapDbFile");
0072   m_connectionPool.setParameters(iConfig.getParameter<edm::ParameterSet>("DBParameters"));
0073   m_connectionPool.configure();
0074 }
0075
0076 template <typename SiStripPayload>
0077 SiStripPayloadHandler<SiStripPayload>::~SiStripPayloadHandler() = default;
0078
0079 template <typename SiStripPayload>
0080 void SiStripPayloadHandler<SiStripPayload>::analyze(const edm::Event& evt, const edm::EventSetup& evtSetup) {
0081   // some extra work: Getting the payload hash of the last IOV from condDb
0082   // Will be compared with the new payload and reported
0083   cond::persistency::Session condDbSession = m_connectionPool.createSession(m_condDb);
0084   condDbSession.transaction().start(true);
0085   cond::persistency::IOVProxy iovProxy = condDbSession.readIov(m_targetTag);
0086   cond::Hash last_hash = iovProxy.getLast().payloadId;
0087
0088   // that's the final goal: obtain the payload to store into the sqlite file for the upload into the DB
0089   std::shared_ptr<SiStripPayload> payloadToUpload;
0090   // first compute the hash of the configuration
0091   std::string configHash = makeConfigHash();
0092   // query the configMap DB to find the corresponding payload hash
0093   std::string mappedPayloadHash = queryConfigMap(configHash);
0094   bool mapUpToDate = false;
0095
0096   if (!mappedPayloadHash.empty()) {
0097     // the payload has been found ( fast O2O case )
0098     // copy the payload from condtition database
0099     mapUpToDate = true;
0100     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0101                                           << "Try retrieving payload from " << m_condDb;
0102     payloadToUpload = condDbSession.fetchPayload<SiStripPayload>(mappedPayloadHash);
0103     std::cout << "@@@[FastO2O:true]@@@" << std::endl;
0104     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0105                                           << " ... Payload is copied from offline condition database.";
0106   } else {
0107     // start the long O2O...
0108     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0109                                           << "NO mapping payload hash found. Will run the long O2O. ";
0110     SiStripPayload* obj = nullptr;
0111     edm::Service<SiStripCondObjBuilderFromDb> condObjBuilder;
0112     if (typeid(SiStripPayload) == typeid(SiStripApvGain)) {
0113       // special treatment for ApvGain : provide last payload in DB
0114       condObjBuilder->setLastIovGain(condDbSession.fetchPayload<SiStripApvGain>(last_hash));
0115     }
0116     condObjBuilder->getValue(obj);
0117     payloadToUpload = std::shared_ptr<SiStripPayload>(obj);
0118     std::cout << "@@@[FastO2O:false]@@@" << std::endl;
0119     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0120                                           << " ... New payload has been created.";
0121   }
0122   condDbSession.transaction().commit();
0123
0124   // write payload and iov in the local file
0125   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0126                                         << "Write payload to local sqlite file: " << m_localCondDbFile;
0127   cond::persistency::Session localFileSession = m_connectionPool.createSession(m_localCondDbFile, true);
0128   localFileSession.transaction().start(false);
0129   // write the payload
0130   cond::Hash thePayloadHash = localFileSession.storePayload<SiStripPayload>(*payloadToUpload);
0131   cond::persistency::IOVEditor iovEditor = localFileSession.createIov<SiStripPayload>(m_targetTag, cond::runnumber);
0132   iovEditor.setDescription("New IOV");
0133   // inserting the iov
0134   iovEditor.insert(m_since, thePayloadHash);
0135   iovEditor.flush();
0136   localFileSession.transaction().commit();
0137   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0138                                         << "Payload " << thePayloadHash << " inserted to sqlite with IOV " << m_since;
0139
0140   // last step, update the configMap if required
0141   if (!mapUpToDate) {
0142     updateConfigMap(configHash, thePayloadHash);
0143   }
0144
0145   // finish the extra work: Compare the new payload with last IOV
0146   if (last_hash == thePayloadHash) {
0147     std::cout << "@@@[PayloadChange:false]@@@" << last_hash << std::endl;
0148   } else {
0149     std::cout << "@@@[PayloadChange:true]@@@" << last_hash << " -> " << thePayloadHash << std::endl;
0150   }
0151 }
0152
0153 template <typename SiStripPayload>
0154 void SiStripPayloadHandler<SiStripPayload>::endJob() {}
0155
0156 template <typename SiStripPayload>
0157 std::string SiStripPayloadHandler<SiStripPayload>::makeConfigHash() {
0158   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0159                                         << "Convert config string to SHA-1 hash for " << p_type
0160                                         << "\n... config: " << p_cfgstr;
0161
0162   // calcuate SHA-1 hash using openssl
0163   // adapted from cond::persistency::makeHash() in CondCore/CondDB/src/IOVSchema.cc
0164   cms::openssl_init();
0165   EVP_MD_CTX* mdctx = EVP_MD_CTX_new();
0166   const EVP_MD* md = EVP_get_digestbyname("SHA1");
0167   if (!EVP_DigestInit_ex(mdctx, md, nullptr)) {
0168     throw cms::Exception("SHA1 initialization error.");
0169   }
0170   if (!EVP_DigestUpdate(mdctx, p_type.c_str(), p_type.size())) {
0171     throw cms::Exception("SHA1 processing error (1).");
0172   }
0173   if (!EVP_DigestUpdate(mdctx, p_cfgstr.c_str(), p_cfgstr.size())) {
0174     throw cms::Exception("SHA1 processing error (2).");
0175   }
0176   unsigned char hash[EVP_MAX_MD_SIZE];
0177   unsigned int md_len;
0178   if (!EVP_DigestFinal_ex(mdctx, hash, &md_len)) {
0179     throw cms::Exception("SHA1 finalization error.");
0180   }
0181   EVP_MD_CTX_free(mdctx);
0182
0183   char tmp[EVP_MAX_MD_SIZE * 2 + 1];
0184   // re-write bytes in hex
0185   for (unsigned int i = 0; i < md_len; i++) {
0186     ::sprintf(&tmp[i * 2], "%02x", hash[i]);
0187   }
0188   tmp[md_len * 2] = 0;
0189
0190   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__
0191                                         << "] "
0192                                            "... hash: "
0193                                         << tmp;
0194   return tmp;
0195 }
0196
0197 template <typename SiStripPayload>
0198 std::string SiStripPayloadHandler<SiStripPayload>::queryConfigMap(std::string configHash) {
0199   if (m_configMapDb.empty())
0200     return "";  // return empty string if m_configMapDb is not specified
0201
0202   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0203                                         << "Query " << m_configMapDb << " to see if the payload is already in DB.";
0204
0205   // if dev/prep, use CMS_COND_STRIP
0206   if (m_condDb.find("prep") != std::string::npos) {
0207     m_cfgMapSchemaName = "CMS_COND_STRIP";
0208   }
0209   auto cmDbSession = m_connectionPool.createCoralSession(m_configMapDb);
0210   // query the STRIP_CONFIG_TO_PAYLOAD_MAP table
0211   cmDbSession->transaction().start(true);
0212   coral::ITable& cmTable = cmDbSession->schema(m_cfgMapSchemaName).tableHandle(m_cfgMapTableName);
0213   std::unique_ptr<coral::IQuery> query(cmTable.newQuery());
0214   query->addToOutputList("PAYLOAD_HASH");
0215   query->defineOutputType("PAYLOAD_HASH", coral::AttributeSpecification::typeNameForType<std::string>());
0216   // also print these for debugging
0217   query->addToOutputList("PAYLOAD_TYPE");
0218   query->addToOutputList("CONFIG_STRING");
0219   query->addToOutputList("INSERTION_TIME");
0220   std::string whereClause("CONFIG_HASH = :CONFIG_HASH");
0221   coral::AttributeList whereData;
0222   whereData.extend<std::string>("CONFIG_HASH");
0223   whereData.begin()->data<std::string>() = configHash;
0224   query->setCondition(whereClause, whereData);
0225   coral::ICursor& cursor = query->execute();
0226   std::string p_hash;
0227   if (cursor.next()) {
0228     // the payload has been found ( fast O2O case )
0229     p_hash = cursor.currentRow()["PAYLOAD_HASH"].data<std::string>();
0230     edm::LogInfo("SiStripPayloadHandler")
0231         << "[SiStripPayloadHandler::" << __func__ << "] "
0232         << "Found associated payload hash " << p_hash
0233         << "\n...  type=" << cursor.currentRow()["PAYLOAD_TYPE"].data<std::string>()
0234         << "\n...  config=" << cursor.currentRow()["CONFIG_STRING"].data<std::string>()
0235         << "\n...  insertion_time=" << cursor.currentRow()["INSERTION_TIME"].data<coral::TimeStamp>().toString();
0236   }
0237   cmDbSession->transaction().commit();
0238
0239   return p_hash;
0240 }
0241
0242 template <typename SiStripPayload>
0243 void SiStripPayloadHandler<SiStripPayload>::updateConfigMap(std::string configHash, std::string payloadHash) {
0244   if (m_cfgMapDbFile.empty())
0245     return;  // skip this if m_cfgMapDbFile is not set
0246
0247   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0248                                         << "Updating the config to payload hash map to " << m_cfgMapDbFile;
0249
0250   // create a writable transaction
0251   auto cmSQLiteSession = m_connectionPool.createCoralSession(m_cfgMapDbFile, true);
0252   cmSQLiteSession->transaction().start(false);
0253
0254   if (!cmSQLiteSession->nominalSchema().existsTable(m_cfgMapTableName)) {
0255     // create the table if it does not exist
0256     coral::TableDescription mapTable;
0257     mapTable.setName(m_cfgMapTableName);
0258     mapTable.insertColumn("CONFIG_HASH", coral::AttributeSpecification::typeNameForType<std::string>());
0259     mapTable.insertColumn("PAYLOAD_HASH", coral::AttributeSpecification::typeNameForType<std::string>());
0260     mapTable.insertColumn("PAYLOAD_TYPE", coral::AttributeSpecification::typeNameForType<std::string>());
0261     mapTable.insertColumn("CONFIG_STRING", coral::AttributeSpecification::typeNameForType<std::string>());
0262     mapTable.insertColumn("INSERTION_TIME", coral::AttributeSpecification::typeNameForType<coral::TimeStamp>());
0263     mapTable.setPrimaryKey("CONFIG_HASH");
0264     mapTable.setNotNullConstraint("CONFIG_HASH");
0265     mapTable.setNotNullConstraint("PAYLOAD_HASH");
0266     mapTable.setNotNullConstraint("PAYLOAD_TYPE");
0267     mapTable.setNotNullConstraint("CONFIG_STRING");
0268     mapTable.setNotNullConstraint("INSERTION_TIME");
0269     cmSQLiteSession->nominalSchema().createTable(mapTable);
0270   }
0271
0272   coral::ITable& cmTable = cmSQLiteSession->nominalSchema().tableHandle(m_cfgMapTableName);
0273   coral::AttributeList insertData;
0274   insertData.extend<std::string>("CONFIG_HASH");
0275   insertData.extend<std::string>("PAYLOAD_HASH");
0276   // also insert these for bookkeeping
0277   insertData.extend<std::string>("PAYLOAD_TYPE");
0278   insertData.extend<std::string>("CONFIG_STRING");
0279   insertData.extend<coral::TimeStamp>("INSERTION_TIME");
0280   insertData["CONFIG_HASH"].data<std::string>() = configHash;
0281   insertData["PAYLOAD_HASH"].data<std::string>() = payloadHash;
0282   insertData["PAYLOAD_TYPE"].data<std::string>() = p_type;
0283   insertData["CONFIG_STRING"].data<std::string>() = p_cfgstr;
0284   insertData["INSERTION_TIME"].data<coral::TimeStamp>() = coral::TimeStamp::now();  // UTC time
0285   cmTable.dataEditor().insertRow(insertData);
0286   cmSQLiteSession->transaction().commit();
0287   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0288                                         << "Updated with mapping (configHash : payloadHash)" << configHash << " : "
0289                                         << payloadHash;
0290 }
0291
0292 // -------------------------------------------------------
0293
0294 typedef SiStripPayloadHandler<SiStripApvGain> SiStripO2OApvGain;
0295 DEFINE_FWK_MODULE(SiStripO2OApvGain);
0296
0297 typedef SiStripPayloadHandler<SiStripBadStrip> SiStripO2OBadStrip;
0298 DEFINE_FWK_MODULE(SiStripO2OBadStrip);
0299
0300 typedef SiStripPayloadHandler<SiStripFedCabling> SiStripO2OFedCabling;
0301 DEFINE_FWK_MODULE(SiStripO2OFedCabling);
0302
0303 typedef SiStripPayloadHandler<SiStripLatency> SiStripO2OLatency;
0304 DEFINE_FWK_MODULE(SiStripO2OLatency);
0305
0306 typedef SiStripPayloadHandler<SiStripNoises> SiStripO2ONoises;
0307 DEFINE_FWK_MODULE(SiStripO2ONoises);
0308
0309 typedef SiStripPayloadHandler<SiStripPedestals> SiStripO2OPedestals;
0310 DEFINE_FWK_MODULE(SiStripO2OPedestals);
0311
0312 typedef SiStripPayloadHandler<SiStripThreshold> SiStripO2OThreshold;
0313 DEFINE_FWK_MODULE(SiStripO2OThreshold);