Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-04-15 00:38:01

0001 #include "FWCore/Framework/interface/Event.h"
0002 #include "FWCore/Framework/interface/one/EDAnalyzer.h"
0003 #include "FWCore/Framework/interface/MakerMacros.h"
0004 #include "FWCore/ServiceRegistry/interface/Service.h"
0005 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0006 #include "Utilities/OpenSSL/interface/openssl_init.h"
0007 
0008 #include <iostream>
0009 #include <sstream>
0010 #include <typeinfo>
0011 
0012 #include "CondCore/CondDB/interface/ConnectionPool.h"
0013 
0014 #include "RelationalAccess/ITransaction.h"
0015 #include "RelationalAccess/ISessionProxy.h"
0016 #include "RelationalAccess/ISchema.h"
0017 #include "RelationalAccess/ITable.h"
0018 #include "RelationalAccess/IQuery.h"
0019 #include "RelationalAccess/ICursor.h"
0020 #include "RelationalAccess/ITableDataEditor.h"
0021 #include "RelationalAccess/TableDescription.h"
0022 #include "CoralBase/Attribute.h"
0023 #include "CoralBase/AttributeList.h"
0024 #include "CoralBase/TimeStamp.h"
0025 
0026 #include "OnlineDB/SiStripESSources/interface/SiStripCondObjBuilderFromDb.h"
0027 
0028 template <typename SiStripPayload>
0029 class SiStripPayloadHandler : public edm::one::EDAnalyzer<> {
0030 public:
0031   explicit SiStripPayloadHandler(const edm::ParameterSet& iConfig);
0032   ~SiStripPayloadHandler() override;
0033   void analyze(const edm::Event& evt, const edm::EventSetup& evtSetup) override;
0034   void endJob() override;
0035 
0036 private:
0037   std::string makeConfigHash();
0038   std::string queryConfigMap(std::string configHash);
0039   void updateConfigMap(std::string configHash, std::string payloadHash);
0040 
0041 private:
0042   cond::persistency::ConnectionPool m_connectionPool;
0043   std::string m_configMapDb;
0044   std::string m_cfgMapSchemaName;
0045   std::string m_cfgMapTableName;
0046   std::string m_condDb;
0047   std::string m_cfgMapDbFile;
0048   std::string m_localCondDbFile;
0049   std::string m_targetTag;
0050   cond::Time_t m_since;
0051 
0052   std::string p_type;
0053   std::string p_cfgstr;
0054 };
0055 
0056 template <typename SiStripPayload>
0057 SiStripPayloadHandler<SiStripPayload>::SiStripPayloadHandler(const edm::ParameterSet& iConfig)
0058     : m_connectionPool(),
0059       m_cfgMapSchemaName(iConfig.getUntrackedParameter<std::string>("cfgMapSchemaName", "CMS_COND_O2O")),
0060       m_cfgMapTableName(iConfig.getUntrackedParameter<std::string>("cfgMapTableName", "STRIP_CONFIG_TO_PAYLOAD_MAP")),
0061       m_condDb(iConfig.getParameter<std::string>("conditionDatabase")),
0062       m_localCondDbFile(iConfig.getParameter<std::string>("condDbFile")),
0063       m_targetTag(iConfig.getParameter<std::string>("targetTag")),
0064       m_since(iConfig.getParameter<uint32_t>("since")),
0065       p_type(cond::demangledName(typeid(SiStripPayload))) {
0066   edm::Service<SiStripCondObjBuilderFromDb> condObjBuilder;
0067   p_cfgstr = condObjBuilder->getConfigString(typeid(SiStripPayload));
0068   if (iConfig.exists("configMapDatabase"))
0069     m_configMapDb = iConfig.getParameter<std::string>("configMapDatabase");
0070   if (iConfig.exists("cfgMapDbFile"))
0071     m_cfgMapDbFile = iConfig.getParameter<std::string>("cfgMapDbFile");
0072   m_connectionPool.setParameters(iConfig.getParameter<edm::ParameterSet>("DBParameters"));
0073   m_connectionPool.configure();
0074 }
0075 
0076 template <typename SiStripPayload>
0077 SiStripPayloadHandler<SiStripPayload>::~SiStripPayloadHandler() = default;
0078 
0079 template <typename SiStripPayload>
0080 void SiStripPayloadHandler<SiStripPayload>::analyze(const edm::Event& evt, const edm::EventSetup& evtSetup) {
0081   // some extra work: Getting the payload hash of the last IOV from condDb
0082   // Will be compared with the new payload and reported
0083   cond::persistency::Session condDbSession = m_connectionPool.createSession(m_condDb);
0084   condDbSession.transaction().start(true);
0085   cond::persistency::IOVProxy iovProxy = condDbSession.readIov(m_targetTag);
0086   cond::Hash last_hash = iovProxy.getLast().payloadId;
0087 
0088   // that's the final goal: obtain the payload to store into the sqlite file for the upload into the DB
0089   std::shared_ptr<SiStripPayload> payloadToUpload;
0090   // first compute the hash of the configuration
0091   std::string configHash = makeConfigHash();
0092   // query the configMap DB to find the corresponding payload hash
0093   std::string mappedPayloadHash = queryConfigMap(configHash);
0094   bool mapUpToDate = false;
0095 
0096   if (!mappedPayloadHash.empty()) {
0097     // the payload has been found ( fast O2O case )
0098     // copy the payload from condtition database
0099     mapUpToDate = true;
0100     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0101                                           << "Try retrieving payload from " << m_condDb;
0102     payloadToUpload = condDbSession.fetchPayload<SiStripPayload>(mappedPayloadHash);
0103     std::cout << "@@@[FastO2O:true]@@@" << std::endl;
0104     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0105                                           << " ... Payload is copied from offline condition database.";
0106   } else {
0107     // start the long O2O...
0108     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0109                                           << "NO mapping payload hash found. Will run the long O2O. ";
0110     SiStripPayload* obj = nullptr;
0111     edm::Service<SiStripCondObjBuilderFromDb> condObjBuilder;
0112     if (typeid(SiStripPayload) == typeid(SiStripApvGain)) {
0113       // special treatment for ApvGain : provide last payload in DB
0114       condObjBuilder->setLastIovGain(condDbSession.fetchPayload<SiStripApvGain>(last_hash));
0115     }
0116     condObjBuilder->getValue(obj);
0117     payloadToUpload = std::shared_ptr<SiStripPayload>(obj);
0118     std::cout << "@@@[FastO2O:false]@@@" << std::endl;
0119     edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0120                                           << " ... New payload has been created.";
0121   }
0122   condDbSession.transaction().commit();
0123 
0124   // write payload and iov in the local file
0125   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0126                                         << "Write payload to local sqlite file: " << m_localCondDbFile;
0127   cond::persistency::Session localFileSession = m_connectionPool.createSession(m_localCondDbFile, true);
0128   localFileSession.transaction().start(false);
0129   // write the payload
0130   cond::Hash thePayloadHash = localFileSession.storePayload<SiStripPayload>(*payloadToUpload);
0131   cond::persistency::IOVEditor iovEditor = localFileSession.createIov<SiStripPayload>(m_targetTag, cond::runnumber);
0132   iovEditor.setDescription("New IOV");
0133   // inserting the iov
0134   iovEditor.insert(m_since, thePayloadHash);
0135   iovEditor.flush();
0136   localFileSession.transaction().commit();
0137   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0138                                         << "Payload " << thePayloadHash << " inserted to sqlite with IOV " << m_since;
0139 
0140   // last step, update the configMap if required
0141   if (!mapUpToDate) {
0142     updateConfigMap(configHash, thePayloadHash);
0143   }
0144 
0145   // finish the extra work: Compare the new payload with last IOV
0146   if (last_hash == thePayloadHash) {
0147     std::cout << "@@@[PayloadChange:false]@@@" << last_hash << std::endl;
0148   } else {
0149     std::cout << "@@@[PayloadChange:true]@@@" << last_hash << " -> " << thePayloadHash << std::endl;
0150   }
0151 }
0152 
0153 template <typename SiStripPayload>
0154 void SiStripPayloadHandler<SiStripPayload>::endJob() {}
0155 
0156 template <typename SiStripPayload>
0157 std::string SiStripPayloadHandler<SiStripPayload>::makeConfigHash() {
0158   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0159                                         << "Convert config string to SHA-1 hash for " << p_type
0160                                         << "\n... config: " << p_cfgstr;
0161 
0162   // calcuate SHA-1 hash using openssl
0163   // adapted from cond::persistency::makeHash() in CondCore/CondDB/src/IOVSchema.cc
0164   cms::openssl_init();
0165   EVP_MD_CTX* mdctx = EVP_MD_CTX_new();
0166   const EVP_MD* md = EVP_get_digestbyname("SHA1");
0167   if (!EVP_DigestInit_ex(mdctx, md, nullptr)) {
0168     throw cms::Exception("SHA1 initialization error.");
0169   }
0170   if (!EVP_DigestUpdate(mdctx, p_type.c_str(), p_type.size())) {
0171     throw cms::Exception("SHA1 processing error (1).");
0172   }
0173   if (!EVP_DigestUpdate(mdctx, p_cfgstr.c_str(), p_cfgstr.size())) {
0174     throw cms::Exception("SHA1 processing error (2).");
0175   }
0176   unsigned char hash[EVP_MAX_MD_SIZE];
0177   unsigned int md_len;
0178   if (!EVP_DigestFinal_ex(mdctx, hash, &md_len)) {
0179     throw cms::Exception("SHA1 finalization error.");
0180   }
0181   EVP_MD_CTX_free(mdctx);
0182 
0183   char tmp[EVP_MAX_MD_SIZE * 2 + 1];
0184   // re-write bytes in hex
0185   for (unsigned int i = 0; i < md_len; i++) {
0186     ::sprintf(&tmp[i * 2], "%02x", hash[i]);
0187   }
0188   tmp[md_len * 2] = 0;
0189 
0190   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__
0191                                         << "] "
0192                                            "... hash: "
0193                                         << tmp;
0194   return tmp;
0195 }
0196 
0197 template <typename SiStripPayload>
0198 std::string SiStripPayloadHandler<SiStripPayload>::queryConfigMap(std::string configHash) {
0199   if (m_configMapDb.empty())
0200     return "";  // return empty string if m_configMapDb is not specified
0201 
0202   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0203                                         << "Query " << m_configMapDb << " to see if the payload is already in DB.";
0204 
0205   auto cmDbSession = m_connectionPool.createCoralSession(m_configMapDb);
0206   // query the STRIP_CONFIG_TO_PAYLOAD_MAP table
0207   cmDbSession->transaction().start(true);
0208   coral::ITable& cmTable = cmDbSession->schema(m_cfgMapSchemaName).tableHandle(m_cfgMapTableName);
0209   std::unique_ptr<coral::IQuery> query(cmTable.newQuery());
0210   query->addToOutputList("PAYLOAD_HASH");
0211   query->defineOutputType("PAYLOAD_HASH", coral::AttributeSpecification::typeNameForType<std::string>());
0212   // also print these for debugging
0213   query->addToOutputList("PAYLOAD_TYPE");
0214   query->addToOutputList("CONFIG_STRING");
0215   query->addToOutputList("INSERTION_TIME");
0216   std::string whereClause("CONFIG_HASH = :CONFIG_HASH");
0217   coral::AttributeList whereData;
0218   whereData.extend<std::string>("CONFIG_HASH");
0219   whereData.begin()->data<std::string>() = configHash;
0220   query->setCondition(whereClause, whereData);
0221   coral::ICursor& cursor = query->execute();
0222   std::string p_hash;
0223   if (cursor.next()) {
0224     // the payload has been found ( fast O2O case )
0225     p_hash = cursor.currentRow()["PAYLOAD_HASH"].data<std::string>();
0226     edm::LogInfo("SiStripPayloadHandler")
0227         << "[SiStripPayloadHandler::" << __func__ << "] "
0228         << "Found associated payload hash " << p_hash
0229         << "\n...  type=" << cursor.currentRow()["PAYLOAD_TYPE"].data<std::string>()
0230         << "\n...  config=" << cursor.currentRow()["CONFIG_STRING"].data<std::string>()
0231         << "\n...  insertion_time=" << cursor.currentRow()["INSERTION_TIME"].data<coral::TimeStamp>().toString();
0232   }
0233   cmDbSession->transaction().commit();
0234 
0235   return p_hash;
0236 }
0237 
0238 template <typename SiStripPayload>
0239 void SiStripPayloadHandler<SiStripPayload>::updateConfigMap(std::string configHash, std::string payloadHash) {
0240   if (m_cfgMapDbFile.empty())
0241     return;  // skip this if m_cfgMapDbFile is not set
0242 
0243   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0244                                         << "Updating the config to payload hash map to " << m_cfgMapDbFile;
0245 
0246   // create a writable transaction
0247   auto cmSQLiteSession = m_connectionPool.createCoralSession(m_cfgMapDbFile, true);
0248   cmSQLiteSession->transaction().start(false);
0249 
0250   if (!cmSQLiteSession->nominalSchema().existsTable(m_cfgMapTableName)) {
0251     // create the table if it does not exist
0252     coral::TableDescription mapTable;
0253     mapTable.setName(m_cfgMapTableName);
0254     mapTable.insertColumn("CONFIG_HASH", coral::AttributeSpecification::typeNameForType<std::string>());
0255     mapTable.insertColumn("PAYLOAD_HASH", coral::AttributeSpecification::typeNameForType<std::string>());
0256     mapTable.insertColumn("PAYLOAD_TYPE", coral::AttributeSpecification::typeNameForType<std::string>());
0257     mapTable.insertColumn("CONFIG_STRING", coral::AttributeSpecification::typeNameForType<std::string>());
0258     mapTable.insertColumn("INSERTION_TIME", coral::AttributeSpecification::typeNameForType<coral::TimeStamp>());
0259     mapTable.setPrimaryKey("CONFIG_HASH");
0260     mapTable.setNotNullConstraint("CONFIG_HASH");
0261     mapTable.setNotNullConstraint("PAYLOAD_HASH");
0262     mapTable.setNotNullConstraint("PAYLOAD_TYPE");
0263     mapTable.setNotNullConstraint("CONFIG_STRING");
0264     mapTable.setNotNullConstraint("INSERTION_TIME");
0265     cmSQLiteSession->nominalSchema().createTable(mapTable);
0266   }
0267 
0268   coral::ITable& cmTable = cmSQLiteSession->nominalSchema().tableHandle(m_cfgMapTableName);
0269   coral::AttributeList insertData;
0270   insertData.extend<std::string>("CONFIG_HASH");
0271   insertData.extend<std::string>("PAYLOAD_HASH");
0272   // also insert these for bookkeeping
0273   insertData.extend<std::string>("PAYLOAD_TYPE");
0274   insertData.extend<std::string>("CONFIG_STRING");
0275   insertData.extend<coral::TimeStamp>("INSERTION_TIME");
0276   insertData["CONFIG_HASH"].data<std::string>() = configHash;
0277   insertData["PAYLOAD_HASH"].data<std::string>() = payloadHash;
0278   insertData["PAYLOAD_TYPE"].data<std::string>() = p_type;
0279   insertData["CONFIG_STRING"].data<std::string>() = p_cfgstr;
0280   insertData["INSERTION_TIME"].data<coral::TimeStamp>() = coral::TimeStamp::now();  // UTC time
0281   cmTable.dataEditor().insertRow(insertData);
0282   cmSQLiteSession->transaction().commit();
0283   edm::LogInfo("SiStripPayloadHandler") << "[SiStripPayloadHandler::" << __func__ << "] "
0284                                         << "Updated with mapping (configHash : payloadHash)" << configHash << " : "
0285                                         << payloadHash;
0286 }
0287 
0288 // -------------------------------------------------------
0289 
0290 typedef SiStripPayloadHandler<SiStripApvGain> SiStripO2OApvGain;
0291 DEFINE_FWK_MODULE(SiStripO2OApvGain);
0292 
0293 typedef SiStripPayloadHandler<SiStripBadStrip> SiStripO2OBadStrip;
0294 DEFINE_FWK_MODULE(SiStripO2OBadStrip);
0295 
0296 typedef SiStripPayloadHandler<SiStripFedCabling> SiStripO2OFedCabling;
0297 DEFINE_FWK_MODULE(SiStripO2OFedCabling);
0298 
0299 typedef SiStripPayloadHandler<SiStripLatency> SiStripO2OLatency;
0300 DEFINE_FWK_MODULE(SiStripO2OLatency);
0301 
0302 typedef SiStripPayloadHandler<SiStripNoises> SiStripO2ONoises;
0303 DEFINE_FWK_MODULE(SiStripO2ONoises);
0304 
0305 typedef SiStripPayloadHandler<SiStripPedestals> SiStripO2OPedestals;
0306 DEFINE_FWK_MODULE(SiStripO2OPedestals);
0307 
0308 typedef SiStripPayloadHandler<SiStripThreshold> SiStripO2OThreshold;
0309 DEFINE_FWK_MODULE(SiStripO2OThreshold);