DT/src/DTUserKeyedConfigHandler.cc

0001 /*
0002  *  See header file for a description of this class.
0003  *
0004  *  \author Paolo Ronchese INFN Padova
0005  *
0006  */
0007
0008 //-----------------------
0009 // This Class' Header --
0010 //-----------------------
0011 #include "CondTools/DT/interface/DTUserKeyedConfigHandler.h"
0012
0013 //-------------------------------
0014 // Collaborating Class Headers --
0015 //-------------------------------
0016 #include "CondFormats/DTObjects/interface/DTCCBConfig.h"
0017 #include "CondFormats/DTObjects/interface/DTKeyedConfig.h"
0018
0019 #include "FWCore/ServiceRegistry/interface/Service.h"
0020 #include "CondCore/DBOutputService/interface/PoolDBOutputService.h"
0021 #include "CondCore/DBOutputService/interface/KeyedElement.h"
0022 #include "CondCore/CondDB/interface/KeyList.h"
0023
0024 #include "RelationalAccess/ISessionProxy.h"
0025 #include "RelationalAccess/ITransaction.h"
0026 #include "RelationalAccess/ISchema.h"
0027 #include "RelationalAccess/ITable.h"
0028 #include "RelationalAccess/ICursor.h"
0029 #include "RelationalAccess/IQuery.h"
0030 #include "CoralBase/AttributeList.h"
0031 #include "CoralBase/AttributeSpecification.h"
0032 #include "CoralBase/Attribute.h"
0033
0034 //---------------
0035 // C++ Headers --
0036 //---------------
0037
0038 #include <iostream>
0039 #include <memory>
0040
0041 //-------------------
0042 // Initializations --
0043 //-------------------
0044
0045 //----------------
0046 // Constructors --
0047 //----------------
0048 DTUserKeyedConfigHandler::DTUserKeyedConfigHandler(const edm::ParameterSet& ps)
0049     : dataRun(ps.getParameter<int>("run")),
0050       dataTag(ps.getParameter<std::string>("tag")),
0051       onlineConnect(ps.getParameter<std::string>("onlineDB")),
0052       onlineAuthentication(ps.getParameter<std::string>("onlineAuthentication")),
0053       onlineAuthSys(ps.getUntrackedParameter<int>("onlineAuthSys", 1)),
0054       brickContainer(ps.getParameter<std::string>("container")),
0055       writeKeys(ps.getParameter<bool>("writeKeys")),
0056       writeData(ps.getParameter<bool>("writeData")),
0057       connection(),
0058       isession() {
0059   std::cout << " PopCon application for DT configuration export " << onlineAuthentication << std::endl;
0060
0061   std::vector<edm::ParameterSet> dtConfigKeys(ps.getParameter<std::vector<edm::ParameterSet> >("DTConfigKeys"));
0062   std::vector<edm::ParameterSet>::const_iterator iter = dtConfigKeys.begin();
0063   std::vector<edm::ParameterSet>::const_iterator iend = dtConfigKeys.end();
0064   while (iter != iend) {
0065     const edm::ParameterSet& cp = *iter++;
0066     int configType = cp.getUntrackedParameter<int>("configType");
0067     int configKey = cp.getUntrackedParameter<int>("configKey");
0068     std::cout << "config: " << configType << " -> " << configKey << std::endl;
0069     DTConfigKey userKey;
0070     userKey.confType = configType;
0071     userKey.confKey = configKey;
0072     userConf.push_back(userKey);
0073   }
0074 }
0075
0076 //--------------
0077 // Destructor --
0078 //--------------
0079 DTUserKeyedConfigHandler::~DTUserKeyedConfigHandler() {}
0080
0081 //--------------
0082 // Operations --
0083 //--------------
0084 void DTUserKeyedConfigHandler::getNewObjects() {
0085   //to access the information on the tag inside the offline database:
0086   cond::TagInfo_t const& ti = tagInfo();
0087   cond::Time_t last = ti.lastInterval.since;
0088   std::cout << "last configuration key already copied for run: " << last << std::endl;
0089
0090   std::vector<DTConfigKey> lastKey;
0091   //  std::cout << "check for last " << std::endl;
0092   if (last == 0) {
0093     DTCCBConfig* dummyConf = new DTCCBConfig(dataTag);
0094     dummyConf->setStamp(0);
0095     dummyConf->setFullKey(lastKey);
0096     cond::Time_t snc = 1;
0097     if (writeKeys && (dataRun > 1))
0098       m_to_transfer.push_back(std::make_pair(dummyConf, snc));
0099   } else {
0100     std::cout << "get last payload" << std::endl;
0101     Ref payload = lastPayload();
0102     std::cout << "get last full key" << std::endl;
0103     lastKey = payload->fullKey();
0104     std::cout << "last key: " << std::endl;
0105     std::vector<DTConfigKey>::const_iterator keyIter = lastKey.begin();
0106     std::vector<DTConfigKey>::const_iterator keyIend = lastKey.end();
0107     while (keyIter != keyIend) {
0108       const DTConfigKey& keyList = *keyIter++;
0109       std::cout << keyList.confType << " : " << keyList.confKey << std::endl;
0110     }
0111   }
0112
0113   std::cout << "configure DbConnection" << std::endl;
0114   //  conn->configure( cond::CmsDefaults );
0115   connection.setAuthenticationPath(onlineAuthentication);
0116   connection.setAuthenticationSystem(onlineAuthSys);
0117   connection.configure();
0118   std::cout << "create/open DbSession" << std::endl;
0119   isession = connection.createCoralSession(onlineConnect);
0120   std::cout << "start transaction" << std::endl;
0121   isession->transaction().start(true);
0122
0123   // get ccb identifiers map
0124   std::cout << "retrieve CCB map" << std::endl;
0125   std::map<int, DTCCBId> ccbMap;
0126   coral::ITable& ccbMapTable = isession->nominalSchema().tableHandle("CCBMAP");
0127   std::unique_ptr<coral::IQuery> ccbMapQuery(ccbMapTable.newQuery());
0128   ccbMapQuery->addToOutputList("CCBID");
0129   ccbMapQuery->addToOutputList("WHEEL");
0130   ccbMapQuery->addToOutputList("SECTOR");
0131   ccbMapQuery->addToOutputList("STATION");
0132   coral::ICursor& ccbMapCursor = ccbMapQuery->execute();
0133   while (ccbMapCursor.next()) {
0134     const coral::AttributeList& row = ccbMapCursor.currentRow();
0135     int ccb = row["CCBID"].data<int>();
0136     int wheel = row["WHEEL"].data<int>();
0137     int sector = row["SECTOR"].data<int>();
0138     int station = row["STATION"].data<int>();
0139     DTCCBId ccbId;
0140     ccbId.wheelId = wheel;
0141     ccbId.stationId = station;
0142     ccbId.sectorId = sector;
0143     ccbMap.insert(std::pair<int, DTCCBId>(ccb, ccbId));
0144   }
0145
0146   // get brick types
0147   std::cout << "retrieve brick types" << std::endl;
0148   std::map<int, int> bktMap;
0149   coral::AttributeList emptyBindVariableList;
0150   std::unique_ptr<coral::IQuery> brickTypeQuery(isession->nominalSchema().newQuery());
0151   brickTypeQuery->addToTableList("CFGBRICKS");
0152   brickTypeQuery->addToTableList("BRKT2CSETT");
0153   std::string bTypeCondition = "CFGBRICKS.BRKTYPE=BRKT2CSETT.BRKTYPE";
0154   brickTypeQuery->addToOutputList("CFGBRICKS.BRKID");
0155   brickTypeQuery->addToOutputList("BRKT2CSETT.CSETTYPE");
0156   brickTypeQuery->setCondition(bTypeCondition, emptyBindVariableList);
0157   coral::ICursor& brickTypeCursor = brickTypeQuery->execute();
0158   while (brickTypeCursor.next()) {
0159     const coral::AttributeList& row = brickTypeCursor.currentRow();
0160     int id = row["CFGBRICKS.BRKID"].data<int>();
0161     int bt = row["BRKT2CSETT.CSETTYPE"].data<short>();
0162     //    std::cout << "brick " << id << " type " << bt << std::endl;
0163     // @@FIX - TEMPORARY PATCH
0164     //    if ( bt > 3 ) bt = 3;
0165     bktMap.insert(std::pair<int, int>(id, bt));
0166   }
0167
0168   // get ccb config keys
0169   std::cout << "retrieve CCB configuration keys" << std::endl;
0170   std::map<int, std::map<int, int>*> keyMap;
0171   std::map<int, int> cckMap;
0172   coral::ITable& ccbRelTable = isession->nominalSchema().tableHandle("CCBRELATIONS");
0173   std::unique_ptr<coral::IQuery> ccbRelQuery(ccbRelTable.newQuery());
0174   ccbRelQuery->addToOutputList("CONFKEY");
0175   ccbRelQuery->addToOutputList("CCBID");
0176   ccbRelQuery->addToOutputList("CONFCCBKEY");
0177   coral::ICursor& ccbRelCursor = ccbRelQuery->execute();
0178   // loop over all full configurations
0179   while (ccbRelCursor.next()) {
0180     const coral::AttributeList& row = ccbRelCursor.currentRow();
0181     int cfg = row["CONFKEY"].data<int>();
0182     int ccb = row["CCBID"].data<int>();
0183     int key = row["CONFCCBKEY"].data<int>();
0184     // check for used configurations
0185     //    if ( cfgMap.find( cfg ) == cfgMap.end() ) continue;
0186     if (userDiscardedKey(cfg))
0187       continue;
0188     std::map<int, std::map<int, int>*>::const_iterator keyIter = keyMap.find(cfg);
0189     std::map<int, std::map<int, int>*>::const_iterator keyIend = keyMap.end();
0190     std::map<int, int>* mapPtr = nullptr;
0191     // check for new full configuration
0192     if (keyIter != keyIend)
0193       mapPtr = keyIter->second;
0194     else
0195       keyMap.insert(std::pair<int, std::map<int, int>*>(cfg, mapPtr = new std::map<int, int>));
0196     // store ccb config key
0197     std::map<int, int>& mapRef(*mapPtr);
0198     mapRef.insert(std::pair<int, int>(ccb, key));
0199     // check for new ccb config key
0200     if (cckMap.find(key) == cckMap.end())
0201       cckMap.insert(std::pair<int, int>(key, ccb));
0202   }
0203
0204   // get brick keys
0205   std::cout << "retrieve CCB configuration bricks" << std::endl;
0206   std::map<int, std::vector<int>*> brkMap;
0207   coral::ITable& confBrickTable = isession->nominalSchema().tableHandle("CFG2BRKREL");
0208   std::unique_ptr<coral::IQuery> confBrickQuery(confBrickTable.newQuery());
0209   confBrickQuery->addToOutputList("CONFID");
0210   confBrickQuery->addToOutputList("BRKID");
0211   coral::ICursor& confBrickCursor = confBrickQuery->execute();
0212   // loop over all brick keys
0213   while (confBrickCursor.next()) {
0214     const coral::AttributeList& row = confBrickCursor.currentRow();
0215     int key = row["CONFID"].data<int>();
0216     int brk = row["BRKID"].data<int>();
0217     // check for used ccb config key
0218     if (cckMap.find(key) == cckMap.end())
0219       continue;
0220     std::map<int, std::vector<int>*>::const_iterator brkIter = brkMap.find(key);
0221     std::map<int, std::vector<int>*>::const_iterator brkIend = brkMap.end();
0222     // check for new ccb config key
0223     std::vector<int>* brkPtr = nullptr;
0224     if (brkIter != brkIend)
0225       brkPtr = brkIter->second;
0226     else
0227       brkMap.insert(std::pair<int, std::vector<int>*>(key, brkPtr = new std::vector<int>));
0228     // store brick key
0229     brkPtr->push_back(brk);
0230   }
0231
0232   // set run and full configuration in payload
0233   DTCCBConfig* fullConf = new DTCCBConfig(dataTag);
0234   fullConf->setStamp(1);
0235   fullConf->setFullKey(userConf);
0236   std::map<int, bool> userBricks;
0237   std::map<int, bool>::const_iterator uBrkIter = userBricks.begin();
0238   std::map<int, bool>::const_iterator uBrkIend = userBricks.end();
0239   std::vector<DTConfigKey>::const_iterator cfgIter = userConf.begin();
0240   std::vector<DTConfigKey>::const_iterator cfgIend = userConf.end();
0241   while (cfgIter != cfgIend) {
0242     const DTConfigKey& cfgEntry = *cfgIter++;
0243     int cft = cfgEntry.confType;
0244     int cfg = cfgEntry.confKey;
0245     // retrieve ccb config map
0246     std::map<int, std::map<int, int>*>::const_iterator keyIter = keyMap.find(cfg);
0247     std::map<int, std::map<int, int>*>::const_iterator keyIend = keyMap.end();
0248     std::map<int, int>* mapPtr = nullptr;
0249     if (keyIter != keyIend)
0250       mapPtr = keyIter->second;
0251     if (mapPtr == nullptr)
0252       continue;
0253     std::map<int, int>::const_iterator ccbIter = mapPtr->begin();
0254     std::map<int, int>::const_iterator ccbIend = mapPtr->end();
0255     while (ccbIter != ccbIend) {
0256       const std::pair<int, int>& ccbEntry = *ccbIter++;
0257       // get ccb config key
0258       int ccb = ccbEntry.first;
0259       int key = ccbEntry.second;
0260       // retrieve chamber id
0261       std::map<int, DTCCBId>::const_iterator ccbIter = ccbMap.find(ccb);
0262       std::map<int, DTCCBId>::const_iterator ccbIend = ccbMap.end();
0263       if (ccbIter == ccbIend)
0264         continue;
0265       const DTCCBId& chaId = ccbIter->second;
0266       // retrieve brick id list
0267       std::map<int, std::vector<int>*>::const_iterator brkIter = brkMap.find(key);
0268       std::map<int, std::vector<int>*>::const_iterator brkIend = brkMap.end();
0269       if (brkIter == brkIend)
0270         continue;
0271       std::vector<int>* brkPtr = brkIter->second;
0272       if (brkPtr == nullptr)
0273         continue;
0274       // brick id lists in payload
0275       std::vector<int> bkList;
0276       bkList.reserve(20);
0277       std::map<int, int>::const_iterator bktIter = bktMap.begin();
0278       std::map<int, int>::const_iterator bktIend = bktMap.end();
0279       std::vector<int>::const_iterator bkiIter = brkPtr->begin();
0280       std::vector<int>::const_iterator bkiIend = brkPtr->end();
0281       while (bkiIter != bkiIend) {
0282         int brickId = *bkiIter++;
0283         bktIter = bktMap.find(brickId);
0284         if (bktIter == bktIend)
0285           continue;
0286         if (bktIter->second == cft) {
0287           bkList.push_back(brickId);
0288           uBrkIter = userBricks.find(brickId);
0289           if (uBrkIter == uBrkIend)
0290             userBricks.insert(std::pair<int, bool>(brickId, true));
0291         }
0292       }
0293       fullConf->appendConfigKey(chaId.wheelId, chaId.stationId, chaId.sectorId, bkList);
0294     }
0295   }
0296   cond::Time_t snc = dataRun;
0297   if (writeKeys)
0298     m_to_transfer.push_back(std::make_pair(fullConf, snc));
0299   std::cout << "writing payload : " << sizeof(*fullConf) << " ( " << (fullConf->end() - fullConf->begin()) << " ) "
0300             << std::endl;
0301   if (writeData)
0302     chkConfigList(userBricks);
0303
0304   isession->transaction().commit();
0305
0306   return;
0307 }
0308
0309 void DTUserKeyedConfigHandler::chkConfigList(const std::map<int, bool>& userBricks) {
0310   std::cout << "open POOL out db " << std::endl;
0311   edm::Service<cond::service::PoolDBOutputService> outdb;
0312
0313   std::map<int, bool>::const_iterator uBrkIter = userBricks.begin();
0314   std::map<int, bool>::const_iterator uBrkIend = userBricks.end();
0315
0316   coral::ITable& brickConfigTable = isession->nominalSchema().tableHandle("CFGBRICKS");
0317   std::unique_ptr<coral::IQuery> brickConfigQuery(brickConfigTable.newQuery());
0318   brickConfigQuery->addToOutputList("BRKID");
0319   brickConfigQuery->addToOutputList("BRKNAME");
0320   coral::ICursor& brickConfigCursor = brickConfigQuery->execute();
0321   std::vector<int> missingList;
0322   std::vector<unsigned long long> checkedKeys;
0323   while (brickConfigCursor.next()) {
0324     const coral::AttributeList& row = brickConfigCursor.currentRow();
0325     int brickConfigId = row["BRKID"].data<int>();
0326     uBrkIter = userBricks.find(brickConfigId);
0327     if (uBrkIter == uBrkIend)
0328       continue;
0329     if (!(uBrkIter->second))
0330       continue;
0331     std::string brickConfigName = row["BRKNAME"].data<std::string>();
0332     std::cout << "brick " << brickConfigId << " : " << brickConfigName << std::endl;
0333     bool brickFound = false;
0334     try {
0335       std::cout << "load brick " << brickConfigId << std::endl;
0336       std::cout << "key list " << keyList << std::endl;
0337       std::cout << "get brick..." << std::endl;
0338       std::shared_ptr<DTKeyedConfig> brickCheck = keyList->getUsingKey<DTKeyedConfig>(brickConfigId);
0339       if (brickCheck.get()) {
0340         brickFound = (brickCheck->getId() == brickConfigId);
0341       }
0342     } catch (std::exception const&) {
0343     }
0344     if (!brickFound) {
0345       std::cout << "brick " << brickConfigId << " missing, copy request" << std::endl;
0346       missingList.push_back(brickConfigId);
0347     }
0348   }
0349
0350   std::vector<int>::const_iterator brickIter = missingList.begin();
0351   std::vector<int>::const_iterator brickIend = missingList.end();
0352   while (brickIter != brickIend) {
0353     int brickConfigId = *brickIter++;
0354     coral::AttributeList bindVariableList;
0355     bindVariableList.extend("brickId", typeid(int));
0356     bindVariableList["brickId"].data<int>() = brickConfigId;
0357     std::unique_ptr<coral::IQuery> brickDataQuery(isession->nominalSchema().newQuery());
0358     brickDataQuery->addToTableList("CFGRELATIONS");
0359     brickDataQuery->addToTableList("CONFIGCMDS");
0360     std::string brickCondition = "CONFIGCMDS.CMDID=CFGRELATIONS.CMDID";
0361     brickCondition += " and CFGRELATIONS.BRKID=:brickId";
0362     brickDataQuery->addToOutputList("CFGRELATIONS.BRKID");
0363     brickDataQuery->addToOutputList("CONFIGCMDS.CONFDATA");
0364     brickDataQuery->setCondition(brickCondition, bindVariableList);
0365     coral::ICursor& brickDataCursor = brickDataQuery->execute();
0366     DTKeyedConfig brickData;
0367     brickData.setId(brickConfigId);
0368     while (brickDataCursor.next()) {
0369       const coral::AttributeList& row = brickDataCursor.currentRow();
0370       brickData.add(row["CONFIGCMDS.CONFDATA"].data<std::string>());
0371     }
0372     cond::KeyedElement k(&brickData, brickConfigId);
0373     std::cout << "now writing brick: " << brickConfigId << std::endl;
0374     outdb->writeOneIOV(*(k.m_obj), k.m_key, brickContainer);
0375   }
0376
0377   return;
0378 }
0379
0380 std::string DTUserKeyedConfigHandler::id() const { return dataTag; }
0381
0382 bool DTUserKeyedConfigHandler::sameConfigList(const std::vector<DTConfigKey>& cfgl,
0383                                               const std::vector<DTConfigKey>& cfgr) {
0384   if (cfgl.size() != cfgr.size())
0385     return false;
0386   std::map<int, int> lmap;
0387   std::vector<DTConfigKey>::const_iterator lIter = cfgl.begin();
0388   std::vector<DTConfigKey>::const_iterator lIend = cfgl.end();
0389   while (lIter != lIend) {
0390     const DTConfigKey& entry = *lIter++;
0391     lmap.insert(std::pair<int, int>(entry.confType, entry.confKey));
0392   }
0393   std::map<int, int> rmap;
0394   std::vector<DTConfigKey>::const_iterator rIter = cfgr.begin();
0395   std::vector<DTConfigKey>::const_iterator rIend = cfgr.end();
0396   while (rIter != rIend) {
0397     const DTConfigKey& entry = *rIter++;
0398     rmap.insert(std::pair<int, int>(entry.confType, entry.confKey));
0399   }
0400   std::map<int, int>::const_iterator lmIter = lmap.begin();
0401   std::map<int, int>::const_iterator lmIend = lmap.end();
0402   std::map<int, int>::const_iterator rmIter = rmap.begin();
0403   std::map<int, int>::const_iterator rmIend = rmap.end();
0404   while ((lmIter != lmIend) && (rmIter != rmIend)) {
0405     const std::pair<int, int>& lEntry = *lmIter++;
0406     const std::pair<int, int>& rEntry = *rmIter++;
0407     if (lEntry.first != rEntry.first)
0408       return false;
0409     if (lEntry.second != rEntry.second)
0410       return false;
0411   }
0412   return true;
0413 }
0414
0415 bool DTUserKeyedConfigHandler::userDiscardedKey(int key) {
0416   std::vector<DTConfigKey>::const_iterator iter = userConf.begin();
0417   std::vector<DTConfigKey>::const_iterator iend = userConf.end();
0418   while (iter != iend) {
0419     const DTConfigKey& entry = *iter++;
0420     if (entry.confKey == key)
0421       return false;
0422   }
0423   return true;
0424 }
0425
0426 void DTUserKeyedConfigHandler::setList(const cond::persistency::KeyList* list) { keyList = list; }