LumiProducer/plugins/LumiCorrectionSource.cc

0001 // -*- C++ -*-
0002 //
0003 // Package:    LumiProducer
0004 // Class:      LumiCorrectionSource
0005 //
0006 /**\class LumiCorrectionSource LumiCorrectionSource.cc RecoLuminosity/LumiProducer/src/LumiCorrectionSource.cc
0007 Description: A essource/esproducer for lumi correction factor and run parameters needed to deduce the corrections
0008       Author: Zhen Xie
0009 */
0010
0011 //#include <memory>
0012 #include "FWCore/Framework/interface/SourceFactory.h"
0013 #include "FWCore/Framework/interface/ESProducer.h"
0014 #include "FWCore/Framework/interface/ESHandle.h"
0015 #include "FWCore/Framework/interface/EventSetup.h"
0016
0017 #include "FWCore/ParameterSet/interface/ParameterSet.h"
0018 #include "FWCore/Framework/interface/Event.h"
0019 #include "FWCore/Framework/interface/LuminosityBlock.h"
0020 #include "FWCore/Framework/interface/Run.h"
0021 #include "DataFormats/Provenance/interface/LuminosityBlockID.h"
0022 #include "FWCore/ServiceRegistry/interface/Service.h"
0023 #include "FWCore/Framework/interface/IOVSyncValue.h"
0024 #include "CoralBase/Exception.h"
0025 #include "CoralBase/AttributeList.h"
0026 #include "CoralBase/Attribute.h"
0027 #include "CoralBase/AttributeSpecification.h"
0028 #include "CoralBase/Exception.h"
0029 #include "CoralKernel/Context.h"
0030 #include "RelationalAccess/IAuthenticationService.h"
0031 #include "RelationalAccess/ConnectionService.h"
0032 #include "CoralBase/Exception.h"
0033 #include "RelationalAccess/ISessionProxy.h"
0034 #include "RelationalAccess/ITransaction.h"
0035 #include "RelationalAccess/AccessMode.h"
0036 #include "RelationalAccess/ITypeConverter.h"
0037 #include "RelationalAccess/IQuery.h"
0038 #include "RelationalAccess/ICursor.h"
0039 #include "RelationalAccess/ISchema.h"
0040 #include "RelationalAccess/ITable.h"
0041 #include "CoralKernel/IPropertyManager.h"
0042 #include "RelationalAccess/AuthenticationServiceException.h"
0043 #include "RecoLuminosity/LumiProducer/interface/Exception.h"
0044 #include "RecoLuminosity/LumiProducer/interface/ConstantDef.h"
0045 #include "RecoLuminosity/LumiProducer/interface/LumiCorrectionParam.h"
0046 #include "RecoLuminosity/LumiProducer/interface/LumiCorrectionParamRcd.h"
0047 #include "RecoLuminosity/LumiProducer/interface/RevisionDML.h"
0048 #include "RecoLuminosity/LumiProducer/interface/NormDML.h"
0049 #include "RecoLuminosity/LumiProducer/interface/LumiNames.h"
0050 #include "LumiCorrectionSource.h"
0051 #include <iostream>
0052 #include <sstream>
0053 #include <string>
0054 #include <memory>
0055 #include <algorithm>
0056 #include <vector>
0057 #include <cstring>
0058 #include <iterator>
0059 #include <filesystem>
0060
0061 #include <boost/tokenizer.hpp>
0062 #include <xercesc/dom/DOM.hpp>
0063 #include <xercesc/parsers/XercesDOMParser.hpp>
0064 #include "Utilities/Xerces/interface/Xerces.h"
0065 #include <xercesc/util/XMLString.hpp>
0066
0067 std::string LumiCorrectionSource::x2s(const XMLCh* toTranscode) const {
0068   std::string tmp(xercesc::XMLString::transcode(toTranscode));
0069   return tmp;
0070 }
0071
0072 XMLCh* LumiCorrectionSource::s2x(const std::string& temp) const {
0073   XMLCh* buff = xercesc::XMLString::transcode(temp.c_str());
0074   return buff;
0075 }
0076
0077 std::string LumiCorrectionSource::toParentString(const xercesc::DOMNode& nodeToConvert) const {
0078   std::ostringstream oss;
0079   xercesc::DOMNodeList* childList = nodeToConvert.getChildNodes();
0080
0081   unsigned int numNodes = childList->getLength();
0082   for (unsigned int i = 0; i < numNodes; ++i) {
0083     xercesc::DOMNode* childNode = childList->item(i);
0084     if (childNode->getNodeType() != xercesc::DOMNode::ELEMENT_NODE)
0085       continue;
0086     xercesc::DOMElement* child = static_cast<xercesc::DOMElement*>(childNode);
0087     xercesc::DOMNamedNodeMap* attributes = child->getAttributes();
0088     unsigned int numAttributes = attributes->getLength();
0089     for (unsigned int j = 0; j < numAttributes; ++j) {
0090       xercesc::DOMNode* attributeNode = attributes->item(j);
0091       if (attributeNode->getNodeType() != xercesc::DOMNode::ATTRIBUTE_NODE)
0092         continue;
0093       xercesc::DOMAttr* attribute = static_cast<xercesc::DOMAttr*>(attributeNode);
0094
0095       oss << "(" << x2s(child->getTagName()) << x2s(attribute->getName()) << "=" << x2s(attribute->getValue()) << ")";
0096     }
0097   }
0098   return oss.str();
0099 }
0100
0101 const std::string LumiCorrectionSource::servletTranslation(const std::string& servlet) const {
0102   std::string frontierConnect;
0103   std::string realconnect;
0104   cms::concurrency::xercesInitialize();
0105   std::unique_ptr<xercesc::XercesDOMParser> parser(new xercesc::XercesDOMParser);
0106   try {
0107     parser->setValidationScheme(xercesc::XercesDOMParser::Val_Auto);
0108     parser->setDoNamespaces(false);
0109     parser->parse(m_siteconfpath.c_str());
0110     xercesc::DOMDocument* doc = parser->getDocument();
0111     if (!doc) {
0112       return "";
0113     }
0114
0115     xercesc::DOMNodeList* frontierConnectList = doc->getElementsByTagName(s2x("frontier-connect"));
0116     if (frontierConnectList->getLength() > 0) {
0117       xercesc::DOMElement* frontierConnectElement = static_cast<xercesc::DOMElement*>(frontierConnectList->item(0));
0118       frontierConnect = toParentString(*frontierConnectElement);
0119     }
0120     // Replace the last component of every "serverurl=" piece (up to the
0121     //   next close-paren) with the servlet
0122     std::string::size_type nextparen = 0;
0123     std::string::size_type serverurl, lastslash;
0124     std::string complexstr = "";
0125     while ((serverurl = frontierConnect.find("(serverurl=", nextparen)) != std::string::npos) {
0126       realconnect.append(frontierConnect, nextparen, serverurl - nextparen);
0127       nextparen = frontierConnect.find(')', serverurl);
0128       lastslash = frontierConnect.rfind('/', nextparen);
0129       realconnect.append(frontierConnect, serverurl, lastslash - serverurl + 1);
0130       realconnect.append(servlet);
0131     }
0132     realconnect.append(frontierConnect, nextparen, frontierConnect.length() - nextparen);
0133   } catch (xercesc::DOMException& e) {
0134   }
0135   return realconnect;
0136 }
0137
0138 std::string LumiCorrectionSource::translateFrontierConnect(const std::string& connectStr) {
0139   std::string result;
0140   const std::string fproto("frontier://");
0141   std::string::size_type startservlet = fproto.length();
0142   std::string::size_type endservlet = connectStr.find('(', startservlet);
0143   if (endservlet == std::string::npos) {
0144     endservlet = connectStr.rfind('/', connectStr.length());
0145   }
0146   std::string servlet = connectStr.substr(startservlet, endservlet - startservlet);
0147   if ((!servlet.empty()) && (servlet.find_first_of(":/)[]") == std::string::npos)) {
0148     if (servlet == "cms_conditions_data")
0149       servlet = "";
0150     if (m_siteconfpath.empty()) {
0151       std::string url = (std::filesystem::path("SITECONF") / std::filesystem::path("local") /
0152                          std::filesystem::path("JobConfig") / std::filesystem::path("site-local-config.xml"))
0153                             .string();
0154       char* tmp = std::getenv("CMS_PATH");
0155       if (tmp) {
0156         m_siteconfpath = (std::filesystem::path(tmp) / std::filesystem::path(url)).string();
0157       }
0158     } else {
0159       if (!std::filesystem::exists(std::filesystem::path(m_siteconfpath))) {
0160         throw cms::Exception("Non existing path ") << m_siteconfpath;
0161       }
0162       m_siteconfpath =
0163           (std::filesystem::path(m_siteconfpath) / std::filesystem::path("site-local-config.xml")).string();
0164     }
0165     result = fproto + servletTranslation(servlet) + connectStr.substr(endservlet);
0166   }
0167   return result;
0168 }
0169
0170 LumiCorrectionSource::LumiCorrectionSource(const edm::ParameterSet& iConfig)
0171     : m_connectStr(""),
0172       m_authfilename(""),
0173       m_datatag(""),
0174       m_globaltag(""),
0175       m_normtag(""),
0176       m_paramcachedrun(0),
0177       m_cachesize(0) {
0178   setWhatProduced(this, &LumiCorrectionSource::produceLumiCorrectionParam);
0179   findingRecord<LumiCorrectionParamRcd>();
0180   std::string connectStr = iConfig.getParameter<std::string>("connect");
0181   std::string globaltag;
0182   if (iConfig.exists("globaltag")) {
0183     m_globaltag = iConfig.getUntrackedParameter<std::string>("globaltag", "");
0184   } else {
0185     m_normtag = iConfig.getUntrackedParameter<std::string>("normtag", "");
0186   }
0187   m_datatag = iConfig.getUntrackedParameter<std::string>("datatag", "");
0188   m_cachesize = iConfig.getUntrackedParameter<unsigned int>("ncacheEntries", 3);
0189   m_siteconfpath = iConfig.getUntrackedParameter<std::string>("siteconfpath", "");
0190   const std::string fproto("frontier://");
0191   if (connectStr.substr(0, fproto.length()) == fproto) {
0192     m_connectStr = translateFrontierConnect(connectStr);
0193   } else if (connectStr.substr(0, 11) == "sqlite_file") {
0194     m_connectStr = connectStr;
0195   } else {
0196     m_connectStr = connectStr;
0197     std::string authpath = iConfig.getUntrackedParameter<std::string>("authpath", "");
0198     std::filesystem::path boostAuthPath(authpath);
0199     if (std::filesystem::is_directory(boostAuthPath)) {
0200       boostAuthPath /= std::filesystem::path("authentication.xml");
0201     }
0202     m_authfilename = boostAuthPath.string();
0203   }
0204 }
0205
0206 LumiCorrectionSource::ReturnParamType LumiCorrectionSource::produceLumiCorrectionParam(const LumiCorrectionParamRcd&) {
0207   unsigned int currentrun = m_pcurrentTime->eventID().run();
0208   if (currentrun == 0 || currentrun == 4294967295) {
0209     return std::make_shared<const LumiCorrectionParam>();
0210   }
0211   if (m_paramcachedrun != currentrun) {                         //i'm in a new run
0212     fillparamcache(currentrun);                                 //fill cache
0213   } else {                                                      //i'm in an old run
0214     if (m_paramcache.find(currentrun) == m_paramcache.end()) {  //i'm not cached
0215       fillparamcache(currentrun);                               //
0216     }
0217   }
0218   if (m_paramcache.empty()) {
0219     return std::make_shared<const LumiCorrectionParam>();
0220   }
0221   m_paramresult = m_paramcache[currentrun];
0222   if (m_paramresult.get() == nullptr) {
0223     return std::make_shared<const LumiCorrectionParam>();
0224   }
0225   return m_paramresult;
0226 }
0227
0228 void LumiCorrectionSource::setIntervalFor(const edm::eventsetup::EventSetupRecordKey& iKey,
0229                                           const edm::IOVSyncValue& iTime,
0230                                           edm::ValidityInterval& oValidity) {
0231   m_pcurrentTime = &iTime;
0232   oValidity.setFirst(iTime);
0233   oValidity.setLast(iTime);
0234 }
0235
0236 void LumiCorrectionSource::reloadAuth() {
0237   //std::cout<<"old authfile "<<coral::Context::instance().PropertyManager().property("AuthenticationFile")->get()<<std::endl;
0238   coral::Context::instance().PropertyManager().property("AuthenticationFile")->set(m_authfilename);
0239   coral::Context::instance().loadComponent("CORAL/Services/XMLAuthenticationService");
0240 }
0241
0242 void LumiCorrectionSource::fillparamcache(unsigned int runnumber) {
0243   m_paramcache.clear();
0244   m_paramcachedrun = runnumber;
0245   if (!m_authfilename.empty()) {
0246     coral::IHandle<coral::IAuthenticationService> authSvc =
0247         coral::Context::instance().query<coral::IAuthenticationService>();
0248     if (authSvc.isValid()) {
0249       try {
0250         authSvc->credentials(m_connectStr);
0251       } catch (const coral::UnknownConnectionException& er) {
0252         reloadAuth();
0253       }
0254     } else {
0255       reloadAuth();
0256     }
0257   }
0258   coral::ConnectionService* mydbservice = new coral::ConnectionService;
0259   if (!m_globaltag.empty()) {
0260     coral::ISessionProxy* gsession = mydbservice->connect(m_connectStr, coral::ReadOnly);
0261     gsession->transaction().start(true);
0262     parseGlobaltagForLumi(gsession->nominalSchema(), m_globaltag);
0263     gsession->transaction().commit();
0264     delete gsession;
0265   }
0266   coral::ISessionProxy* session = mydbservice->connect(m_connectStr, coral::ReadOnly);
0267   coral::ITypeConverter& tconverter = session->typeConverter();
0268   tconverter.setCppTypeForSqlType(std::string("float"), std::string("FLOAT(63)"));
0269   tconverter.setCppTypeForSqlType(std::string("unsigned int"), std::string("NUMBER(10)"));
0270   tconverter.setCppTypeForSqlType(std::string("unsigned short"), std::string("NUMBER(1)"));
0271   auto result = std::make_unique<LumiCorrectionParam>(LumiCorrectionParam::HF);
0272   try {
0273     session->transaction().start(true);
0274     coral::ISchema& schema = session->nominalSchema();
0275     lumi::RevisionDML dml;
0276     unsigned long long tagid = 0;
0277     if (m_datatag.empty()) {
0278       tagid = dml.currentHFDataTagId(schema);  //get datatag id
0279     } else {
0280       tagid = dml.HFDataTagIdByName(schema, m_datatag);
0281     }
0282     lumi::RevisionDML::DataID dataid = dml.dataIDForRun(schema, runnumber, tagid);  //get data id
0283     unsigned int lumiid = dataid.lumi_id;
0284     if (lumiid == 0) {
0285       result->setNBX(0);
0286       std::shared_ptr<const LumiCorrectionParam> const_result = std::move(result);
0287       m_paramcache.insert(std::make_pair(runnumber, const_result));
0288       session->transaction().commit();
0289       delete session;
0290       delete mydbservice;
0291       return;
0292     }
0293
0294     coral::AttributeList lumidataBindVariables;
0295     lumidataBindVariables.extend("dataid", typeid(unsigned long long));
0296     lumidataBindVariables["dataid"].data<unsigned long long>() = lumiid;
0297     std::string conditionStr("DATA_ID=:dataid");
0298     coral::AttributeList lumiparamOutput;
0299     lumiparamOutput.extend("NCOLLIDINGBUNCHES", typeid(unsigned int));
0300     coral::IQuery* lumiparamQuery = schema.newQuery();
0301     lumiparamQuery->addToTableList(std::string("LUMIDATA"));
0302     lumiparamQuery->setCondition(conditionStr, lumidataBindVariables);
0303     lumiparamQuery->addToOutputList("NCOLLIDINGBUNCHES");
0304     lumiparamQuery->defineOutput(lumiparamOutput);
0305     coral::ICursor& lumiparamcursor = lumiparamQuery->execute();
0306     unsigned int ncollidingbx = 0;
0307     while (lumiparamcursor.next()) {
0308       const coral::AttributeList& row = lumiparamcursor.currentRow();
0309       if (!row["NCOLLIDINGBUNCHES"].isNull()) {
0310         ncollidingbx = row["NCOLLIDINGBUNCHES"].data<unsigned int>();
0311       }
0312       result->setNBX(ncollidingbx);
0313     }
0314     delete lumiparamQuery;
0315     lumi::NormDML normdml;
0316     unsigned long long normid = 0;
0317     std::map<std::string, unsigned long long> normidmap;
0318     if (m_normtag.empty()) {
0319       normdml.normIdByType(schema, normidmap, lumi::NormDML::HF, true);
0320       m_normtag = normidmap.begin()->first;
0321       normid = normidmap.begin()->second;
0322     } else {
0323       normid = normdml.normIdByName(schema, m_normtag);
0324     }
0325
0326     std::map<unsigned int, lumi::NormDML::normData> normDataMap;
0327     normdml.normById(schema, normid, normDataMap);
0328
0329     std::map<unsigned int, lumi::NormDML::normData>::iterator normIt = --normDataMap.end();
0330     if (runnumber < normIt->first) {
0331       normIt = normDataMap.upper_bound(runnumber);
0332       --normIt;
0333     }
0334     result->setNormtag(m_normtag);
0335     result->setcorrFunc(normIt->second.corrfunc);
0336     result->setnonlinearCoeff(normIt->second.coefficientmap);
0337     result->setafterglows(normIt->second.afterglows);
0338     result->setdescription(normIt->second.amodetag, normIt->second.beamegev);
0339     if (normIt->second.coefficientmap["DRIFT"] != 0.) {
0340       float intglumi = fetchIntglumi(schema, runnumber);
0341       result->setintglumi(intglumi);
0342     }
0343     m_paramcache.insert(std::make_pair(runnumber, std::shared_ptr<LumiCorrectionParam>(std::move(result))));
0344     session->transaction().commit();
0345   } catch (const coral::Exception& er) {
0346     session->transaction().rollback();
0347     delete session;
0348     delete mydbservice;
0349     throw cms::Exception("DatabaseError ") << er.what();
0350   }
0351   delete session;
0352   delete mydbservice;
0353 }
0354 void LumiCorrectionSource::parseGlobaltagForLumi(coral::ISchema& schema, const std::string& globaltag) {
0355   /**select i.pfn,i.tagname from TAGINVENTORY_TABLE i,TAGTREE_TABLE_GLOBALTAG v from i.tagid=v.tagid and i.recordname='LumiCorrectionParamRcd' **/
0356   std::string tagtreetabname("TAGTREE_TABLE_");
0357   tagtreetabname += std::string(globaltag);
0358   coral::IQuery* qHandle = schema.newQuery();
0359   qHandle->addToTableList("TAGINVENTORY_TABLE", "i");
0360   qHandle->addToTableList(tagtreetabname, "v");
0361   coral::AttributeList qResult;
0362   qResult.extend("pfn", typeid(std::string));
0363   qResult.extend("tagname", typeid(std::string));
0364   std::string conditionStr("v.tagid=i.tagid and i.recordname=:recordname");
0365   coral::AttributeList qCondition;
0366   qCondition.extend("recordname", typeid(std::string));
0367   qCondition["recordname"].data<std::string>() = std::string("LumiCorrectionParamRcd");
0368   qHandle->setCondition(conditionStr, qCondition);
0369   qHandle->addToOutputList("i.pfn");
0370   qHandle->addToOutputList("i.tagname");
0371   qHandle->defineOutput(qResult);
0372   coral::ICursor& iCursor = qHandle->execute();
0373   while (iCursor.next()) {
0374     const coral::AttributeList& row = iCursor.currentRow();
0375     std::string connectStr = row["pfn"].data<std::string>();
0376     const std::string fproto("frontier://");
0377     if (connectStr.substr(0, fproto.length()) == fproto) {
0378       m_connectStr = translateFrontierConnect(connectStr);
0379     } else {
0380       m_connectStr = connectStr;
0381     }
0382     m_normtag = row["tagname"].data<std::string>();
0383   }
0384   delete qHandle;
0385 }
0386 float LumiCorrectionSource::fetchIntglumi(coral::ISchema& schema, unsigned int runnumber) {
0387   float result = 0.;
0388   coral::IQuery* qHandle = schema.newQuery();
0389   qHandle->addToTableList(lumi::LumiNames::intglumiv2TableName());
0390   coral::AttributeList qResult;
0391   qResult.extend("INTGLUMI", typeid(float));
0392   std::string conditionStr("RUNNUM=:runnumber");
0393   coral::AttributeList qCondition;
0394   qCondition.extend("runnumber", typeid(unsigned int));
0395   qCondition["runnumber"].data<unsigned int>() = runnumber;
0396   qHandle->setCondition(conditionStr, qCondition);
0397   qHandle->addToOutputList("INTGLUMI");
0398   qHandle->defineOutput(qResult);
0399   coral::ICursor& intglumiCursor = qHandle->execute();
0400   while (intglumiCursor.next()) {
0401     const coral::AttributeList& row = intglumiCursor.currentRow();
0402     if (!row["INTGLUMI"].isNull()) {
0403       result = row["INTGLUMI"].data<float>();
0404     }
0405   }
0406   delete qHandle;
0407   return result;
0408 }
0409
0410 LumiCorrectionSource::~LumiCorrectionSource() {}
0411 //define this as a plug-in
0412 DEFINE_FWK_EVENTSETUP_SOURCE(LumiCorrectionSource);