Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-02-14 13:32:01

0001 #include "OnlineDB/EcalCondDB/interface/LMFDat.h"
0002 
0003 #include <sstream>
0004 #include <cmath>
0005 
0006 using std::cout;
0007 using std::endl;
0008 
0009 LMFDat::LMFDat() : LMFUnique() {
0010   m_tableName = "";
0011   m_max = -1;
0012   _where = "";
0013   _wherePars.clear();
0014 }
0015 
0016 LMFDat::LMFDat(EcalDBConnection *c) : LMFUnique(c) {
0017   m_tableName = "";
0018   m_max = -1;
0019   _where = "";
0020   _wherePars.clear();
0021 }
0022 
0023 LMFDat::LMFDat(oracle::occi::Environment *env, oracle::occi::Connection *conn) : LMFUnique(env, conn) {
0024   m_tableName = "";
0025   m_max = -1;
0026   _where = "";
0027   _wherePars.clear();
0028 }
0029 
0030 std::string LMFDat::foreignKeyName() const { return "lmfRunIOV"; }
0031 
0032 int LMFDat::getLMFRunIOVID() {
0033   int id = getInt(foreignKeyName());
0034   if (id == 0) {
0035     // try to get it from the list of foreign keys
0036     std::map<std::string, LMFUnique *>::iterator i = m_foreignKeys.find(foreignKeyName());
0037     if (i != m_foreignKeys.end()) {
0038       LMFRunIOV *iov = (LMFRunIOV *)(i->second);
0039       if (iov != nullptr) {
0040         id = iov->fetchID();
0041         setInt(foreignKeyName(), id);
0042       }
0043     }
0044   }
0045   return id;
0046 }
0047 
0048 LMFDat &LMFDat::setMaxDataToDump(int n) {
0049   m_max = n;
0050   return *this;
0051 }
0052 
0053 std::map<unsigned int, std::string> LMFDat::getReverseMap() const {
0054   std::map<unsigned int, std::string> m;
0055   std::map<std::string, unsigned int>::const_iterator i = m_keys.begin();
0056   std::map<std::string, unsigned int>::const_iterator e = m_keys.end();
0057   while (i != e) {
0058     m[i->second] = i->first;
0059     i++;
0060   }
0061   return m;
0062 }
0063 
0064 void LMFDat::dump() const { dump(0, m_max); }
0065 
0066 void LMFDat::dump(int n) const { dump(n, m_max); }
0067 
0068 void LMFDat::dump(int n, int max) const {
0069   LMFUnique::dump(n);
0070   int s = m_data.size();
0071   cout << "Stored data: " << s << endl;
0072   if (max >= 0) {
0073     std::map<int, std::vector<float> >::const_iterator p = m_data.begin();
0074     std::map<int, std::vector<float> >::const_iterator end = m_data.end();
0075     int c = 0;
0076     std::map<unsigned int, std::string> rm = getReverseMap();
0077     while ((p != end) && (c < max)) {
0078       int id = p->first;
0079       std::vector<float> x = p->second;
0080       cout << c << " -------------------------------------------" << endl;
0081       cout << "   ID: " << id << endl;
0082       for (unsigned int j = 0; j < x.size(); j++) {
0083         if (j % 4 == 0) {
0084           cout << endl << "   ";
0085         }
0086         cout << rm[j] << ":" << x[j] << "\t";
0087       }
0088       cout << endl;
0089       p++;
0090       c++;
0091     }
0092   }
0093 }
0094 
0095 std::string LMFDat::buildInsertSql() {
0096   // create the insert statement
0097   std::stringstream sql;
0098   sql << "INSERT INTO " + getTableName() + " VALUES (";
0099   unsigned int nParameters = m_keys.size() + 2;
0100   for (unsigned int i = 0; i < nParameters - 1; i++) {
0101     sql << ":" << i + 1 << ", ";
0102   }
0103   sql << ":" << nParameters << ")";
0104   std::string sqls = sql.str();
0105   if (m_debug) {
0106     cout << m_className << "::writeDB: " << sqls << endl;
0107   }
0108   return sqls;
0109 }
0110 
0111 std::string LMFDat::getIovIdFieldName() const { return "LMF_IOV_ID"; }
0112 
0113 void LMFDat::setWhereClause(std::string where) {
0114   // to be used by experts to restrict the results of a query
0115   _where = where;
0116 }
0117 
0118 void LMFDat::setWhereClause(std::string where, const std::vector<std::string> &parameters) {
0119   // to be used by experts to restrict the results of a query
0120   // in this case the where clause can contains positional parameter,
0121   // identified as :/I, :/S, :/F for, respectively, integer, string or
0122   // float parameters. The parameters are all passed as strings
0123   // in parameters
0124   _wherePars = parameters;
0125   _where = where;
0126 }
0127 
0128 std::string LMFDat::buildSelectSql(int logic_id, int direction) {
0129   // create the insert statement
0130   // if logic_id = 0 select all channels for a given iov_id
0131   std::stringstream sql;
0132   int count = 1;
0133   if (getLMFRunIOVID() > 0) {
0134     if (_where.length() > 0) {
0135       // check if this is an expert query. If so, add a WHERE clause
0136       _where = " AND " + _where;
0137     }
0138     // in this case we are looking for all data collected during the same
0139     // IOV. There can be many logic_ids per IOV.
0140     sql << "SELECT * FROM CMS_ECAL_LASER_COND." << getTableName() << " WHERE " << getIovIdFieldName() << " = "
0141         << getLMFRunIOVID() << _where;
0142     // the expert query must be specified each time the expert makes the query
0143     // then empty it
0144     _where = "";
0145   } else {
0146     // in this case we are looking for a specific logic_id whose
0147     // data have been collected at a given time. There is only
0148     // one record in this case.
0149     std::string op = ">";
0150     std::string order = "ASC";
0151     if (direction < 0) {
0152       op = "<";
0153       order = "DESC";
0154     }
0155     sql << "SELECT * FROM (SELECT CMS_ECAL_LASER_COND." << getTableName() << ".* FROM CMS_ECAL_LASER_COND."
0156         << getTableName() << " JOIN LMF_RUN_IOV ON "
0157         << "LMF_RUN_IOV.LMF_IOV_ID = " << getTableName() << "." << getIovIdFieldName() << " "
0158         << "WHERE SUBRUN_START " << op << "= TO_DATE(:" << count;
0159     count++;
0160     sql << ", 'YYYY-MM-DD HH24:MI:SS') ORDER BY SUBRUN_START " << order << ") WHERE ROWNUM <= 1";
0161   }
0162   if (logic_id > 0) {
0163     sql << " AND LOGIC_ID = :" << count;
0164   }
0165   std::string sqls = sql.str();
0166   if (m_debug) {
0167     cout << m_className << "::buildSelectSqlDB: " << sqls << endl;
0168   }
0169   return sqls;
0170 }
0171 
0172 void LMFDat::getPrevious(LMFDat *dat) noexcept(false) { getNeighbour(dat, -1); }
0173 
0174 void LMFDat::getNext(LMFDat *dat) noexcept(false) { getNeighbour(dat, +1); }
0175 
0176 void LMFDat::getNeighbour(LMFDat *dat, int which) noexcept(false) {
0177   // there should be just one record in this case
0178   if (m_data.size() == 1) {
0179     dat->setConnection(this->getEnv(), this->getConn());
0180     int logic_id = m_data.begin()->first;
0181     Tm lastMeasuredOn = getSubrunStart();
0182     lastMeasuredOn += which;
0183     dat->fetch(logic_id, &lastMeasuredOn, which);
0184     dat->setMaxDataToDump(m_max);
0185   } else {
0186     dump();
0187     throw(std::runtime_error(m_className + "::getPrevious: Too many LOGIC_IDs in "
0188                                            "this object"));
0189   }
0190 }
0191 
0192 void LMFDat::fetch(const EcalLogicID &id) noexcept(false) { fetch(id.getLogicID()); }
0193 
0194 void LMFDat::fetch(const EcalLogicID &id, const Tm &tm) noexcept(false) { fetch(id.getLogicID(), &tm, 1); }
0195 
0196 void LMFDat::fetch(const EcalLogicID &id, const Tm &tm, int direction) noexcept(false) {
0197   setInt(foreignKeyName(), 0); /* set the LMF_IOV_ID to undefined */
0198   fetch(id.getLogicID(), &tm, direction);
0199 }
0200 
0201 void LMFDat::fetch() noexcept(false) { fetch(0); }
0202 
0203 void LMFDat::fetch(int logic_id) noexcept(false) { fetch(logic_id, nullptr, 0); }
0204 
0205 void LMFDat::fetch(int logic_id, const Tm &tm) noexcept(false) { fetch(logic_id, &tm, 1); }
0206 
0207 void LMFDat::adjustParameters(int count, std::string &sql, Statement *stmt) {
0208   // adjust positional parameters and change them according to their
0209   // decalred type
0210   std::size_t nw = 0;
0211   std::size_t n = sql.find(':');
0212   for (int done = 1; done < count; done++) {
0213     // skip already bound variables
0214     n = sql.find(':', n + 1);
0215   }
0216   while (n != std::string::npos) {
0217     char type = sql.at(n + 1);
0218     if (type == 'S') {
0219       stmt->setString(nw + count, _wherePars[nw]);
0220       nw++;
0221     } else if (type == 'F') {
0222       stmt->setFloat(nw + count, atof(_wherePars[nw].c_str()));
0223       nw++;
0224     } else if (type == 'I') {
0225       stmt->setInt(nw + count, atoi(_wherePars[nw].c_str()));
0226       nw++;
0227     }
0228     n = sql.find(':', n + 1);
0229   }
0230 }
0231 
0232 void LMFDat::fetch(int logic_id, const Tm *timestamp, int direction) noexcept(false) {
0233   bool ok = check();
0234   if ((timestamp == nullptr) && (getLMFRunIOVID() == 0)) {
0235     throw(std::runtime_error(m_className + "::fetch: Cannot fetch data with "
0236                                            "timestamp = 0 and LMFRunIOV = 0"));
0237   }
0238   if (ok && isValid()) {
0239     if (m_debug) {
0240       std::cout << "[LMFDat] This object is valid..." << std::endl;
0241     }
0242     try {
0243       Statement *stmt = m_conn->createStatement();
0244       std::string sql = buildSelectSql(logic_id, direction);
0245       if (m_debug) {
0246         std::cout << "[LMFDat] Executing query " << std::endl;
0247         std::cout << "         " << sql << std::endl << std::flush;
0248       }
0249       if (logic_id == 0) {
0250         // get data for all crystals with a given timestamp
0251         stmt->setPrefetchRowCount(10000);
0252       }
0253       stmt->setSQL(sql);
0254       int count = 1;
0255       if (logic_id > 0) {
0256         if (timestamp != nullptr) {
0257           stmt->setString(count, timestamp->str());
0258           count++;
0259         }
0260         stmt->setInt(count++, logic_id);
0261       }
0262       adjustParameters(count, sql, stmt);
0263       ResultSet *rset = stmt->executeQuery();
0264       std::vector<float> x;
0265       int nData = m_keys.size();
0266       x.reserve(nData);
0267       while (rset->next() != 0) {
0268         for (int i = 0; i < nData; i++) {
0269           x.push_back(rset->getFloat(i + 3));
0270         }
0271         int id = rset->getInt(2);
0272         if (timestamp != nullptr) {
0273           setInt(foreignKeyName(), rset->getInt(1));
0274         }
0275         this->setData(id, x);
0276         x.clear();
0277       }
0278       stmt->setPrefetchRowCount(0);
0279       m_conn->terminateStatement(stmt);
0280     } catch (oracle::occi::SQLException &e) {
0281       throw(std::runtime_error(m_className + "::fetch: " + e.getMessage()));
0282     }
0283     m_ID = m_data.size();
0284   }
0285 }
0286 
0287 bool LMFDat::isValid() {
0288   bool ret = true;
0289   if (m_foreignKeys.find(foreignKeyName()) == m_foreignKeys.end()) {
0290     ret = false;
0291     m_Error += " Can't find lmfRunIOV within foreign keys.";
0292     if (m_debug) {
0293       cout << m_className << ": Foreign keys map size: " << m_foreignKeys.size() << endl;
0294     }
0295   }
0296   return ret;
0297 }
0298 
0299 std::map<int, std::vector<float> > LMFDat::fetchData() noexcept(false) {
0300   // see if any of the data is already in the database
0301   std::map<int, std::vector<float> > s = m_data;
0302   std::string sql =
0303       "SELECT LOGIC_ID FROM CMS_ECAL_LASER_COND." + getTableName() + " WHERE " + getIovIdFieldName() + " = :1";
0304   if (m_debug) {
0305     cout << m_className << ":: candidate data items to be written = " << s.size() << endl;
0306     cout << m_className << "   Executing " << sql;
0307     cout << " where " << getIovIdFieldName() << " = " << getLMFRunIOVID() << endl;
0308   }
0309   try {
0310     Statement *stmt = m_conn->createStatement();
0311     stmt->setSQL(sql);
0312     stmt->setInt(1, getLMFRunIOVID());
0313     stmt->setPrefetchRowCount(10000);
0314     ResultSet *rset = stmt->executeQuery();
0315     std::map<int, std::vector<float> >::iterator i = s.end();
0316     std::map<int, std::vector<float> >::iterator e = s.end();
0317     while (rset->next() != 0) {
0318       if (m_debug) {
0319         cout << m_className << ":: checking " << rset->getInt(1) << endl << std::flush;
0320       }
0321       i = s.find(rset->getInt(1));
0322       if (i != e) {
0323         s.erase(i);
0324       }
0325     }
0326     stmt->setPrefetchRowCount(0);
0327     m_conn->terminateStatement(stmt);
0328   } catch (oracle::occi::SQLException &e) {
0329     throw(std::runtime_error(m_className + "::fetchData:  " + e.getMessage()));
0330   }
0331   if (m_debug) {
0332     cout << m_className << ":: data items to write = " << s.size() << endl;
0333   }
0334   return s;
0335 }
0336 
0337 int LMFDat::writeDB() noexcept(false) {
0338   // first of all check if data already present
0339   if (m_debug) {
0340     cout << m_className << ": Writing foreign keys" << endl;
0341   }
0342   LMFUnique::writeForeignKeys();
0343   if (m_debug) {
0344     cout << m_className << ": Foreign keys written" << endl;
0345   }
0346   // write data on the database
0347   int ret = 0;
0348   std::map<int, std::vector<float> > data2write = fetchData();
0349   if (!data2write.empty()) {
0350     this->checkConnection();
0351     bool ok = check();
0352     // write
0353     if (ok && isValid()) {
0354       std::list<dvoid *> bufPointers;
0355       int nParameters = m_keys.size();
0356       int nData = data2write.size();
0357       if (m_debug) {
0358         cout << m_className << ": # data items = " << nData << endl;
0359         cout << m_className << ": # parameters = " << nParameters << endl;
0360       }
0361       int *iovid_vec = new int[nData];
0362       int *logicid_vec = new int[nData];
0363       int *intArray = new int[nData];
0364       float *floatArray = new float[nData];
0365       ub2 *intSize = new ub2[nData];
0366       ub2 *floatSize = new ub2[nData];
0367       size_t intTotalSize = sizeof(int) * nData;
0368       size_t floatTotalSize = sizeof(float) * nData;
0369       try {
0370         Statement *stmt = m_conn->createStatement();
0371         std::string sql = buildInsertSql();
0372         stmt->setSQL(sql);
0373         // build the array of the size of each column
0374         for (int i = 0; i < nData; i++) {
0375           intSize[i] = sizeof(int);
0376           floatSize[i] = sizeof(int);
0377         }
0378         // build the data array for first column: the same run iov id
0379         LMFRunIOV *runiov = (LMFRunIOV *)m_foreignKeys[foreignKeyName()];
0380         int iov_id = runiov->getID();
0381         std::map<int, std::vector<float> >::const_iterator b = data2write.begin();
0382         std::map<int, std::vector<float> >::const_iterator e = data2write.end();
0383         for (int i = 0; i < nData; i++) {
0384           iovid_vec[i] = iov_id;
0385         }
0386         stmt->setDataBuffer(1, (dvoid *)iovid_vec, oracle::occi::OCCIINT, sizeof(iovid_vec[0]), intSize);
0387         // build the data array for second column: the logic ids
0388         int c = 0;
0389         while (b != e) {
0390           int id = b->first;
0391           logicid_vec[c++] = id;
0392           b++;
0393         }
0394         stmt->setDataBuffer(2, (dvoid *)logicid_vec, oracle::occi::OCCIINT, sizeof(logicid_vec[0]), intSize);
0395         // for each column build the data array
0396         oracle::occi::Type type = oracle::occi::OCCIFLOAT;
0397         for (int i = 0; i < nParameters; i++) {
0398           b = data2write.begin();
0399           // loop on all logic ids
0400           c = 0;
0401           while (b != e) {
0402             std::vector<float> x = b->second;
0403             if (m_type[i] == "INT") {
0404               intArray[c] = (int)rint(x[i]);
0405             } else if ((m_type[i] == "FLOAT") || (m_type[i] == "NUMBER")) {
0406               floatArray[c] = x[i];
0407             } else {
0408               throw(std::runtime_error("ERROR: LMFDat::writeDB: unsupported type"));
0409             }
0410             c++;
0411             b++;
0412           }
0413           // copy data into a "permanent" buffer
0414           dvoid *buffer;
0415           type = oracle::occi::OCCIINT;
0416           ub2 *sizeArray = intSize;
0417           int size = sizeof(intArray[0]);
0418           if ((m_type[i] == "FLOAT") || (m_type[i] == "NUMBER")) {
0419             buffer = (dvoid *)malloc(sizeof(float) * nData);
0420             memcpy(buffer, floatArray, floatTotalSize);
0421             type = oracle::occi::OCCIFLOAT;
0422             sizeArray = floatSize;
0423             size = sizeof(floatArray[0]);
0424           } else {
0425             buffer = (dvoid *)malloc(sizeof(int) * nData);
0426             memcpy(buffer, intArray, intTotalSize);
0427           }
0428           bufPointers.push_back(buffer);
0429           if (m_debug) {
0430             for (int k = 0; ((k < nData) && (k < m_max)); k++) {
0431               cout << m_className << ": === Index=== " << k << endl;
0432               cout << m_className << ": RUN_IOV_ID = " << iovid_vec[k] << endl;
0433               cout << m_className << ": LOGIC_ID = " << logicid_vec[k] << endl;
0434               cout << m_className << ": FIELD " << i << ": " << ((float *)(buffer))[k] << endl;
0435             }
0436           }
0437           stmt->setDataBuffer(i + 3, buffer, type, size, sizeArray);
0438         }
0439         stmt->executeArrayUpdate(nData);
0440         delete[] intArray;
0441         delete[] floatArray;
0442         delete[] intSize;
0443         delete[] floatSize;
0444         delete[] logicid_vec;
0445         delete[] iovid_vec;
0446         std::list<dvoid *>::const_iterator bi = bufPointers.begin();
0447         std::list<dvoid *>::const_iterator be = bufPointers.end();
0448         while (bi != be) {
0449           free(*bi);
0450           bi++;
0451         }
0452         m_conn->commit();
0453         m_conn->terminateStatement(stmt);
0454         ret = nData;
0455       } catch (oracle::occi::SQLException &e) {
0456         debug();
0457         setMaxDataToDump(nData);
0458         // get the Foreign Key
0459         LMFRunIOV *runiov = (LMFRunIOV *)m_foreignKeys[foreignKeyName()];
0460         int iov_id = runiov->getID();
0461         std::cout << "==== This object refers to IOV " << iov_id << std::endl;
0462         dump();
0463         m_conn->rollback();
0464         throw(std::runtime_error(m_className + "::writeDB: " + e.getMessage()));
0465       }
0466     } else {
0467       cout << m_className << "::writeDB: Cannot write because " << m_Error << endl;
0468       dump();
0469     }
0470   }
0471   return ret;
0472 }
0473 
0474 void LMFDat::getKeyTypes() noexcept(false) {
0475   m_type.reserve(m_keys.size());
0476   for (unsigned int i = 0; i < m_keys.size(); i++) {
0477     m_type.push_back("");
0478   }
0479   // get the description of the table
0480   std::string sql = "";
0481   try {
0482     Statement *stmt = m_conn->createStatement();
0483     sql = "SELECT * FROM TABLE(CMS_ECAL_LASER_COND.LMF_TAB_COLS(:1, :2))";
0484     /*
0485     sql = "SELECT COLUMN_NAME, DATA_TYPE FROM " 
0486       "USER_TAB_COLS WHERE TABLE_NAME = '" + getTableName() + "' " 
0487       "AND COLUMN_NAME != '" + getIovIdFieldName() +  "' AND COLUMN_NAME != " 
0488       "'LOGIC_ID'";
0489     */
0490     stmt->setSQL(sql);
0491     stmt->setString(1, getTableName());
0492     stmt->setString(2, getIovIdFieldName());
0493     ResultSet *rset = stmt->executeQuery();
0494     while (rset->next() != 0) {
0495       std::string name = rset->getString(1);
0496       std::string t = rset->getString(2);
0497       m_type[m_keys[name]] = t;
0498     }
0499     m_conn->terminateStatement(stmt);
0500   } catch (oracle::occi::SQLException &e) {
0501     throw(std::runtime_error(m_className + "::getKeyTypes: " + e.getMessage() + " [" + sql + "]"));
0502   }
0503 }
0504 
0505 bool LMFDat::check() {
0506   // check that everything has been correctly setup
0507   bool ret = true;
0508   m_Error = "";
0509   // first of all we need to check that the class name has been set
0510   if (m_className == "LMFUnique") {
0511     m_Error = "class name not set ";
0512     ret = false;
0513   }
0514   //then check that the table name has been set
0515   if (getTableName().empty()) {
0516     m_Error += "table name not set ";
0517     ret = false;
0518   }
0519   // fill key types if not yet done
0520   if (m_type.size() != m_keys.size()) {
0521     getKeyTypes();
0522     if (m_type.size() != m_keys.size()) {
0523       m_Error += "key size does not correspond to table definition";
0524       ret = false;
0525     }
0526   }
0527   return ret;
0528 }
0529 
0530 /* unsafe methods */
0531 
0532 std::vector<float> LMFDat::getData(int id) {
0533   std::vector<float> ret;
0534   if (m_data.find(id) != m_data.end()) {
0535     ret = m_data[id];
0536   }
0537   return ret;
0538 }
0539 
0540 std::vector<float> LMFDat::operator[](int id) { return getData(id); }
0541 
0542 std::vector<float> LMFDat::getData(const EcalLogicID &id) { return getData(id.getLogicID()); }
0543 
0544 /* safe methods */
0545 
0546 bool LMFDat::getData(int id, std::vector<float> &ret) {
0547   bool retval = false;
0548   if (m_data.find(id) != m_data.end()) {
0549     ret = m_data[id];
0550     retval = true;
0551   }
0552   return retval;
0553 }
0554 
0555 bool LMFDat::getData(const EcalLogicID &id, std::vector<float> &ret) { return getData(id.getLogicID(), ret); }
0556 
0557 /* all data */
0558 
0559 std::map<int, std::vector<float> > LMFDat::getData() { return m_data; }
0560 
0561 /* unsafe */
0562 
0563 float LMFDat::getData(int id, unsigned int k) { return m_data[id][k]; }
0564 
0565 float LMFDat::getData(const EcalLogicID &id, unsigned int k) { return getData(id.getLogicID(), k); }
0566 
0567 float LMFDat::getData(const EcalLogicID &id, const std::string &key) { return getData(id.getLogicID(), m_keys[key]); }
0568 
0569 float LMFDat::getData(int id, const std::string &key) { return getData(id, m_keys[key]); }
0570 
0571 /* safe */
0572 
0573 bool LMFDat::getData(int id, unsigned int k, float &ret) {
0574   bool retval = false;
0575   std::vector<float> v;
0576   retval = getData(id, v);
0577   if ((retval) && (v.size() > k)) {
0578     ret = v[k];
0579     retval = true;
0580   } else {
0581     retval = false;
0582   }
0583   return retval;
0584 }
0585 
0586 bool LMFDat::getData(const EcalLogicID &id, unsigned int k, float &ret) { return getData(id.getLogicID(), k, ret); }
0587 
0588 bool LMFDat::getData(int id, const std::string &key, float &ret) {
0589   bool retval = false;
0590   if (m_keys.find(key) != m_keys.end()) {
0591     retval = getData(id, m_keys[key], ret);
0592   }
0593   return retval;
0594 }
0595 
0596 bool LMFDat::getData(const EcalLogicID &id, const std::string &key, float &ret) {
0597   return getData(id.getLogicID(), key, ret);
0598 }