File indexing completed on 2024-04-06 11:56:20
0001
0002
0003
0004 #ifndef GENERS_ROWPACKER_HH_
0005 #define GENERS_ROWPACKER_HH_
0006
0007 #include "Alignment/Geners/interface/CPP11_config.hh"
0008 #ifdef CPP11_STD_AVAILABLE
0009
0010 #include <utility>
0011
0012 #include "Alignment/Geners/interface/AbsArchive.hh"
0013 #include "Alignment/Geners/interface/ClassId.hh"
0014 #include "Alignment/Geners/interface/CharBuffer.hh"
0015 #include "Alignment/Geners/interface/RPHeaderRecord.hh"
0016 #include "Alignment/Geners/interface/RPBufferRecord.hh"
0017 #include "Alignment/Geners/interface/RPFooterRecord.hh"
0018 #include "Alignment/Geners/interface/RPBufferReference.hh"
0019 #include "Alignment/Geners/interface/RPReference.hh"
0020
0021 namespace gs {
0022 template<typename Pack>
0023 class RowPacker
0024 {
0025 template<typename Pack2> friend class RowPacker;
0026 static const unsigned defaultBufferSize = 800000U;
0027
0028 public:
0029 typedef Pack value_type;
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040
0041
0042
0043
0044
0045
0046
0047
0048
0049
0050
0051
0052
0053
0054
0055
0056 RowPacker(const std::vector<std::string>& columnNames,
0057 const char* title, AbsArchive& archive,
0058 const char* name, const char* category,
0059 unsigned bufferSize = defaultBufferSize);
0060
0061
0062
0063
0064
0065
0066 RowPacker(const char* title, AbsArchive& archive,
0067 const char* name, const char* category,
0068 const Pack& protoPack,
0069 unsigned bufferSize = defaultBufferSize);
0070
0071
0072
0073
0074
0075
0076 RowPacker(AbsArchive& archive,
0077 const char* name, const char* category,
0078 unsigned bufferSize = defaultBufferSize);
0079
0080 inline ~RowPacker() {write();}
0081
0082
0083 inline AbsArchive& archive() const {return ar_;}
0084 inline const std::string& name() const {return name_;}
0085 inline const std::string& category() const {return category_;}
0086 inline unsigned bufferSize() const {return bufferSize_;}
0087 inline bool isReadable() const {return readable_;}
0088 inline bool isWritable() const {return writable_;}
0089 inline const std::string& title() const {return title_;}
0090
0091
0092
0093 inline unsigned long objectNumber() const {return objectNumber_;}
0094
0095
0096 inline void setTitle(const char* newtitle)
0097 {title_ = newtitle ? newtitle : "";}
0098 inline void setBufferSize(const unsigned newsize)
0099 {bufferSize_ = newsize;}
0100
0101
0102 inline unsigned long nRows() const {return fillCount_;}
0103
0104 inline unsigned long nColumns() const
0105 {return std::tuple_size<Pack>::value;}
0106
0107 inline const std::string& columnName(const unsigned long i) const
0108 {return colNames_.at(i);}
0109
0110 inline const std::vector<std::string>& columnNames() const
0111 {return colNames_;}
0112
0113 unsigned long columnNumber(const char* columnName) const;
0114
0115
0116
0117
0118
0119 bool setColumnName(unsigned long i, const char* newname);
0120
0121
0122
0123 bool fill(const Pack& tuple);
0124
0125
0126
0127 void rowContents(unsigned long row, Pack* tuple) const;
0128
0129 template<typename Pack2>
0130 bool operator==(const RowPacker<Pack2>& r) const;
0131
0132 template<typename Pack2>
0133 inline bool operator!=(const RowPacker<Pack2>& r) const
0134 {return !(*this == r);}
0135
0136
0137 bool write();
0138 inline ClassId classId() const {return ClassId(*this);}
0139
0140 static const char* classname();
0141 static inline unsigned version() {return 1;}
0142
0143 private:
0144 friend class Private::RPHeaderRecord<RowPacker<Pack> >;
0145 friend class Private::RPBufferRecord<RowPacker<Pack> >;
0146 friend class Private::RPFooterRecord<RowPacker<Pack> >;
0147 friend class Private::RPBufferReference<RowPacker<Pack> >;
0148 friend class RPReference<RowPacker<Pack> >;
0149
0150 RowPacker();
0151 RowPacker(const RowPacker&);
0152 RowPacker& operator=(const RowPacker&);
0153
0154
0155 static RowPacker* read(AbsArchive& ar,
0156 std::istream& is,
0157 unsigned long long headId);
0158
0159 static unsigned long nextObjectNumber();
0160
0161 void prepareToUnpack() const;
0162 bool unpackTuple(std::istream& is, Pack* tuple) const;
0163 std::istream& getRowStream(unsigned long row,
0164 unsigned long* len = 0) const;
0165
0166 bool loadRowData(unsigned long rowNumber) const;
0167 void saveHeader();
0168 void saveFillBuffer();
0169
0170 mutable CharBuffer fillBuffer_;
0171 unsigned long firstFillBufferRow_;
0172 std::vector<std::streampos> fillBufferOffsets_;
0173
0174 mutable CharBuffer readBuffer_;
0175 mutable unsigned long firstReadBufferRow_;
0176 mutable std::vector<std::streampos> readBufferOffsets_;
0177 ClassId bufferClass_;
0178 ClassId thisClass_;
0179
0180 AbsArchive& ar_;
0181 std::vector<std::string> colNames_;
0182
0183
0184
0185 std::vector<std::pair<unsigned long,unsigned long long> > idlist_;
0186
0187 std::string name_;
0188 std::string category_;
0189 std::string title_;
0190 unsigned long long headerSaved_;
0191 unsigned long bufferSize_;
0192 unsigned long fillCount_;
0193 unsigned long objectNumber_;
0194 bool readable_;
0195 bool writable_;
0196 mutable bool firstUnpack_;
0197 bool unused_;
0198 mutable std::vector<std::vector<ClassId> > iostack_;
0199
0200 static const std::vector<std::string>& defaultColumnNames();
0201 };
0202 }
0203
0204 #include <memory>
0205 #include <cstring>
0206 #include <algorithm>
0207
0208 #include "Alignment/Geners/interface/tupleIO.hh"
0209 #include "Alignment/Geners/interface/PackerIOCycle.hh"
0210 #include "Alignment/Geners/interface/RPFooterReference.hh"
0211 #include "Alignment/Geners/interface/collectTupleNames.hh"
0212 #include "Alignment/Geners/interface/allUnique.hh"
0213 #include "Alignment/Geners/interface/findName.hh"
0214 #include "Alignment/Geners/interface/IOException.hh"
0215
0216 namespace gs {
0217 template <typename T>
0218 unsigned long RowPacker<T>::nextObjectNumber()
0219 {
0220 static unsigned long ocounter = 0;
0221 return ocounter++;
0222 }
0223
0224
0225 template <typename T>
0226 unsigned long RowPacker<T>::columnNumber(const char* columnName) const
0227 {
0228 return findName(colNames_, columnName);
0229 }
0230
0231
0232 template<typename T>
0233 inline RowPacker<T>::RowPacker(const std::vector<std::string>& colNames,
0234 const char* ititle, AbsArchive& iarchive,
0235 const char* iname, const char* icategory,
0236 const unsigned ibufferSize)
0237 : firstFillBufferRow_(0),
0238 firstReadBufferRow_(0),
0239 bufferClass_(fillBuffer_.classId()),
0240 thisClass_(ClassId::makeId<RowPacker<T> >()),
0241 ar_(iarchive),
0242 colNames_(colNames),
0243 name_(iname ? iname : ""),
0244 category_(icategory ? icategory : ""),
0245 title_(ititle ? ititle : ""),
0246 headerSaved_(0),
0247 bufferSize_(ibufferSize),
0248 fillCount_(0),
0249 objectNumber_(nextObjectNumber()),
0250 readable_(ar_.isReadable()),
0251 writable_(ar_.isWritable()),
0252 firstUnpack_(true)
0253 {
0254 if (!std::tuple_size<T>::value) throw gs::IOInvalidArgument(
0255 "In RowPacker constructor: can not use empty tuple");
0256 if (std::tuple_size<T>::value != colNames.size())
0257 throw gs::IOInvalidArgument("In RowPacker constructor: "
0258 "wrong # of column names");
0259 if (!allUnique(colNames_)) throw gs::IOInvalidArgument(
0260 "In RowPacker constructor: all column names must be unique");
0261 }
0262
0263
0264 template<typename T>
0265 inline RowPacker<T>::RowPacker(const char* ititle, AbsArchive& iarchive,
0266 const char* iname, const char* icategory,
0267 const T& protoPack,
0268 const unsigned ibufferSize)
0269 : firstFillBufferRow_(0),
0270 firstReadBufferRow_(0),
0271 bufferClass_(fillBuffer_.classId()),
0272 thisClass_(ClassId::makeId<RowPacker<T> >()),
0273 ar_(iarchive),
0274 colNames_(collectTupleNames(protoPack)),
0275 name_(iname ? iname : ""),
0276 category_(icategory ? icategory : ""),
0277 title_(ititle ? ititle : ""),
0278 headerSaved_(0),
0279 bufferSize_(ibufferSize),
0280 fillCount_(0),
0281 objectNumber_(nextObjectNumber()),
0282 readable_(ar_.isReadable()),
0283 writable_(ar_.isWritable()),
0284 firstUnpack_(true)
0285 {
0286 if (!std::tuple_size<T>::value) throw gs::IOInvalidArgument(
0287 "In RowPacker constructor: can not use empty tuple");
0288 if (!allUnique(colNames_)) throw gs::IOInvalidArgument(
0289 "In RowPacker constructor: all column names must be unique");
0290 }
0291
0292
0293
0294
0295 template <typename T>
0296 inline RowPacker<T>::RowPacker(AbsArchive& iarchive,
0297 const char* iname, const char* icategory,
0298 const unsigned ibufferSize)
0299 : firstFillBufferRow_(0),
0300 firstReadBufferRow_(0),
0301 bufferClass_(fillBuffer_.classId()),
0302 thisClass_(ClassId::makeId<RowPacker<T> >()),
0303 ar_(iarchive),
0304 colNames_(defaultColumnNames()),
0305 name_(iname ? iname : ""),
0306 category_(icategory ? icategory : ""),
0307 title_(""),
0308 headerSaved_(0),
0309 bufferSize_(ibufferSize),
0310 fillCount_(0),
0311 objectNumber_(nextObjectNumber()),
0312 readable_(ar_.isReadable()),
0313 writable_(ar_.isWritable()),
0314 firstUnpack_(true)
0315 {
0316 if (!std::tuple_size<T>::value) throw gs::IOInvalidArgument(
0317 "In RowPacker constructor: can not use empty tuple");
0318 }
0319
0320
0321 template <typename T>
0322 inline void RowPacker<T>::saveHeader()
0323 {
0324 if (!headerSaved_ && writable_)
0325 {
0326 Private::RPHeaderRecord<RowPacker<T> > record(*this);
0327 ar_ << record;
0328 headerSaved_ = record.id();
0329
0330
0331
0332 assert(headerSaved_);
0333 }
0334 }
0335
0336
0337 template <typename T>
0338 void RowPacker<T>::saveFillBuffer()
0339 {
0340 saveHeader();
0341 Private::RPBufferRecord<RowPacker<T> > record(*this);
0342 ar_ << record;
0343 idlist_.push_back(std::make_pair(firstFillBufferRow_, record.id()));
0344 fillBuffer_.seekp(0);
0345 firstFillBufferRow_ = fillCount_;
0346 fillBufferOffsets_.clear();
0347 }
0348
0349
0350 template <typename Pack>
0351 bool RowPacker<Pack>::fill(const Pack& tuple)
0352 {
0353 if (!writable_)
0354 return false;
0355
0356 if (fillBuffer_.size() > bufferSize_)
0357 saveFillBuffer();
0358
0359 fillBufferOffsets_.push_back(fillBuffer_.tellp());
0360 const bool wstatus = write_item(fillBuffer_, tuple, false);
0361 if (wstatus)
0362 ++fillCount_;
0363 return wstatus;
0364 }
0365
0366
0367 template <typename T>
0368 const char* RowPacker<T>::classname()
0369 {
0370 static const std::string myClass(
0371 template_class_name<T>("gs::RowPacker"));
0372 return myClass.c_str();
0373 }
0374
0375
0376 template <typename T>
0377 bool RowPacker<T>::write()
0378 {
0379 if (!writable_)
0380 return false;
0381 saveHeader();
0382 if (!fillBufferOffsets_.empty())
0383 saveFillBuffer();
0384 ar_ << Private::RPFooterRecord<RowPacker<T> >(*this);
0385 writable_ = false;
0386 return true;
0387 }
0388
0389
0390
0391
0392 template <typename T>
0393 bool RowPacker<T>::loadRowData(const unsigned long rowNumber) const
0394 {
0395 assert(readable_);
0396 if (rowNumber >= fillCount_)
0397 throw gs::IOOutOfRange("In RowPacker::loadRowData: "
0398 "row number is out of range");
0399 if (rowNumber >= firstFillBufferRow_ &&
0400 rowNumber < firstFillBufferRow_ + fillBufferOffsets_.size())
0401 return true;
0402 if (rowNumber >= firstReadBufferRow_ &&
0403 rowNumber < firstReadBufferRow_ + readBufferOffsets_.size())
0404 return false;
0405
0406
0407
0408 const unsigned long nSaved = idlist_.size();
0409 unsigned long bucket = std::lower_bound(
0410 idlist_.begin(), idlist_.end(), std::make_pair(rowNumber, 0ULL)) -
0411 idlist_.begin();
0412 if (bucket == nSaved)
0413 --bucket;
0414 else if (idlist_[bucket].first != rowNumber)
0415 --bucket;
0416 const Private::RPBufferReference<RowPacker<T> >& ref =
0417 Private::RPBufferReference<RowPacker<T> >(
0418 *this, idlist_.at(bucket).second);
0419 ref.restore(0);
0420 return false;
0421 }
0422
0423
0424 template <typename Pack>
0425 void RowPacker<Pack>::prepareToUnpack() const
0426 {
0427
0428 std::vector<std::vector<ClassId> > dummy;
0429 thisClass_.templateParameters(&dummy);
0430 assert(dummy.size() == 1U);
0431 dummy[0].at(0).templateParameters(&iostack_);
0432 assert(iostack_.size() == std::tuple_size<Pack>::value);
0433 }
0434
0435
0436 template <typename Pack>
0437 inline bool RowPacker<Pack>::unpackTuple(
0438 std::istream& is, Pack* pack) const
0439 {
0440 if (firstUnpack_)
0441 {
0442 prepareToUnpack();
0443 firstUnpack_ = false;
0444 }
0445 return Private::PackerIOCycle<Pack,std::tuple_size<Pack>::value>::read(
0446 pack, is, iostack_);
0447 }
0448
0449
0450 template <typename T>
0451 std::istream& RowPacker<T>::getRowStream(
0452 const unsigned long row, unsigned long* len) const
0453 {
0454 if (loadRowData(row))
0455 {
0456
0457 const unsigned long idx = row - firstFillBufferRow_;
0458 fillBuffer_.clear();
0459 fillBuffer_.seekg(fillBufferOffsets_.at(idx));
0460 assert(!fillBuffer_.fail());
0461 if (len)
0462 {
0463 const unsigned long thispos = fillBufferOffsets_[idx];
0464 if (idx + 1UL < fillBufferOffsets_.size())
0465 {
0466 const unsigned long nextpos = fillBufferOffsets_[idx+1UL];
0467 *len = nextpos - thispos;
0468 }
0469 else
0470 *len = fillBuffer_.size() - thispos;
0471 }
0472 return fillBuffer_;
0473 }
0474 else
0475 {
0476
0477 const unsigned long idx = row - firstReadBufferRow_;
0478 readBuffer_.clear();
0479 readBuffer_.seekg(readBufferOffsets_.at(idx));
0480 assert(!readBuffer_.fail());
0481 if (len)
0482 {
0483 const unsigned long thispos = readBufferOffsets_[idx];
0484 if (idx + 1UL < readBufferOffsets_.size())
0485 {
0486 const unsigned long nextpos = readBufferOffsets_[idx+1UL];
0487 *len = nextpos - thispos;
0488 }
0489 else
0490 *len = readBuffer_.size() - thispos;
0491 }
0492 return readBuffer_;
0493 }
0494 }
0495
0496
0497 template <typename T>
0498 void RowPacker<T>::rowContents(const unsigned long row, T* tuple) const
0499 {
0500 if (row < fillCount_)
0501 {
0502 assert(tuple);
0503 if (!unpackTuple(getRowStream(row), tuple))
0504 throw IOInvalidData("In gs::RowPacker::rowContents: "
0505 "failed to unpack tuple data");
0506 }
0507 else
0508 throw gs::IOOutOfRange("In gs::RowPacker::rowContents: "
0509 "row number is out of range");
0510 }
0511
0512
0513 template <typename T>
0514 template <typename Pack2>
0515 bool RowPacker<T>::operator==(const RowPacker<Pack2>& r) const
0516 {
0517 if ((void *)this == (void *)(&r))
0518 return true;
0519 if (!readable_ || !r.readable_)
0520 return false;
0521 if (nColumns() != r.nColumns())
0522 return false;
0523 if (fillCount_ != r.fillCount_)
0524 return false;
0525 if (thisClass_.name() != r.thisClass_.name())
0526 return false;
0527 if (title_ != r.title_)
0528 return false;
0529 if (colNames_ != r.colNames_)
0530 return false;
0531 for (unsigned long row=0; row<fillCount_; ++row)
0532 {
0533 unsigned long len1=0, len2=0;
0534 std::istream& s1 = getRowStream(row, &len1);
0535 std::istream& s2 = r.getRowStream(row, &len2);
0536 if (len1 != len2)
0537 return false;
0538 std::streambuf* buf1 = s1.rdbuf();
0539 std::streambuf* buf2 = s2.rdbuf();
0540 unsigned long i=0;
0541 for (; i<len1 && buf1->sbumpc() == buf2->sbumpc(); ++i) {;}
0542 if (i < len1) return false;
0543 }
0544 return true;
0545 }
0546
0547
0548 template <typename T>
0549 RowPacker<T>* RowPacker<T>::read(AbsArchive& ar,
0550 std::istream& is,
0551 const unsigned long long headerId)
0552 {
0553 static const ClassId current(ClassId::makeId<RowPacker<T> >());
0554
0555 if (!ar.isReadable()) throw gs::IOInvalidArgument(
0556 "In RowPacker::read: archive not readable");
0557 if (!headerId) throw gs::IOInvalidArgument(
0558 "In RowPacker::read: invalid header record id");
0559 std::shared_ptr<const CatalogEntry> headerRecord =
0560 ar.catalogEntry(headerId);
0561 if (!headerRecord.get()) throw gs::IOInvalidArgument(
0562 "In RowPacker::read: header record not found");
0563 if (headerRecord->id() != headerId) throw IOInvalidData(
0564 "In RowPacker::read: recorded header id does not match catalog id");
0565
0566
0567 ClassId packerClass(is, 1);
0568 current.ensureSameName(packerClass);
0569
0570 ClassId bufferClass(is, 1);
0571 std::vector<std::string> columnNames;
0572 read_pod_vector(is, &columnNames);
0573 std::string titl;
0574 read_pod(is, &titl);
0575 unsigned long bufSiz;
0576 read_pod(is, &bufSiz);
0577 if (is.fail())
0578 throw IOReadFailure("In RowPacker::read: input stream failure");
0579
0580
0581
0582 Private::RPFooterReference footerInfo(
0583 ar, packerClass, headerRecord->name().c_str(),
0584 headerRecord->category().c_str());
0585 const unsigned long nfooters = footerInfo.size();
0586
0587
0588
0589 if (nfooters == 0)
0590 throw IOInvalidData("In RowPacker::read: footer record not found");
0591
0592 unsigned long nrows = 0;
0593 unsigned long long writtenHeaderId = 0;
0594 std::vector<std::pair<unsigned long,unsigned long long> > bufferIds;
0595 unsigned long long offset = 0;
0596
0597
0598 for (unsigned long ifoot = 0; ifoot < nfooters; ++ifoot)
0599 {
0600 footerInfo.fillItems(
0601 &nrows, &writtenHeaderId, &bufferIds, &offset, ifoot);
0602 if (writtenHeaderId + offset == headerId)
0603 break;
0604 }
0605 if (writtenHeaderId + offset != headerId)
0606 throw IOInvalidData("In RowPacker::read: "
0607 "incompatible footer record");
0608
0609
0610 if (offset)
0611 {
0612 const unsigned long nbuf = bufferIds.size();
0613 for (unsigned long ibuf=0; ibuf<nbuf; ++ibuf)
0614 bufferIds[ibuf].second += offset;
0615 }
0616
0617
0618 std::unique_ptr<RowPacker<T> > nt(new RowPacker<T>(
0619 columnNames, titl.c_str(), ar, headerRecord->name().c_str(),
0620 headerRecord->category().c_str(), bufSiz));
0621
0622 nt->bufferClass_ = bufferClass;
0623 nt->thisClass_ = packerClass;
0624 nt->headerSaved_ = writtenHeaderId;
0625 nt->fillCount_ = nrows;
0626 nt->writable_ = false;
0627
0628 const bool buffersEmpty = bufferIds.empty();
0629 if (!(nrows ? !buffersEmpty : buffersEmpty))
0630 throw IOInvalidData("In RowPacker::read: "
0631 "corrupted footer record");
0632 if (nrows)
0633 {
0634 nt->idlist_ = bufferIds;
0635 nt->firstFillBufferRow_ = nrows;
0636 }
0637
0638 return nt.release();
0639 }
0640
0641
0642 template<typename Pack>
0643 inline const std::vector<std::string>&
0644 RowPacker<Pack>::defaultColumnNames()
0645 {
0646 return default_tuple_columns<std::tuple_size<Pack>::value>();
0647 }
0648
0649
0650 template<typename Pack>
0651 bool RowPacker<Pack>::setColumnName(unsigned long i, const char* newname)
0652 {
0653 const unsigned long n = colNames_.size();
0654 if (i >= n)
0655 return false;
0656 if (columnNumber(newname) < n)
0657 return false;
0658 colNames_[i] = newname;
0659 return true;
0660 }
0661 }
0662
0663
0664 #endif
0665 #endif
0666