Streamer/bin/DiagStreamerFile.cpp

0001 /** Sample code to Read Streammer and recover file without bad events
0002     This is meant to debug streamer data files.
0003
0004    compares_bad:
0005        Compares the streamer header info for two events and return true
0006        if any header information that should be the same is different
0007
0008   uncompressBuffer:
0009        Tries to uncompress the event data blob if it was compressed
0010        and return true if successful (or was not compressed)
0011
0012   readfile:
0013        Reads a streamer file, dumps the headers for the INIT message
0014        and the first event, and then looks to see if there are any
0015        events with streamer header problems or uncompress problems
0016        optionally writes a streamer file without bad events
0017
0018   main():
0019
0020       Code entry point, comment the function call that you don't want to make.
0021
0022 */
0023
0024 #include "FWCore/Utilities/interface/Adler32Calculator.h"
0025 #include "FWCore/Utilities/interface/Exception.h"
0026 #include "IOPool/Streamer/interface/DumpTools.h"
0027 #include "IOPool/Streamer/interface/EventMessage.h"
0028 #include "IOPool/Streamer/interface/InitMessage.h"
0029 #include "IOPool/Streamer/interface/MsgTools.h"
0030 #include "IOPool/Streamer/interface/StreamerInputFile.h"
0031 #include "IOPool/Streamer/interface/StreamerOutputFile.h"
0032
0033 #include "zlib.h"
0034
0035 #include <iostream>
0036 #include <map>
0037 #include <memory>
0038
0039 bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2);
0040 bool uncompressBuffer(unsigned char* inputBuffer,
0041                       unsigned int inputSize,
0042                       std::vector<unsigned char>& outputBuffer,
0043                       unsigned int expectedFullSize);
0044 bool test_chksum(EventMsgView const* eview);
0045 bool test_uncompress(EventMsgView const* eview, std::vector<unsigned char>& dest);
0046 void readfile(std::string filename, std::string outfile);
0047 void help();
0048
0049 //==========================================================================
0050 int main(int argc, char* argv[]) {
0051   if (argc < 2) {
0052     std::cout << "No command line argument supplied\n";
0053     help();
0054     return 1;
0055   }
0056
0057   std::string streamfile(argv[1]);
0058   std::string outfile("/dev/null");
0059   if (argc == 3) {
0060     outfile = argv[2];
0061   }
0062
0063   readfile(streamfile, outfile);
0064   std::cout << "\n\nDiagStreamerFile TEST DONE\n" << std::endl;
0065
0066   return 0;
0067 }
0068
0069 //==========================================================================
0070 void help() {
0071   std::cout << "Usage: DiagStreamerFile streamer_file_name"
0072             << " [output_file_name]" << std::endl;
0073 }
0074 //==========================================================================
0075 void readfile(std::string filename, std::string outfile) {
0076   uint32 num_events(0);
0077   uint32 num_badevents(0);
0078   uint32 num_baduncompress(0);
0079   uint32 num_badchksum(0);
0080   uint32 num_goodevents(0);
0081   uint32 num_duplevents(0);
0082   std::vector<unsigned char> compress_buffer(7000000);
0083   std::map<uint64, uint32> seenEventMap;
0084   bool output(false);
0085   if (outfile != "/dev/null") {
0086     output = true;
0087   }
0088   StreamerOutputFile stream_output(outfile);
0089   try {
0090     // ----------- init
0091     edm::StreamerInputFile stream_reader(filename);
0092     //if(output) StreamerOutputFile stream_output(outfile);
0093
0094     std::cout << "Trying to Read The Init message from Streamer File: " << std::endl << filename << std::endl;
0095     InitMsgView const* init = stream_reader.startMessage();
0096     std::cout << "\n\n-------------INIT Message---------------------" << std::endl;
0097     std::cout << "Dump the Init Message from Streamer:-" << std::endl;
0098     dumpInitView(init);
0099     if (output) {
0100       stream_output.write(*init);
0101     }
0102
0103     // ------- event
0104     std::cout << "\n\n-------------EVENT Messages-------------------" << std::endl;
0105
0106     bool first_event(true);
0107     std::unique_ptr<EventMsgView> firstEvtView(nullptr);
0108     std::vector<unsigned char> savebuf(0);
0109     EventMsgView const* eview(nullptr);
0110     seenEventMap.clear();
0111
0112     while (edm::StreamerInputFile::Next::kEvent == stream_reader.next()) {
0113       eview = stream_reader.currentRecord();
0114       ++num_events;
0115       bool good_event(true);
0116       if (seenEventMap.find(eview->event()) == seenEventMap.end()) {
0117         seenEventMap.insert(std::make_pair(eview->event(), 1));
0118       } else {
0119         ++seenEventMap[eview->event()];
0120         ++num_duplevents;
0121         std::cout << "??????? duplicate event Id for count " << num_events << " event number " << eview->event()
0122                   << " seen " << seenEventMap[eview->event()] << " times" << std::endl;
0123       }
0124       if (first_event) {
0125         std::cout << "----------dumping first EVENT-----------" << std::endl;
0126         dumpEventView(eview);
0127         first_event = false;
0128         unsigned char* src = (unsigned char*)eview->startAddress();
0129         unsigned int srcSize = eview->size();
0130         savebuf.resize(srcSize);
0131         std::copy(src, src + srcSize, &(savebuf)[0]);
0132         firstEvtView = std::make_unique<EventMsgView>(&(savebuf)[0]);
0133         //firstEvtView, reset(new EventMsgView((void*)eview->startAddress()));
0134         if (!test_chksum(eview)) {
0135           std::cout << "checksum error for count " << num_events << " event number " << eview->event()
0136                     << " from host name " << eview->hostName() << std::endl;
0137           ++num_badchksum;
0138           std::cout << "----------dumping bad checksum EVENT-----------" << std::endl;
0139           dumpEventView(eview);
0140           good_event = false;
0141         }
0142         if (!test_uncompress(eview, compress_buffer)) {
0143           std::cout << "uncompress error for count " << num_events << " event number " << firstEvtView->event()
0144                     << std::endl;
0145           ++num_baduncompress;
0146           std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl;
0147           dumpEventView(firstEvtView.get());
0148           good_event = false;
0149         }
0150       } else {
0151         if (compares_bad(firstEvtView.get(), eview)) {
0152           std::cout << "Bad event at count " << num_events << " dumping event " << std::endl
0153                     << "----------dumping bad EVENT-----------" << std::endl;
0154           dumpEventView(eview);
0155           ++num_badevents;
0156           good_event = false;
0157         }
0158         if (!test_chksum(eview)) {
0159           std::cout << "checksum error for count " << num_events << " event number " << eview->event()
0160                     << " from host name " << eview->hostName() << std::endl;
0161           ++num_badchksum;
0162           std::cout << "----------dumping bad checksum EVENT-----------" << std::endl;
0163           dumpEventView(eview);
0164           good_event = false;
0165         }
0166         if (!test_uncompress(eview, compress_buffer)) {
0167           std::cout << "uncompress error for count " << num_events << " event number " << eview->event() << std::endl;
0168           ++num_baduncompress;
0169           std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl;
0170           dumpEventView(eview);
0171           good_event = false;
0172         }
0173       }
0174       if (good_event) {
0175         if (output) {
0176           ++num_goodevents;
0177           stream_output.write(*eview);
0178         }
0179         //dumpEventView(eview);
0180       }
0181       if ((num_events % 50) == 0) {
0182         std::cout << "Read " << num_events << " events, and " << num_badevents << " events with bad headers, and "
0183                   << num_badchksum << " events with bad check sum, and " << num_baduncompress
0184                   << " events with bad uncompress" << std::endl;
0185         if (output)
0186           std::cout << "Wrote " << num_goodevents << " good events " << std::endl;
0187       }
0188     }
0189
0190     std::cout << std::endl
0191               << "------------END--------------" << std::endl
0192               << "read " << num_events << " events" << std::endl
0193               << "and " << num_badevents << " events with bad headers" << std::endl
0194               << "and " << num_badchksum << " events with bad check sum" << std::endl
0195               << "and " << num_baduncompress << " events with bad uncompress" << std::endl
0196               << "and " << num_duplevents << " duplicated event Id" << std::endl;
0197
0198     if (output) {
0199       std::cout << "Wrote " << num_goodevents << " good events " << std::endl;
0200     }
0201
0202   } catch (cms::Exception& e) {
0203     std::cerr << "Exception caught:  " << e.what() << std::endl
0204               << "After reading " << num_events << " events, and " << num_badevents << " events with bad headers"
0205               << std::endl
0206               << "and " << num_badchksum << " events with bad check sum" << std::endl
0207               << "and " << num_baduncompress << " events with bad uncompress" << std::endl
0208               << "and " << num_duplevents << " duplicated event Id" << std::endl;
0209   }
0210 }
0211
0212 //==========================================================================
0213 bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2) {
0214   bool is_bad(false);
0215   if (eview1->code() != eview2->code()) {
0216     std::cout << "non-matching EVENT message code " << std::endl;
0217     is_bad = true;
0218   }
0219   if (eview1->protocolVersion() != eview2->protocolVersion()) {
0220     std::cout << "non-matching EVENT message protocol version" << std::endl;
0221     is_bad = true;
0222   }
0223   if (eview1->run() != eview2->run()) {
0224     std::cout << "non-matching run number " << std::endl;
0225     is_bad = true;
0226   }
0227   if (eview1->lumi() != eview2->lumi()) {
0228     std::cout << "non-matching lumi number" << std::endl;
0229     is_bad = true;
0230   }
0231   if (eview1->outModId() != eview2->outModId()) {
0232     std::cout << "non-matching output module id" << std::endl;
0233     is_bad = true;
0234   }
0235   if (eview1->hltCount() != eview2->hltCount()) {
0236     std::cout << "non-matching HLT count" << std::endl;
0237     is_bad = true;
0238   }
0239   if (eview1->l1Count() != eview2->l1Count()) {
0240     std::cout << "non-matching L1 count" << std::endl;
0241     is_bad = true;
0242   }
0243   return is_bad;
0244 }
0245
0246 //==========================================================================
0247 bool test_chksum(EventMsgView const* eview) {
0248   uint32_t adler32_chksum = cms::Adler32((char const*)eview->eventData(), eview->eventLength());
0249   //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl;
0250   //std::cout << "Adler32 checksum from header = " << eview->adler32_chksum() << std::endl;
0251   //std::cout << "event from host name = " << eview->hostName() << std::endl;
0252   if ((uint32)adler32_chksum != eview->adler32_chksum()) {
0253     std::cout << "Bad chekcsum: Adler32 checksum of event data  = " << adler32_chksum
0254               << " from header = " << eview->adler32_chksum() << " host name = " << eview->hostName() << std::endl;
0255     return false;
0256   }
0257   return true;
0258 }
0259
0260 //==========================================================================
0261 bool test_uncompress(EventMsgView const* eview, std::vector<unsigned char>& dest) {
0262   unsigned long origsize = eview->origDataSize();
0263   bool success = false;
0264   if (origsize != 0 && origsize != 78) {
0265     // compressed
0266     success = uncompressBuffer(
0267         const_cast<unsigned char*>((unsigned char const*)eview->eventData()), eview->eventLength(), dest, origsize);
0268   } else {
0269     // uncompressed anyway
0270     success = true;
0271   }
0272   return success;
0273 }
0274
0275 //==========================================================================
0276 bool uncompressBuffer(unsigned char* inputBuffer,
0277                       unsigned int inputSize,
0278                       std::vector<unsigned char>& outputBuffer,
0279                       unsigned int expectedFullSize) {
0280   unsigned long origSize = expectedFullSize;
0281   unsigned long uncompressedSize = expectedFullSize * 1.1;
0282   outputBuffer.resize(uncompressedSize);
0283   int ret = uncompress(&outputBuffer[0], &uncompressedSize, inputBuffer, inputSize);
0284   if (ret == Z_OK) {
0285     // check the length against original uncompressed length
0286     if (origSize != uncompressedSize) {
0287       std::cout << "Problem with uncompress, original size = " << origSize << " uncompress size = " << uncompressedSize
0288                 << std::endl;
0289       return false;
0290     }
0291   } else {
0292     std::cout << "Problem with uncompress, return value = " << ret << std::endl;
0293     return false;
0294   }
0295   return true;
0296 }