Streamer/bin/DiagStreamerFile.cpp

0001 /** Sample code to Read Streammer and recover file without bad events
0002     This is meant to debug streamer data files.
0003
0004    compares_bad:
0005        Compares the streamer header info for two events and return true
0006        if any header information that should be the same is different
0007
0008   uncompressBuffer:
0009        Tries to uncompress the event data blob if it was compressed
0010        and return true if successful (or was not compressed)
0011
0012   readfile:
0013        Reads a streamer file, dumps the headers for the INIT message
0014        and the first event, and then looks to see if there are any
0015        events with streamer header problems or uncompress problems
0016        optionally writes a streamer file without bad events
0017
0018   main():
0019
0020       Code entry point, comment the function call that you don't want to make.
0021
0022 */
0023
0024 #include "FWCore/Utilities/interface/Adler32Calculator.h"
0025 #include "FWCore/Utilities/interface/Exception.h"
0026 #include "IOPool/Streamer/interface/DumpTools.h"
0027 #include "IOPool/Streamer/interface/EventMessage.h"
0028 #include "IOPool/Streamer/interface/InitMessage.h"
0029 #include "IOPool/Streamer/interface/MsgTools.h"
0030 #include "IOPool/Streamer/interface/StreamerInputFile.h"
0031 #include "IOPool/Streamer/interface/StreamerOutputFile.h"
0032
0033 #include "zlib.h"
0034
0035 #include <iostream>
0036 #include <map>
0037 #include <memory>
0038
0039 using namespace edm::streamer;
0040
0041 namespace {
0042   bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2);
0043   bool uncompressBuffer(unsigned char* inputBuffer,
0044                         unsigned int inputSize,
0045                         std::vector<unsigned char>& outputBuffer,
0046                         unsigned int expectedFullSize);
0047   bool test_chksum(EventMsgView const* eview);
0048   bool test_uncompress(EventMsgView const* eview, std::vector<unsigned char>& dest);
0049   void readfile(std::string filename, std::string outfile);
0050   void help();
0051 }  // namespace
0052 //==========================================================================
0053 int main(int argc, char* argv[]) {
0054   if (argc < 2) {
0055     std::cout << "No command line argument supplied\n";
0056     help();
0057     return 1;
0058   }
0059
0060   std::string streamfile(argv[1]);
0061   std::string outfile("/dev/null");
0062   if (argc == 3) {
0063     outfile = argv[2];
0064   }
0065
0066   readfile(streamfile, outfile);
0067   std::cout << "\n\nDiagStreamerFile TEST DONE\n" << std::endl;
0068
0069   return 0;
0070 }
0071
0072 namespace {
0073   //==========================================================================
0074   void help() {
0075     std::cout << "Usage: DiagStreamerFile streamer_file_name"
0076               << " [output_file_name]" << std::endl;
0077   }
0078   //==========================================================================
0079   void readfile(std::string filename, std::string outfile) {
0080     uint32 num_events(0);
0081     uint32 num_badevents(0);
0082     uint32 num_baduncompress(0);
0083     uint32 num_badchksum(0);
0084     uint32 num_goodevents(0);
0085     uint32 num_duplevents(0);
0086     std::vector<unsigned char> compress_buffer(7000000);
0087     std::map<uint64, uint32> seenEventMap;
0088     bool output(false);
0089     if (outfile != "/dev/null") {
0090       output = true;
0091     }
0092     StreamerOutputFile stream_output(outfile);
0093     try {
0094       // ----------- init
0095       StreamerInputFile stream_reader(filename);
0096       //if(output) StreamerOutputFile stream_output(outfile);
0097
0098       std::cout << "Trying to Read The Init message from Streamer File: " << std::endl << filename << std::endl;
0099       InitMsgView const* init = stream_reader.startMessage();
0100       std::cout << "\n\n-------------INIT Message---------------------" << std::endl;
0101       std::cout << "Dump the Init Message from Streamer:-" << std::endl;
0102       dumpInitView(init);
0103       if (output) {
0104         stream_output.write(*init);
0105       }
0106
0107       // ------- event
0108       std::cout << "\n\n-------------EVENT Messages-------------------" << std::endl;
0109
0110       bool first_event(true);
0111       std::unique_ptr<EventMsgView> firstEvtView(nullptr);
0112       std::vector<unsigned char> savebuf(0);
0113       EventMsgView const* eview(nullptr);
0114       seenEventMap.clear();
0115
0116       while (StreamerInputFile::Next::kEvent == stream_reader.next()) {
0117         eview = stream_reader.currentRecord();
0118         ++num_events;
0119         bool good_event(true);
0120         if (seenEventMap.find(eview->event()) == seenEventMap.end()) {
0121           seenEventMap.insert(std::make_pair(eview->event(), 1));
0122         } else {
0123           ++seenEventMap[eview->event()];
0124           ++num_duplevents;
0125           std::cout << "??????? duplicate event Id for count " << num_events << " event number " << eview->event()
0126                     << " seen " << seenEventMap[eview->event()] << " times" << std::endl;
0127         }
0128         if (first_event) {
0129           std::cout << "----------dumping first EVENT-----------" << std::endl;
0130           dumpEventView(eview);
0131           first_event = false;
0132           unsigned char* src = (unsigned char*)eview->startAddress();
0133           unsigned int srcSize = eview->size();
0134           savebuf.resize(srcSize);
0135           std::copy(src, src + srcSize, &(savebuf)[0]);
0136           firstEvtView = std::make_unique<EventMsgView>(&(savebuf)[0]);
0137           //firstEvtView, reset(new EventMsgView((void*)eview->startAddress()));
0138           if (!test_chksum(eview)) {
0139             std::cout << "checksum error for count " << num_events << " event number " << eview->event()
0140                       << " from host name " << eview->hostName() << std::endl;
0141             ++num_badchksum;
0142             std::cout << "----------dumping bad checksum EVENT-----------" << std::endl;
0143             dumpEventView(eview);
0144             good_event = false;
0145           }
0146           if (!test_uncompress(eview, compress_buffer)) {
0147             std::cout << "uncompress error for count " << num_events << " event number " << firstEvtView->event()
0148                       << std::endl;
0149             ++num_baduncompress;
0150             std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl;
0151             dumpEventView(firstEvtView.get());
0152             good_event = false;
0153           }
0154         } else {
0155           if (compares_bad(firstEvtView.get(), eview)) {
0156             std::cout << "Bad event at count " << num_events << " dumping event " << std::endl
0157                       << "----------dumping bad EVENT-----------" << std::endl;
0158             dumpEventView(eview);
0159             ++num_badevents;
0160             good_event = false;
0161           }
0162           if (!test_chksum(eview)) {
0163             std::cout << "checksum error for count " << num_events << " event number " << eview->event()
0164                       << " from host name " << eview->hostName() << std::endl;
0165             ++num_badchksum;
0166             std::cout << "----------dumping bad checksum EVENT-----------" << std::endl;
0167             dumpEventView(eview);
0168             good_event = false;
0169           }
0170           if (!test_uncompress(eview, compress_buffer)) {
0171             std::cout << "uncompress error for count " << num_events << " event number " << eview->event() << std::endl;
0172             ++num_baduncompress;
0173             std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl;
0174             dumpEventView(eview);
0175             good_event = false;
0176           }
0177         }
0178         if (good_event) {
0179           if (output) {
0180             ++num_goodevents;
0181             stream_output.write(*eview);
0182           }
0183           //dumpEventView(eview);
0184         }
0185         if ((num_events % 50) == 0) {
0186           std::cout << "Read " << num_events << " events, and " << num_badevents << " events with bad headers, and "
0187                     << num_badchksum << " events with bad check sum, and " << num_baduncompress
0188                     << " events with bad uncompress" << std::endl;
0189           if (output)
0190             std::cout << "Wrote " << num_goodevents << " good events " << std::endl;
0191         }
0192       }
0193
0194       std::cout << std::endl
0195                 << "------------END--------------" << std::endl
0196                 << "read " << num_events << " events" << std::endl
0197                 << "and " << num_badevents << " events with bad headers" << std::endl
0198                 << "and " << num_badchksum << " events with bad check sum" << std::endl
0199                 << "and " << num_baduncompress << " events with bad uncompress" << std::endl
0200                 << "and " << num_duplevents << " duplicated event Id" << std::endl;
0201
0202       if (output) {
0203         std::cout << "Wrote " << num_goodevents << " good events " << std::endl;
0204       }
0205
0206     } catch (cms::Exception& e) {
0207       std::cerr << "Exception caught:  " << e.what() << std::endl
0208                 << "After reading " << num_events << " events, and " << num_badevents << " events with bad headers"
0209                 << std::endl
0210                 << "and " << num_badchksum << " events with bad check sum" << std::endl
0211                 << "and " << num_baduncompress << " events with bad uncompress" << std::endl
0212                 << "and " << num_duplevents << " duplicated event Id" << std::endl;
0213     }
0214   }
0215
0216   //==========================================================================
0217   bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2) {
0218     bool is_bad(false);
0219     if (eview1->code() != eview2->code()) {
0220       std::cout << "non-matching EVENT message code " << std::endl;
0221       is_bad = true;
0222     }
0223     if (eview1->protocolVersion() != eview2->protocolVersion()) {
0224       std::cout << "non-matching EVENT message protocol version" << std::endl;
0225       is_bad = true;
0226     }
0227     if (eview1->run() != eview2->run()) {
0228       std::cout << "non-matching run number " << std::endl;
0229       is_bad = true;
0230     }
0231     if (eview1->lumi() != eview2->lumi()) {
0232       std::cout << "non-matching lumi number" << std::endl;
0233       is_bad = true;
0234     }
0235     if (eview1->outModId() != eview2->outModId()) {
0236       std::cout << "non-matching output module id" << std::endl;
0237       is_bad = true;
0238     }
0239     if (eview1->hltCount() != eview2->hltCount()) {
0240       std::cout << "non-matching HLT count" << std::endl;
0241       is_bad = true;
0242     }
0243     if (eview1->l1Count() != eview2->l1Count()) {
0244       std::cout << "non-matching L1 count" << std::endl;
0245       is_bad = true;
0246     }
0247     return is_bad;
0248   }
0249
0250   //==========================================================================
0251   bool test_chksum(EventMsgView const* eview) {
0252     uint32_t adler32_chksum = cms::Adler32((char const*)eview->eventData(), eview->eventLength());
0253     //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl;
0254     //std::cout << "Adler32 checksum from header = " << eview->adler32_chksum() << std::endl;
0255     //std::cout << "event from host name = " << eview->hostName() << std::endl;
0256     if ((uint32)adler32_chksum != eview->adler32_chksum()) {
0257       std::cout << "Bad chekcsum: Adler32 checksum of event data  = " << adler32_chksum
0258                 << " from header = " << eview->adler32_chksum() << " host name = " << eview->hostName() << std::endl;
0259       return false;
0260     }
0261     return true;
0262   }
0263
0264   //==========================================================================
0265   bool test_uncompress(EventMsgView const* eview, std::vector<unsigned char>& dest) {
0266     unsigned long origsize = eview->origDataSize();
0267     bool success = false;
0268     if (origsize != 0 && origsize != 78) {
0269       // compressed
0270       success = uncompressBuffer(
0271           const_cast<unsigned char*>((unsigned char const*)eview->eventData()), eview->eventLength(), dest, origsize);
0272     } else {
0273       // uncompressed anyway
0274       success = true;
0275     }
0276     return success;
0277   }
0278
0279   //==========================================================================
0280   bool uncompressBuffer(unsigned char* inputBuffer,
0281                         unsigned int inputSize,
0282                         std::vector<unsigned char>& outputBuffer,
0283                         unsigned int expectedFullSize) {
0284     unsigned long origSize = expectedFullSize;
0285     unsigned long uncompressedSize = expectedFullSize * 1.1;
0286     outputBuffer.resize(uncompressedSize);
0287     int ret = uncompress(&outputBuffer[0], &uncompressedSize, inputBuffer, inputSize);
0288     if (ret == Z_OK) {
0289       // check the length against original uncompressed length
0290       if (origSize != uncompressedSize) {
0291         std::cout << "Problem with uncompress, original size = " << origSize
0292                   << " uncompress size = " << uncompressedSize << std::endl;
0293         return false;
0294       }
0295     } else {
0296       std::cout << "Problem with uncompress, return value = " << ret << std::endl;
0297       return false;
0298     }
0299     return true;
0300   }
0301 }  // namespace