StorageFactory/test/t0Repack.cpp

0001 #include "FWCore/Utilities/interface/Exception.h"
0002 #include "Utilities/StorageFactory/test/Test.h"
0003 #include "Utilities/StorageFactory/interface/Storage.h"
0004 #include <iostream>
0005 #include <cmath>
0006 #include <limits>
0007 #include <string>
0008 #include <vector>
0009 #include <sstream>
0010 #include <memory>
0011
0012 int main(int argc, char **argv) try {
0013   initTest();
0014
0015   if (argc < 4) {
0016     std::cerr << "usage: " << argv[0] << " NUM-DATASETS OUTPUT-FILE INDEX-FILE...\n";
0017     return EXIT_FAILURE;
0018   }
0019
0020   int datasetN = ::atoi(argv[1]);
0021   std::string outputURL = argv[2];
0022
0023   using namespace edm::storage;
0024   std::unique_ptr<Storage> outputFile = 0;
0025   IOSize totSize = 0;
0026   std::vector<std::unique_ptr<Storage>> indexFiles;
0027   std::vector<IOOffset> indexSizes;
0028
0029   StorageFactory::getToModify()->enableAccounting(true);
0030   std::cerr << "write to file " << outputURL << " " << datasetN << " datasets\n";
0031
0032   for (int i = 3; i < argc; ++i) {
0033     IOOffset size = -1;
0034     if (StorageFactory::get()->check(argv[i], &size)) {
0035       indexFiles.push_back(StorageFactory::get()->open(argv[i], IOFlags::OpenRead));
0036       indexSizes.push_back(size);
0037     } else {
0038       std::cerr << "index file " << argv[i] << " does not exist\n";
0039       return EXIT_FAILURE;
0040     }
0041   }
0042
0043   // open output file
0044   try {
0045     outputFile =
0046         StorageFactory::get()->open(outputURL, IOFlags::OpenWrite | IOFlags::OpenCreate | IOFlags::OpenTruncate);
0047   } catch (cms::Exception &e) {
0048     std::cerr << "error in opening output file " << outputURL << ": " << e.explainSelf() << std::endl;
0049     return EXIT_FAILURE;
0050   }
0051
0052   // parse index file, read buffer, select and copy to output file
0053   for (size_t i = 0; i < indexFiles.size(); ++i) {
0054     // suck in the index file.
0055     std::cout << "reading from index file " << argv[i + 3] << std::endl;
0056     std::istringstream in;
0057     try {
0058       std::vector<char> lbuf(indexSizes[i] + 1, '\0');
0059       IOSize nn = indexFiles[i]->read(&lbuf[0], indexSizes[i]);
0060
0061       if (indexSizes[i] < 0 || static_cast<IOOffset>(nn) != indexSizes[i]) {
0062         std::cerr << "error in reading from index file " << argv[i + 3] << std::endl;
0063         std::cerr << "asked for " << indexSizes[i] << " bytes, got " << nn << " bytes\n";
0064         return EXIT_FAILURE;
0065       }
0066
0067       in.str(&lbuf[0]);
0068     } catch (cms::Exception &e) {
0069       std::cerr << "error in reading from index file " << argv[i + 3] << std::endl << e.explainSelf() << std::endl;
0070       return EXIT_FAILURE;
0071     } catch (...) {
0072       std::cerr << "error in reading from index file " << argv[i + 3] << std::endl;
0073       return EXIT_FAILURE;
0074     }
0075
0076     std::string line1;
0077     std::getline(in, line1);
0078     std::cout << "first line is '" << line1 << "'\n";
0079     std::string::size_type pos = line1.find('=');
0080     if (pos != std::string::npos)
0081       pos = line1.find_first_not_of(' ', pos + 1);
0082     if (pos == std::string::npos) {
0083       std::cerr << "badly formed index file " << argv[i + 3] << std::endl;
0084       std::cerr << "first line is:\n" << line1 << std::endl;
0085       return EXIT_FAILURE;
0086     }
0087     line1.erase(0, pos);
0088
0089     std::unique_ptr<Storage> s;
0090     IOOffset size = 0;
0091     try {
0092       std::cout << "input event file " << i << " is " << line1 << std::endl;
0093       if (StorageFactory::get()->check(line1, &size))
0094         s = StorageFactory::get()->open(line1, IOFlags::OpenRead);
0095       else {
0096         std::cerr << "input file " << line1 << " does not exist\n";
0097         return EXIT_FAILURE;
0098       }
0099     } catch (cms::Exception &e) {
0100       std::cerr << "error in opening input file " << line1 << std::endl << e.explainSelf() << std::endl;
0101       return EXIT_FAILURE;
0102     } catch (...) {
0103       std::cerr << "error in opening input file " << line1 << std::endl;
0104       return EXIT_FAILURE;
0105     }
0106
0107     std::vector<char> vbuf;
0108     while (in) {
0109       int dataset = -1;
0110       IOOffset bufLoc = -1;
0111       IOSize bufSize = 0;
0112
0113       in >> dataset >> bufLoc >> bufSize;
0114
0115       if (dataset != datasetN)
0116         continue;
0117
0118       std::cout << "copy buf at " << bufLoc << " of size " << bufSize << std::endl;
0119       if (bufSize > vbuf.size())
0120         vbuf.resize(bufSize);
0121
0122       char *buf = &vbuf[0];
0123       try {
0124         s->position(bufLoc);
0125         IOSize n = s->read(buf, bufSize);
0126         totSize += n;
0127         if (n != bufSize) {
0128           std::cerr << "error in reading from input file " << line1 << std::endl;
0129           std::cerr << "asked for " << bufSize << " bytes, got " << n << " bytes\n";
0130           return EXIT_FAILURE;
0131         }
0132       } catch (cms::Exception &e) {
0133         std::cerr << "error in reading input file " << line1 << std::endl << e.explainSelf() << std::endl;
0134         return EXIT_FAILURE;
0135       } catch (...) {
0136         std::cerr << "error in reading input file " << line1 << std::endl;
0137         return EXIT_FAILURE;
0138       }
0139
0140       try {
0141         outputFile->write(buf, bufSize);
0142       } catch (cms::Exception &e) {
0143         std::cerr << "error in writing output file " << outputURL << std::endl << e.explainSelf() << std::endl;
0144         return EXIT_FAILURE;
0145       } catch (...) {
0146         std::cerr << "error in writing output file " << outputURL << std::endl;
0147         return EXIT_FAILURE;
0148       }
0149     }
0150
0151     s->close();
0152   }
0153
0154   outputFile->close();
0155
0156   std::cout << "copied a total of " << totSize << " bytes" << std::endl;
0157   std::cout << StorageAccount::summaryText(true) << std::endl;
0158   return EXIT_SUCCESS;
0159 } catch (cms::Exception const &e) {
0160   std::cerr << e.explainSelf() << std::endl;
0161   return EXIT_FAILURE;
0162 } catch (std::exception const &e) {
0163   std::cerr << e.what() << std::endl;
0164   return EXIT_FAILURE;
0165 }