Geners/test/cmerge.cc

0001 // The following program merges contents of several object catalogs
0002 // stored in "geners" binary metafiles
0003
0004 #include <fstream>
0005 #include <iostream>
0006 #include <vector>
0007
0008 #include <memory>
0009 #include "Alignment/Geners/interface/CatalogIO.hh"
0010 #include "Alignment/Geners/interface/ContiguousCatalog.hh"
0011 #include "Alignment/Geners/interface/uriUtils.hh"
0012
0013 #include "CmdLine.hh"
0014
0015 using namespace gs;
0016 using namespace std;
0017
0018 static void print_usage(const char *progname) {
0019   cout << "\nUsage: " << progname << " -o output_file input_file0 input_file1 ...\n\n"
0020        << "This program merges the contents of several object catalogs stored "
0021           "in \"geners\"\n"
0022        << "binary metafiles. The output metafile can be used as a single "
0023           "\"geners\" archive\n"
0024        << "which includes all items from the merged catalogs.\n"
0025        << endl;
0026 }
0027
0028 static const char *update_uri(const char *inputfile, const std::string &originalUri) {
0029   static std::string result, old_inputfile, old_URI;
0030
0031   if (old_inputfile != inputfile || old_URI != originalUri) {
0032     old_inputfile = inputfile;
0033     old_URI = originalUri;
0034     result = joinDir1WithName2(inputfile, originalUri.c_str());
0035   }
0036   return result.c_str();
0037 }
0038
0039 static ItemLocation update_location(const char *inputfile, const ItemLocation &original) {
0040   ItemLocation loc(original);
0041   loc.setURI(update_uri(inputfile, original.URI()));
0042   return loc;
0043 }
0044
0045 int main(int argc, char const *argv[]) {
0046   CmdLine cmdline(argc, argv);
0047
0048   if (argc == 1) {
0049     print_usage(cmdline.progname());
0050     return 0;
0051   }
0052
0053   std::string outputfile;
0054   std::vector<std::string> inputfiles;
0055
0056   try {
0057     cmdline.require("-o") >> outputfile;
0058     cmdline.optend();
0059
0060     while (cmdline) {
0061       std::string s;
0062       cmdline >> s;
0063       inputfiles.push_back(s);
0064     }
0065
0066     if (inputfiles.empty())
0067       throw CmdLineError("must specify at least one input file");
0068   } catch (CmdLineError &e) {
0069     cerr << "Error in " << cmdline.progname() << ": " << e.str() << endl;
0070     print_usage(cmdline.progname());
0071     return 1;
0072   }
0073
0074   // Before we perform any significant data processing,
0075   // make sure that we can open all input files
0076   const unsigned nfiles = inputfiles.size();
0077   for (unsigned ifile = 0; ifile < nfiles; ++ifile) {
0078     const std::string &inputfile(inputfiles[ifile]);
0079     ifstream in(inputfile.c_str(), ios_base::binary);
0080     if (!in.is_open()) {
0081       cerr << "Error: failed to open file \"" << inputfile << "\"" << endl;
0082       return 1;
0083     }
0084   }
0085
0086   // Variables which summarize the combined catalog
0087   unsigned totalMergeLevel = 0, lastCompressionCode = 0;
0088   bool compressionCodeMixed = false;
0089   std::vector<std::string> allAnnotations;
0090   ContiguousCatalog merged;
0091
0092   // Now, do the real cycle over the input files
0093   for (unsigned ifile = 0; ifile < nfiles; ++ifile) {
0094     const std::string &inputfile(inputfiles[ifile]);
0095     const unsigned long long fileOffset = merged.size();
0096
0097     ifstream in(inputfile.c_str(), ios_base::binary);
0098     if (!in.is_open()) {
0099       cerr << "Error: failed to open file \"" << inputfile << "\"" << endl;
0100       return 1;
0101     }
0102
0103     unsigned compressionCode = 0, mergeLevel = 0;
0104     std::vector<std::string> annotations;
0105     std::unique_ptr<ContiguousCatalog> cat;
0106     try {
0107       cat = std::unique_ptr<ContiguousCatalog>(
0108           readBinaryCatalog<ContiguousCatalog>(in, &compressionCode, &mergeLevel, &annotations, true));
0109     } catch (std::exception &e) {
0110       cerr << "Failed to read catalog from file \"" << inputfile << "\". " << e.what() << endl;
0111       return 1;
0112     }
0113
0114     // Update compression code info
0115     if (ifile)
0116       if (compressionCode != lastCompressionCode)
0117         compressionCodeMixed = true;
0118     lastCompressionCode = compressionCode;
0119
0120     // Update merge level
0121     if (mergeLevel == 0)
0122       ++mergeLevel;
0123     totalMergeLevel += mergeLevel;
0124
0125     // Update annotations
0126     std::copy(annotations.begin(), annotations.end(), std::back_inserter(allAnnotations));
0127
0128     const unsigned long long last = cat->largestId();
0129     for (unsigned long long id = cat->smallestId(); id <= last; ++id) {
0130       if (!cat->itemExists(id))
0131         continue;
0132       std::shared_ptr<const CatalogEntry> e = cat->retrieveEntry(id);
0133       const unsigned long long newid = merged.makeEntry(*e,
0134                                                         e->compressionCode(),
0135                                                         e->itemLength(),
0136                                                         update_location(inputfile.c_str(), e->location()),
0137                                                         e->offset() + fileOffset);
0138       assert(newid);
0139     }
0140   }
0141
0142   ofstream of(outputfile.c_str(), ios_base::binary);
0143   if (!of.is_open()) {
0144     cerr << "Error: failed to open file \"" << outputfile << "\"" << endl;
0145     return 1;
0146   }
0147   const unsigned compress = compressionCodeMixed ? 1 : lastCompressionCode;
0148   if (!writeBinaryCatalog(of, compress, totalMergeLevel, allAnnotations, merged)) {
0149     cerr << "Error: failed to write merged catalog to file \"" << outputfile << "\"" << endl;
0150     return 1;
0151   }
0152   of.close();
0153
0154   return 0;
0155 }