SherpaInterface/src/SherpackUtilities.cc

0001 #include "GeneratorInterface/SherpaInterface/interface/SherpackUtilities.h"
0002 #include "Utilities/OpenSSL/interface/openssl_init.h"
0003 #include <unistd.h>
0004 #include <cstdlib>
0005 namespace spu {
0006
0007   // functions for inflating (and deflating)
0008
0009   //~ /* Compress from file source to file dest until EOF on source.
0010   //~ def() returns Z_OK on success, Z_MEM_ERROR if memory could not be
0011   //~ allocated for processing, Z_STREAM_ERROR if an invalid compression
0012   //~ level is supplied, Z_VERSION_ERROR if the version of zlib.h and the
0013   //~ version of the library linked do not match, or Z_ERRNO if there is
0014   //~ an error reading or writing the files. */
0015   int def(FILE *source, FILE *dest, int level) {
0016     int ret, flush;
0017     unsigned have;
0018     z_stream strm;
0019     unsigned char in[CHUNK];
0020     unsigned char out[CHUNK];
0021
0022     /* allocate deflate state */
0023     strm.zalloc = Z_NULL;
0024     strm.zfree = Z_NULL;
0025     strm.opaque = Z_NULL;
0026     ret = deflateInit(&strm, level);
0027     if (ret != Z_OK)
0028       return ret;
0029
0030     /* compress until end of file */
0031     do {
0032       strm.avail_in = fread(in, 1, CHUNK, source);
0033       if (ferror(source)) {
0034         (void)deflateEnd(&strm);
0035         return Z_ERRNO;
0036       }
0037       flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
0038       strm.next_in = in;
0039
0040       /* run deflate() on input until output buffer not full, finish
0041            compression if all of source has been read in */
0042       do {
0043         strm.avail_out = CHUNK;
0044         strm.next_out = out;
0045         ret = deflate(&strm, flush);   /* no bad return value */
0046         assert(ret != Z_STREAM_ERROR); /* state not clobbered */
0047         have = CHUNK - strm.avail_out;
0048         if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
0049           (void)deflateEnd(&strm);
0050           return Z_ERRNO;
0051         }
0052       } while (strm.avail_out == 0);
0053       assert(strm.avail_in == 0); /* all input will be used */
0054
0055       /* done when last data in file processed */
0056     } while (flush != Z_FINISH);
0057     assert(ret == Z_STREAM_END); /* stream will be complete */
0058
0059     /* clean up and return */
0060     (void)deflateEnd(&strm);
0061     return Z_OK;
0062   }
0063
0064   /* Decompress from file source to file dest until stream ends or EOF.
0065    inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be
0066    allocated for processing, Z_DATA_ERROR if the deflate data is
0067    invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and
0068    the version of the library linked do not match, or Z_ERRNO if there
0069    is an error reading or writing the files. */
0070   int inf(FILE *source, FILE *dest) {
0071     int ret;
0072     unsigned have;
0073     z_stream strm;
0074     unsigned char in[CHUNK];
0075     unsigned char out[CHUNK];
0076
0077     /* allocate inflate state */
0078     strm.zalloc = Z_NULL;
0079     strm.zfree = Z_NULL;
0080     strm.opaque = Z_NULL;
0081     strm.avail_in = 0;
0082     strm.next_in = Z_NULL;
0083     //~ ret = inflateInit(&strm,15);
0084     ret = inflateInit2(&strm, (16 + MAX_WBITS));
0085     if (ret != Z_OK)
0086       return ret;
0087
0088     /* decompress until deflate stream ends or end of file */
0089     do {
0090       strm.avail_in = fread(in, 1, CHUNK, source);
0091       if (ferror(source)) {
0092         (void)inflateEnd(&strm);
0093         return Z_ERRNO;
0094       }
0095       if (strm.avail_in == 0)
0096         break;
0097       strm.next_in = in;
0098
0099       /* run inflate() on input until output buffer not full */
0100       do {
0101         strm.avail_out = CHUNK;
0102         strm.next_out = out;
0103         ret = inflate(&strm, Z_NO_FLUSH);
0104         assert(ret != Z_STREAM_ERROR); /* state not clobbered */
0105         switch (ret) {
0106           case Z_NEED_DICT:
0107             ret = Z_DATA_ERROR;
0108             [[fallthrough]];
0109           case Z_DATA_ERROR:
0110           case Z_MEM_ERROR:
0111             (void)inflateEnd(&strm);
0112             return ret;
0113         }
0114         have = CHUNK - strm.avail_out;
0115         if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
0116           (void)inflateEnd(&strm);
0117           return Z_ERRNO;
0118         }
0119       } while (strm.avail_out == 0);
0120
0121       /* done when inflate() says it's done */
0122     } while (ret != Z_STREAM_END);
0123
0124     /* clean up and return */
0125     (void)inflateEnd(&strm);
0126     return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
0127   }
0128
0129   /* report a zlib or i/o error */
0130   void zerr(int ret) {
0131     fputs("zpipe: ", stderr);
0132     switch (ret) {
0133       case Z_ERRNO:
0134         if (ferror(stdin))
0135           fputs("error reading stdin\n", stderr);
0136         if (ferror(stdout))
0137           fputs("error writing stdout\n", stderr);
0138         break;
0139       case Z_STREAM_ERROR:
0140         fputs("invalid compression level\n", stderr);
0141         break;
0142       case Z_DATA_ERROR:
0143         fputs("invalid or incomplete deflate data\n", stderr);
0144         break;
0145       case Z_MEM_ERROR:
0146         fputs("out of memory\n", stderr);
0147         break;
0148       case Z_VERSION_ERROR:
0149         fputs("zlib version mismatch!\n", stderr);
0150     }
0151   }
0152
0153   /* compress or decompress from stdin to stdout */
0154   int Unzip(std::string infile, std::string outfile) {
0155     /////////////////////////////////////////////
0156     /////////////// BUG FIX FOR MPI /////////////
0157     /////////////////////////////////////////////
0158     const char *tmpdir = std::getenv("TMPDIR");
0159     if (tmpdir && (strlen(tmpdir) > 50)) {
0160       setenv("TMPDIR", "/tmp", true);
0161     }
0162     /////////////////////////////////////////////
0163     /////////////////////////////////////////////
0164     /////////////////////////////////////////////
0165     int ret;
0166     FILE *in = fopen(infile.c_str(), "r");
0167     if (!in)
0168       return -1;
0169     FILE *out = fopen(outfile.c_str(), "w");
0170     if (!out)
0171       return -2;
0172     /* avoid end-of-line conversions */
0173     SET_BINARY_MODE(in);
0174     SET_BINARY_MODE(out);
0175
0176     ret = inf(in, out);
0177     if (ret != Z_OK)
0178       zerr(ret);
0179
0180     fclose(in);
0181     fclose(out);
0182     return ret;
0183   }
0184
0185   // functions for untaring Sherpacks
0186   /* Parse an octal number, ignoring leading and trailing nonsense. */
0187   int parseoct(const char *p, size_t n) {
0188     int i = 0;
0189
0190     while (*p < '0' || *p > '7') {
0191       ++p;
0192       --n;
0193     }
0194     while (*p >= '0' && *p <= '7' && n > 0) {
0195       i *= 8;
0196       i += *p - '0';
0197       ++p;
0198       --n;
0199     }
0200     return (i);
0201   }
0202
0203   /* Returns true if this is 512 zero bytes. */
0204   int is_end_of_archive(const char *p) {
0205     int n;
0206     for (n = 511; n >= 0; --n)
0207       if (p[n] != '\0')
0208         return (0);
0209     return (1);
0210   }
0211
0212   /* Create a directory, including parent directories as necessary. */
0213   void create_dir(char *pathname, int mode) {
0214     char *p;
0215     int r;
0216
0217     /* Strip trailing '/' */
0218     if (pathname[strlen(pathname) - 1] == '/')
0219       pathname[strlen(pathname) - 1] = '\0';
0220
0221     /* Try creating the directory. */
0222     r = mkdir(pathname, mode);
0223
0224     if (r != 0) {
0225       /* On failure, try creating parent directory. */
0226       p = strrchr(pathname, '/');
0227       if (p != nullptr) {
0228         *p = '\0';
0229         create_dir(pathname, 0755);
0230         *p = '/';
0231         r = mkdir(pathname, mode);
0232       }
0233     }
0234     if (r != 0)
0235       fprintf(stderr, "Could not create directory %s\n", pathname);
0236   }
0237
0238   /* Create a file, including parent directory as necessary. */
0239   FILE *create_file(char *pathname, int mode) {
0240     FILE *f;
0241     f = fopen(pathname, "w+");
0242     if (f == nullptr) {
0243       /* Try creating parent dir and then creating file. */
0244       char *p = strrchr(pathname, '/');
0245       if (p != nullptr) {
0246         *p = '\0';
0247         create_dir(pathname, 0755);
0248         *p = '/';
0249         f = fopen(pathname, "w+");
0250       }
0251     }
0252     return (f);
0253   }
0254
0255   /* Verify the tar checksum. */
0256   int verify_checksum(const char *p) {
0257     int n, u = 0;
0258     for (n = 0; n < 512; ++n) {
0259       if (n < 148 || n > 155)
0260         /* Standard tar checksum adds unsigned bytes. */
0261         u += ((unsigned char *)p)[n];
0262       else
0263         u += 0x20;
0264     }
0265     return (u == parseoct(p + 148, 8));
0266   }
0267
0268   /* Extract a tar archive. */
0269   void Untar(FILE *a, const char *path) {
0270     bool longpathname = false;
0271     bool longlinkname = false;
0272     char newlongpathname[512];
0273     char newlonglinkname[512];
0274     char buff[512];
0275     FILE *f = nullptr;
0276     size_t bytes_read;
0277     int filesize;
0278
0279     printf("Extracting from %s\n", path);
0280     for (;;) {
0281       bytes_read = fread(buff, 1, 512, a);
0282       if (bytes_read < 512) {
0283         fprintf(stderr, "Short read on %s: expected 512, got %d\n", path, (int)bytes_read);
0284         return;
0285       }
0286       if (is_end_of_archive(buff)) {
0287         printf("End of %s\n", path);
0288         return;
0289       }
0290       if (!verify_checksum(buff)) {
0291         fprintf(stderr, "Checksum failure\n");
0292         return;
0293       }
0294       filesize = parseoct(buff + 124, 12);
0295       //        printf("%c %d\n",buff[156],filesize);
0296       switch (buff[156]) {
0297         case '1':
0298           printf(" Ignoring hardlink %s\n", buff);
0299           break;
0300         case '2':
0301           if (longpathname && longlinkname) {
0302             longlinkname = false;
0303             longpathname = false;
0304             printf(" Extracting symlink %s\n", newlongpathname);
0305             symlink(newlonglinkname, newlongpathname);
0306           } else if (longpathname) {
0307             longpathname = false;
0308             printf(" Extracting symlink %s\n", newlongpathname);
0309             symlink(buff + 157, newlongpathname);
0310           } else if (longlinkname) {
0311             longlinkname = false;
0312             printf(" Extracting symlink %s\n", buff);
0313             symlink(newlonglinkname, buff);
0314           } else {
0315             printf(" Extracting symlink %s\n", buff);
0316             symlink(buff + 157, buff);
0317           }
0318           break;
0319         case '3':
0320           printf(" Ignoring character device %s\n", buff);
0321           break;
0322         case '4':
0323           printf(" Ignoring block device %s\n", buff);
0324           break;
0325         case '5':
0326           if (!longpathname) {
0327             int endposition = -1;
0328             for (int k = 99; k >= 0; k--) {
0329               if (buff[k] == '\0')
0330                 endposition = k;
0331             }
0332             if (endposition == -1) {
0333               //~ printf("OLDNAME : %s\n",buff);
0334               longpathname = true;
0335               for (int k = 0; k < 100; k++) {
0336                 newlongpathname[k] = buff[k];
0337               }
0338               newlongpathname[100] = '\0';
0339               //~ printf("NEWNAME : %s\n",newlongpathname);
0340             }
0341           }
0342
0343           if (longpathname) {
0344             printf(" Extracting dir %s\n", newlongpathname);
0345             create_dir(newlongpathname, parseoct(buff + 100, 8));
0346             longpathname = false;
0347           } else {
0348             printf(" Extracting dir %s\n", buff);
0349             create_dir(buff, parseoct(buff + 100, 8));
0350           }
0351           //~ printf(" Extracting dir %s\n", buff);
0352           //~ create_dir(buff, parseoct(buff + 100, 8));
0353           filesize = 0;
0354           break;
0355         case '6':
0356           printf(" Ignoring FIFO %s\n", buff);
0357           break;
0358         case 'L':
0359           longpathname = true;
0360           //~ printf(" Long Filename found 0 %s\n", buff);
0361           //~ printf(" Long Filename found 100 %s\n", buff+100);
0362           //~ printf(" Long Filename found 108 %s\n", buff+108);
0363           //~ printf(" Long Filename found 116 %s\n", buff+116);
0364           //~ printf(" Long Filename found 124 %s\n", buff+124);
0365           //~ printf(" Long Filename found 136 %s\n", buff+136);
0366           //~ printf(" Long Filename found 148 %s\n", buff+148);
0367           //~ printf(" Long Filename found 156 %s\n", buff+156);
0368           //~ printf(" Long Filename found 157 %s\n", buff+157);
0369           //~ printf(" Long Filename found 158 %s\n", buff+158);
0370           //~ printf(" Long Filename found 159 %s\n", buff+159);
0371           //~ printf(" Long Filename found 257 %s\n", buff+257);
0372           //~ printf(" Long Filename found 263 %s\n", buff+263);
0373           //~ printf(" Long Filename found 265 %s\n", buff+265);
0374           //~ printf(" Long Filename found 297 %s\n", buff+297);
0375           //~ printf(" Long Filename found 329 %s\n", buff+329);
0376           //~ printf(" Long Filename found 337 %s\n", buff+337);
0377           //~ printf(" Long Filename found 345 %s\n", buff+345);
0378           //~ printf(" Long Filename found 346 %s\n", buff+346);
0379           //~ printf(" Long Filename found 347 %s\n", buff+347);
0380           break;
0381
0382         case 'K':
0383           longlinkname = true;
0384           break;
0385
0386         default:
0387           if (!longpathname) {
0388             int endposition = -1;
0389             for (int k = 99; k >= 0; k--) {
0390               if (buff[k] == '\0')
0391                 endposition = k;
0392             }
0393             if (endposition == -1) {
0394               //~ printf("OLDNAME : %s\n",buff);
0395               longpathname = true;
0396               for (int k = 0; k < 100; k++) {
0397                 newlongpathname[k] = buff[k];
0398               }
0399               newlongpathname[100] = '\0';
0400               //~ printf("NEWNAME : %s\n",newlongpathname);
0401             }
0402           }
0403           if (longpathname) {
0404             printf(" Extracting file %s\n", newlongpathname);
0405             f = create_file(newlongpathname, parseoct(buff + 100, 8));
0406             longpathname = false;
0407           } else {
0408             printf(" Extracting file %s\n", buff);
0409             f = create_file(buff, parseoct(buff + 100, 8));
0410           }
0411           break;
0412       }
0413
0414       if (longlinkname || longpathname) {
0415         if (buff[156] == 'K') {
0416           for (int ll = 0; ll < 512; ll++) {
0417             printf("%c", buff[ll]);
0418           }
0419           printf("\n");
0420           bytes_read = fread(buff, 1, 512, a);
0421           for (int ll = 0; ll < 512; ll++) {
0422             printf("%c", buff[ll]);
0423           }
0424           printf("\n");
0425           for (int k = 0; k < filesize; k++) {
0426             newlonglinkname[k] = buff[k];
0427           }
0428           newlonglinkname[filesize] = '\0';
0429           for (int k = filesize + 1; k < 512; k++) {
0430             newlonglinkname[k] = '0';
0431           }
0432           //~ printf("NEW LinkNAME: %s\n",newlonglinkname);
0433         } else if (buff[156] == 'L') {
0434           bytes_read = fread(buff, 1, 512, a);
0435           for (int k = 0; k < filesize; k++) {
0436             newlongpathname[k] = buff[k];
0437           }
0438           newlongpathname[filesize] = '\0';
0439           for (int k = filesize + 1; k < 512; k++) {
0440             newlongpathname[k] = '0';
0441           }
0442           //~ printf("NEW FILENAME: %s\n",newlongpathname);
0443         }
0444       }
0445
0446       //~
0447       //~ if (longpathname) {
0448       //~ bytes_read = fread(buff, 1, 512, a);
0449       //~ for (int k=0; k<filesize; k++){
0450       //~ newlongpathname[k]=buff[k];
0451       //~ }
0452       //~ newlongpathname[filesize]='\0';
0453       //~ for (int k=filesize+1; k<512; k++){
0454       //~ newlongpathname[k]='0';
0455       //~ }
0456       //~ printf("NEW FILENAME: %s\n",newlongpathname);
0457       //~
0458       //~ }
0459       //~ else if (!longpathname && !longlinkname) {
0460       if (!longpathname && !longlinkname) {
0461         while (filesize > 0) {
0462           bytes_read = fread(buff, 1, 512, a);
0463           if (bytes_read < 512) {
0464             fprintf(stderr, "Short read on %s: Expected 512, got %d\n", path, (int)bytes_read);
0465             return;
0466           }
0467           if (filesize < 512)
0468             bytes_read = filesize;
0469           if (f != nullptr) {
0470             if (fwrite(buff, 1, bytes_read, f) != bytes_read) {
0471               fprintf(stderr, "Failed write\n");
0472               fclose(f);
0473               f = nullptr;
0474             }
0475           }
0476           filesize -= bytes_read;
0477         }
0478         if (f != nullptr) {
0479           fclose(f);
0480           f = nullptr;
0481         }
0482       }
0483     }
0484   }
0485
0486   // function for calculating the MD5 checksum of a file
0487   void md5_File(std::string filename, char *result) {
0488     char buffer[4096];
0489     cms::openssl_init();
0490     EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
0491     const EVP_MD *md = EVP_get_digestbyname("MD5");
0492     EVP_DigestInit_ex(mdctx, md, nullptr);
0493
0494     //Open File
0495     int fd = open(filename.c_str(), O_RDONLY);
0496     int nb_read;
0497     while ((nb_read = read(fd, buffer, 4096 - 1))) {
0498       EVP_DigestUpdate(mdctx, buffer, nb_read);
0499       memset(buffer, 0, 4096);
0500     }
0501
0502     unsigned int md_len = 0;
0503     unsigned char tmp[EVP_MAX_MD_SIZE];
0504     EVP_DigestFinal_ex(mdctx, tmp, &md_len);
0505     EVP_MD_CTX_free(mdctx);
0506
0507     assert(result);
0508     //Convert the result
0509     for (unsigned int k = 0; k < md_len; ++k) {
0510       sprintf(result + k * 2, "%02x", tmp[k]);
0511     }
0512   }
0513
0514 }  // End namespace spu