Warning, /DQM/Integration/scripts/igfilemgr/igmerge is written in an unsupported language. File is not indexed.
0001 #! /usr/bin/env python3
0002 """
0003 zipmerge [-sret] [-n size ] [--help] [-o name] file1
0004 file2 [filen...]
0005
0006 Options:
0007 -s Sort file entries by file name in merged files.
0008 -r If s is specifyed, will append to the filename the name of the last
0009 file and the first file.
0010 -e Erase source files.If process interrupted in the midle it could render
0011 the files useless. Always make backup copys of your files before using
0012 this option.
0013 -t Test print out the inner structure but don't take any action.
0014 -n size Specify the maximum file size in Kb. If the merge exeeds the maximum size,
0015 additional files will be created also not exeeding this size.
0016 -o name Name to be used for resulting merged file(s).
0017 file 1 list of files to be merged, the program accepts a minimum of 2 files,
0018 file 2 wild cards are accepted.
0019 filen
0020
0021 """
0022
0023
0024 import sys
0025 import os.path
0026 import getopt as gop
0027 import zipfile as zp
0028
0029 SORTED=False
0030 RANGE=False
0031 KEEP=True
0032 TEST=False
0033 MFILENAME="merged.zip"
0034 CHUNKSIZE=2097152
0035 ZFHANDLES=[]
0036 OUTFILES=[]
0037 MEMBERS=[]
0038 #FILETREE={}
0039 TOTALSIZE=0.0
0040 ALL=False
0041 def numericalSort(x,y):
0042 """Events/Run_84168/Event_9998876"""
0043 xvec=x.split("/")
0044 yvec=y.split("/")
0045 if len(yvec) < 3 or len(xvec) < 3:
0046 return cmp(x,y)
0047
0048 xrun=int(xvec[1].split("_")[1])
0049 xevent=int(xvec[2].split("_")[1])
0050 yrun=int(yvec[1].split("_")[1])
0051 yevent=int(yvec[2].split("_")[1])
0052 if xrun < yrun:
0053 return -1
0054 elif xrun > yrun:
0055 return 1
0056 if xevent < yevent:
0057 return -1
0058 elif xevent > yevent:
0059 return 1
0060 else:
0061 return 0
0062
0063 def sortMembers():
0064 print "Sorting start"
0065 MEMBERS.sort(cmp=numericalSort, key=lambda x:x[1].filename)
0066 print "Sorted"
0067 def openOutputZipFile(filename):
0068 global ALL,OUTFILES
0069 try:
0070 if os.path.exists(filename):
0071 key="x"
0072 while key not in ["y","Y","n","N","a","A","all","All",""] and not ALL:
0073 print "\nOutput file %s already exist and will be overwriten!!!" % filename
0074 key=input("Do you want to proceed? (y,n,[all])")
0075 ALL=key in ["a","A","all","All",""]
0076 if key in ["y","Y",""] or ALL:
0077 os.remove(filename)
0078 else:
0079 sys.exit()
0080 zpout=zp.ZipFile(filename,"w")
0081 zpout.close()
0082 zpout=zp.ZipFile(filename,"a")
0083 OUTFILES.append(zpout)
0084 return zpout
0085 except:
0086 a,b,c=sys.exc_info()
0087 b.code == None or sys.stderr.write("Could not create output file\n")
0088 sys.exit(3)
0089 def cleanup():
0090 global OUTFILES,ZFHANDLES
0091 for handle in ZFHANDLES:
0092 try:
0093 handle.close()
0094 except:
0095 pass
0096 for handle in OUTFILES:
0097 try:
0098 handle.close()
0099 except:
0100 pass
0101
0102 if __name__ == "__main__":
0103 try:
0104 (args,filenames)=gop.getopt(sys.argv[1:],"srn:o:",["help"])
0105 except getopt.GetoptError:
0106 sys.stderr.write( "Sintax Error unrecognised option" )
0107 sys.stderr.write( __doc__ )
0108 sys.exit(2)
0109
0110 for item in args:
0111 if item[0]=="-s":
0112 SORTED=True
0113 elif item[0]=="-r":
0114 RANGE=True
0115 elif item[0]=="-e":
0116 KEEP=False
0117 elif item[0]=="-n":
0118 CHUNKSIZE=int(item[1])
0119 elif item[0]=="-o":
0120 MFILENAME=item[1]
0121 elif item[0]=="--help":
0122 sys.stdout.write(__doc__)
0123 sys.exit(1)
0124
0125 if len(filenames) < 2 :
0126 sys.stdout.write(__doc__)
0127 sys.exit(2)
0128
0129 basename,extension=MFILENAME.rsplit(".",1)
0130 for filename in filenames:
0131 print "Opening %s" % filename
0132 ZFHANDLES.append(zp.ZipFile(filename,"r"))
0133
0134 print "gathering archived files statistics"
0135 for handle in ZFHANDLES:
0136 for info in handle.infolist():
0137 MEMBERS.append([handle,info])
0138 #FILETREE.setdefault(handle,[]).append(info)
0139 TOTALSIZE+=info.compress_size/1024
0140 print "found %d archived files in %d archives" % (len(MEMBERS),len(ZFHANDLES))
0141 numfiles=TOTALSIZE/CHUNKSIZE
0142 numfiles=int(round(numfiles) + (numfiles-round(numfiles)>0 and 1 or 0 ))
0143 print "The Number of expected requiered files is: %d (could be more)" % numfiles
0144
0145 print "Starting Merging process..."
0146
0147 print "Creating file structure"
0148 not SORTED or sortMembers()
0149 i=1
0150 while len(MEMBERS) > 0:
0151 filesize=0
0152 m=0
0153 filename="%s_%03d.%s" % (basename,i,extension)
0154 print "opening file %s" % filename
0155 outfile=openOutputZipFile(filename)
0156 firstmember=MEMBERS[m][1].filename.rsplit("/",1)[-1]
0157 while len(MEMBERS) > 0 and m < len(MEMBERS):
0158 member=MEMBERS[m]
0159 if (member[1].compress_size+filesize)/1024 < CHUNKSIZE:
0160 try:
0161 print member[1].filename
0162 outfile.writestr(member[1],member[0].read(member[1].filename))
0163 except zp.BadZipfile:
0164 print member[1].filename
0165 print "Problem with file member: %s skiping" % member[1].filename
0166 del member
0167 del MEMBERS[m]
0168 continue
0169 lastmember=member[1].filename.rsplit("/",1)[-1]
0170 filesize+=member[1].compress_size
0171 del member
0172 del MEMBERS[m]
0173 elif SORTED:
0174 break
0175 else:
0176 m+=1
0177 i+=1
0178 print "Closing file %s" % filename
0179 outfile.close()
0180 if RANGE:
0181 nfname="%s_%s_to_%s.%s" % (basename,firstmember,lastmember,extension)
0182 os.rename(filename,nfname)
0183
0184 cleanup()
0185
0186
