HLTanalyzers/test/CreateFileLists.py

0001 #!/usr/bin/env python
0002 #
0003
0004 import sys,string,time,os
0005
0006 ### parameters ###
0007
0008 FilesPerCfg=1
0009 CFGBASE="MinimumBias_"
0010
0011 ####
0012
0013 def usage():
0014     """ Usage: CreateFileList <FileList> <OutputDir>
0015
0016  Splits a large file list into a set of smaller lists with <FilesPerCfg> files
0017     """
0018     pass
0019
0020 def OpenFile(file_in,iodir):
0021     """  file_in -- Input file name
0022          iodir   -- 'r' readonly  'r+' read+write """
0023     try:
0024         ifile=open(file_in, iodir)
0025         # print "Opened file: ",file_in," iodir ",iodir
0026     except:
0027         print("Could not open file: ",file_in)
0028         sys.exit(1)
0029     return ifile
0030
0031 def CloseFile(ifile):
0032     ifile.close()
0033
0034 def ReadFile(file):
0035
0036
0037     infile=OpenFile(file,'r')
0038     iline=0
0039
0040     x = infile.readline()
0041
0042     files=[]
0043     while x != "":
0044         iline+=1
0045         xx=string.strip(x)
0046         if (len(xx)>0):
0047             files.append(xx)
0048
0049         x = infile.readline()
0050     CloseFile(infile)
0051
0052     return files
0053
0054 def createCFG(i,filelist):
0055
0056     CFGFILE=CFGBASE + str(i) +".py"
0057     print(i, CFGFILE)
0058     file = open(CFGFILE,'w')
0059     file.write("import FWCore.ParameterSet.Config as cms \n")
0060     file.write("\n")
0061     file.write("maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n")
0062     file.write("readFiles = cms.untracked.vstring()\n")
0063     file.write("secFiles = cms.untracked.vstring()\n")
0064     file.write("source = cms.Source (\"PoolSource\",fileNames = readFiles, secondaryFileNames = secFiles)\n")
0065     file.write("readFiles.extend( [\n")
0066
0067     i=0
0068     for infile in filelist:
0069        i=i+1
0070        if i<len(filelist):
0071            endstr="',"
0072        else:
0073            endstr="' ]);"
0074        outstring="    " + infile + endstr
0075        file.write(outstring + "\n")
0076
0077     file.write("\n")
0078     file.write("secFiles.extend( [ ])\n")
0079     CloseFile(file)
0080
0081 if __name__ == '__main__':
0082
0083
0084     narg=len(sys.argv)
0085     if narg < 3 :
0086         print(usage.__doc__)
0087         sys.exit(1)
0088
0089
0090     InputFile=sys.argv[1]
0091     OutputDir=sys.argv[2]
0092
0093     if not os.path.exists(OutputDir):
0094         os.mkdir(OutputDir)
0095
0096     filelist = ReadFile(InputFile)
0097     print("Number of files in input filelist: ", len(filelist))
0098
0099     os.chdir(OutputDir)
0100
0101     i=0
0102     ibatch=0
0103     cfglist=[]
0104     for file in filelist:
0105         i=i+1
0106         #if (i>MAXFILES): break
0107         filename=file
0108         cfglist.append(filename)
0109         if i%FilesPerCfg == 0 or i==len(filelist):
0110             createCFG(ibatch,cfglist)
0111             ibatch=ibatch+1
0112             cfglist=[]
0113
0114         #print i%FilesPerCfg, i, filename
0115