HLTanalyzers/test/CreateFileLists.py

0001 #!/usr/bin/env python
0002 #
0003
0004 from __future__ import print_function
0005 import sys,string,time,os
0006
0007 ### parameters ###
0008
0009 FilesPerCfg=1
0010 CFGBASE="MinimumBias_"
0011
0012 ####
0013
0014 def usage():
0015     """ Usage: CreateFileList <FileList> <OutputDir>
0016
0017  Splits a large file list into a set of smaller lists with <FilesPerCfg> files
0018     """
0019     pass
0020
0021 def OpenFile(file_in,iodir):
0022     """  file_in -- Input file name
0023          iodir   -- 'r' readonly  'r+' read+write """
0024     try:
0025         ifile=open(file_in, iodir)
0026         # print "Opened file: ",file_in," iodir ",iodir
0027     except:
0028         print("Could not open file: ",file_in)
0029         sys.exit(1)
0030     return ifile
0031
0032 def CloseFile(ifile):
0033     ifile.close()
0034
0035 def ReadFile(file):
0036
0037
0038     infile=OpenFile(file,'r')
0039     iline=0
0040
0041     x = infile.readline()
0042
0043     files=[]
0044     while x != "":
0045         iline+=1
0046         xx=string.strip(x)
0047         if (len(xx)>0):
0048             files.append(xx)
0049
0050         x = infile.readline()
0051     CloseFile(infile)
0052
0053     return files
0054
0055 def createCFG(i,filelist):
0056
0057     CFGFILE=CFGBASE + str(i) +".py"
0058     print(i, CFGFILE)
0059     file = open(CFGFILE,'w')
0060     file.write("import FWCore.ParameterSet.Config as cms \n")
0061     file.write("\n")
0062     file.write("maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n")
0063     file.write("readFiles = cms.untracked.vstring()\n")
0064     file.write("secFiles = cms.untracked.vstring()\n")
0065     file.write("source = cms.Source (\"PoolSource\",fileNames = readFiles, secondaryFileNames = secFiles)\n")
0066     file.write("readFiles.extend( [\n")
0067
0068     i=0
0069     for infile in filelist:
0070        i=i+1
0071        if i<len(filelist):
0072            endstr="',"
0073        else:
0074            endstr="' ]);"
0075        outstring="    " + infile + endstr
0076        file.write(outstring + "\n")
0077
0078     file.write("\n")
0079     file.write("secFiles.extend( [ ])\n")
0080     CloseFile(file)
0081
0082 if __name__ == '__main__':
0083
0084
0085     narg=len(sys.argv)
0086     if narg < 3 :
0087         print(usage.__doc__)
0088         sys.exit(1)
0089
0090
0091     InputFile=sys.argv[1]
0092     OutputDir=sys.argv[2]
0093
0094     if not os.path.exists(OutputDir):
0095         os.mkdir(OutputDir)
0096
0097     filelist = ReadFile(InputFile)
0098     print("Number of files in input filelist: ", len(filelist))
0099
0100     os.chdir(OutputDir)
0101
0102     i=0
0103     ibatch=0
0104     cfglist=[]
0105     for file in filelist:
0106         i=i+1
0107         #if (i>MAXFILES): break
0108         filename=file
0109         cfglist.append(filename)
0110         if i%FilesPerCfg == 0 or i==len(filelist):
0111             createCFG(ibatch,cfglist)
0112             ibatch=ibatch+1
0113             cfglist=[]
0114
0115         #print i%FilesPerCfg, i, filename
0116