test/Cosmic_B38/launchOnBatch.py

0001 #!/usr/bin/env python
0002
0003 import urllib
0004 import string
0005 import os
0006 import sys
0007
0008 Input_ConfigFile = "MultiJob_cfg.py"
0009                                         # The name of your config file
0010                                         # where you have to replace the OutputAdresse by XXX_OUTPUT_XXX
0011                                         # and the Number of Events by XXX_NEVENTS_XXX
0012                                         # and the Number of Event to skip is XXX_SKIPEVENT_XXX
0013 Input_CffFile    = "InputFiles_cff.py"
0014 Input_CffN       = 1
0015
0016
0017 Output_RootFile = "ALCA" # The name of your output file  (will replace XXX_OUTPUT_XXX)
0018
0019 Job_NEvents = -1                        # Number of Events by job (will replace XXX_NEVENTS_XXX)
0020 Job_Start   = 0                         # The Index of your first job
0021 Job_End     = 115                        # The Index of your last  job
0022 Job_SEvents = 0                                # Event that you want to skip
0023                                         # The first event will be Job_SEvents+1
0024
0025 FarmDirectory = "FARM"
0026 QUEUE         = "cmscaf"
0027
0028
0029
0030 def CreateTheConfigFile(PATH,CONFIG_FILE,NEVENTS,OUTPUTFILE,INPUTFILE,INDEX):
0031
0032         config_file=open(CONFIG_FILE,'r')
0033         config_txt   = config_file.read()               # Read all data
0034         config_file.close()
0035         newconfig_path = PATH + "/"+FarmDirectory+"/InputFile/%04i_" % INDEX
0036         newconfig_path = newconfig_path + Output_RootFile + "_cfg.py"
0037
0038         mylogo1 = "#  -----------------------------------------------\n"
0039         mylogo2 = "# |   cfg modified by the LaunchOnFarm Script     |\n"
0040         mylogo3 = "# |   Created by Loic Quertenmont                 |\n"
0041         mylogo4 = "# |   Loic.quertenmont@cern.ch                    |\n"
0042         mylogo5 = "#  -----------------------------------------------\n\n\n\n"
0043         config_txt = mylogo1 + mylogo2 + mylogo3 + mylogo4 + mylogo5 + config_txt
0044
0045         i=0
0046         while i < len(config_txt) :
0047                 if config_txt[i:i+15]=='XXX_NEVENTS_XXX':
0048                         Skip = INDEX*NEVENTS+Job_SEvents
0049                         MaxEvent = NEVENTS
0050                         print("job #%d" %INDEX + "\t\tNumber of Events fixed to \t\t%d"%MaxEvent)
0051                         newconfig_file=open(newconfig_path,'w')
0052                         newconfig_file.write("%s" % config_txt[0:i])
0053                         newconfig_file.write("%d" % MaxEvent)
0054                         newconfig_file.write("%s" % config_txt[i+15:len(config_txt)])
0055                         newconfig_file.close()
0056                         newconfig_file=open(newconfig_path,'r')
0057                         config_txt   = newconfig_file.read()
0058                         newconfig_file.close()
0059                         i = 0
0060                 if config_txt[i:i+14]=='XXX_OUTPUT_XXX':
0061                         print("job #%d" %INDEX + "\tOutput file fixed to\t\t%s"%OUTPUTFILE)
0062                         newconfig_file=open(newconfig_path,'w')
0063                         newconfig_file.write("%s" % config_txt[0:i])
0064                         newconfig_file.write("%s"% OUTPUTFILE)
0065                         newconfig_file.write("_%04i.root" % INDEX)
0066                         newconfig_file.write("%s" % config_txt[i+14:len(config_txt)])
0067                         newconfig_file.close()
0068                         newconfig_file=open(newconfig_path,'r')
0069                         config_txt   = newconfig_file.read()
0070                         newconfig_file.close()
0071                         i = 0
0072                 if config_txt[i:i+17]=='XXX_SKIPEVENT_XXX':
0073                         Skip = INDEX*NEVENTS+Job_SEvents
0074                         print("job #%d" %INDEX + "\tNumber of Event to skip is fixed to\t\t%i"%Skip)
0075                         newconfig_file=open(newconfig_path,'w')
0076                         newconfig_file.write("%s" % config_txt[0:i])
0077                         newconfig_file.write("%i"%Skip)
0078                         newconfig_file.write("%s" % config_txt[i+17:len(config_txt)])
0079                         newconfig_file.close()
0080                         newconfig_file=open(newconfig_path,'r')
0081                         config_txt   = newconfig_file.read()
0082                         newconfig_file.close()
0083                         i = 0
0084                 if config_txt[i:i+9]=='XXX_I_XXX':
0085                         newconfig_file=open(newconfig_path,'w')
0086                         newconfig_file.write("%s" % config_txt[0:i])
0087                         newconfig_file.write("%04i"%INDEX)
0088                         newconfig_file.write("%s" % config_txt[i+9:len(config_txt)])
0089                         newconfig_file.close()
0090                         newconfig_file=open(newconfig_path,'r')
0091                         config_txt   = newconfig_file.read()
0092                         newconfig_file.close()
0093                         i = 0
0094                 if config_txt[i:i+13]=='XXX_INPUT_XXX':
0095                         print("job #%d" %INDEX + "\tInput file fixed to\t\t%s"%INPUTFILE)
0096                         newconfig_file=open(newconfig_path,'w')
0097                         newconfig_file.write("%s" % config_txt[0:i])
0098                         newconfig_file.write("%s" % GetInputFiles(PATH,Input_CffFile,NEVENTS,OUTPUTFILE,INDEX))
0099                         newconfig_file.write("%s" % config_txt[i+13:len(config_txt)])
0100                         newconfig_file.close()
0101                         newconfig_file=open(newconfig_path,'r')
0102                         config_txt   = newconfig_file.read()
0103                         newconfig_file.close()
0104                         i = 0
0105
0106
0107
0108                 i = i+1
0109
0110 def GetInputFiles(PATH,INPUT_FILE,NEVENTS,OUTPUTFILE,INDEX):
0111
0112         config_file=open(INPUT_FILE,'r')
0113         config_txt = ""
0114         i=0
0115         iMin = (INDEX+0)*Input_CffN
0116         iMax = (INDEX+1)*Input_CffN-1
0117         for line in config_file:
0118                 if(line[0:1]!='\''):
0119                         continue
0120
0121                 if( (i>=iMin) and (i<=iMax) ):
0122                         config_txt = config_txt + line
0123                 i = i+1
0124
0125         if(iMax>=i):
0126                 return 0
0127         config_file.close()
0128
0129
0130         if(config_txt[len(config_txt)-2:len(config_txt)-1]==','):
0131                 config_txt = config_txt[0:len(config_txt)-2]
0132         newconfig_path = PATH + "/"+FarmDirectory+"/InputFile/%04i_" % INDEX
0133         newconfig_path = newconfig_path + Output_RootFile + "_cff.py"
0134
0135         return config_txt
0136
0137
0138
0139 def CreateTheShellFile(PATH,INDEX):
0140         shell_path = "./"+FarmDirectory+"/InputFile/%04i_" % INDEX
0141         shell_path = shell_path + Output_RootFile + ".sh"
0142
0143         cfg_path = PATH + "/" + FarmDirectory + "/InputFile/%04i_" % INDEX
0144         cfg_path = cfg_path + Output_RootFile + "_cfg.py"
0145
0146         shell_file=open(shell_path,'w')
0147         shell_file.write("#! /bin/sh\n")
0148         shell_file.write("#  ----------------------------------------------- \n")
0149         shell_file.write("# |   Script created by the LaunchOnFarm Script   |\n")
0150         shell_file.write("# |   Created by Loic Quertenmont                 |\n")
0151         shell_file.write("# |   Loic.quertenmont@cern.ch                    |\n")
0152         shell_file.write("#  ----------------------------------------------- \n\n\n\n")
0153         shell_file.write("%s" % "cd " + PATH + "/" + FarmDirectory + "\n")
0154         shell_file.write("%s\n" % "eval `scramv1 runtime -sh`")
0155 #        shell_file.write("%s\n" % "export STAGE_SVCCLASS=cmscaf")
0156 #        shell_file.write("%s\n" % "export STAGER_TRACE=3")
0157         shell_file.write("%s" % "cmsRun " + cfg_path +"\n")
0158         shell_file.close()
0159         chmod_path = "chmod 777 "+shell_path
0160         os.system(chmod_path)
0161
0162
0163 path = os.getcwd()         #Get the current path
0164 os.system('mkdir '+FarmDirectory)
0165 os.system('mkdir '+FarmDirectory+'/RootFiles')
0166 os.system('mkdir '+FarmDirectory+'/Log')
0167 os.system('mkdir '+FarmDirectory+'/InputFile')
0168
0169 for i in range(Job_Start,Job_End):
0170         print('Submitting job number %d' %i)
0171
0172         input_path = FarmDirectory + ".InputFile.%04i_" % i
0173         input_path = input_path + Output_RootFile + "_cff.py"
0174
0175
0176 #        if( GetInputFiles(path,Input_CffFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,i) == 0)
0177 #               print('error during the _cfg.py file creation --> are you sure InputFile_cff.py contains enough lines? \n')
0178 #               continue
0179
0180
0181 #        cff_created = 0
0182 #        if(len(Input_CffFile)>3):
0183 #                cff_created = CreateTheInputFile(path,Input_CffFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,i)
0184 #        if(cff_created==0):
0185 #                print('error during the cff file creation --> are you sure it contains enough lines? \n')
0186 #                continue
0187         CreateTheConfigFile(path,Input_ConfigFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,input_path,i)
0188         CreateTheShellFile(path,i)
0189
0190         condor_path = "./"+FarmDirectory+"/InputFile/%04i_" % i
0191         condor_path = condor_path + Output_RootFile + ".cmd"
0192
0193         shell_path = path + "/" + FarmDirectory + "/InputFile/%04i_" % i
0194         shell_path = shell_path + Output_RootFile + ".sh"
0195
0196         JobName = "'" + Output_RootFile + "%04i'" % i
0197 #        OutputPath = "Log/%04i_" % i
0198 #        OutputPath = OutputPath + Output_RootFile + "/"
0199         OutputPath = "'out/'"
0200
0201 #        batchSubmit = "bsub -q" + QUEUE + " -J" + JobName  + "'" + shell_path + " 0 ele'"
0202 #        batchSubmit = "bsub -q " + QUEUE + " -J " + JobName  + " -oo " + OutputPath + " -eo " + OutputPath + " '" + shell_path + " 0 ele'"
0203         batchSubmit = "bsub -q " + QUEUE + " -J " + JobName  + " '" + shell_path + " 0 ele'"
0204         os.system (batchSubmit)
0205
0206         print('\n')
0207 NJobs = Job_End - Job_Start
0208 print("\n\n")
0209 print("\t\t\t%i Jobs submitted by the LaunchOnFarm script" % NJobs)
0210 print("\t\t\t         Created by Loic Quertenmont")
0211 print("\t\t\t           Loic.quertenmont@cern.ch")
0212 print("\n\n")