test/Cosmic_B38/launchOnBatch.py

0001 #!/usr/bin/env python
0002
0003 from __future__ import print_function
0004 import urllib
0005 import string
0006 import os
0007 import sys
0008
0009 Input_ConfigFile = "MultiJob_cfg.py"
0010                     # The name of your config file
0011                     # where you have to replace the OutputAdresse by XXX_OUTPUT_XXX
0012                     # and the Number of Events by XXX_NEVENTS_XXX
0013                     # and the Number of Event to skip is XXX_SKIPEVENT_XXX
0014 Input_CffFile    = "InputFiles_cff.py"
0015 Input_CffN       = 1
0016
0017
0018 Output_RootFile = "ALCA" # The name of your output file  (will replace XXX_OUTPUT_XXX)
0019
0020 Job_NEvents = -1            # Number of Events by job (will replace XXX_NEVENTS_XXX)
0021 Job_Start   = 0             # The Index of your first job
0022 Job_End     = 115           # The Index of your last  job
0023 Job_SEvents = 0             # Event that you want to skip
0024                     # The first event will be Job_SEvents+1
0025
0026 FarmDirectory = "FARM"
0027 QUEUE         = "cmscaf"
0028
0029
0030
0031 def CreateTheConfigFile(PATH,CONFIG_FILE,NEVENTS,OUTPUTFILE,INPUTFILE,INDEX):
0032
0033     config_file=open(CONFIG_FILE,'r')
0034     config_txt   = config_file.read()               # Read all data
0035     config_file.close()
0036     newconfig_path = PATH + "/"+FarmDirectory+"/InputFile/%04i_" % INDEX
0037         newconfig_path = newconfig_path + Output_RootFile + "_cfg.py"
0038
0039     mylogo1 = "#  -----------------------------------------------\n"
0040     mylogo2 = "# |   cfg modified by the LaunchOnFarm Script     |\n"
0041     mylogo3 = "# |   Created by Loic Quertenmont                 |\n"
0042     mylogo4 = "# |   Loic.quertenmont@cern.ch                    |\n"
0043     mylogo5 = "#  -----------------------------------------------\n\n\n\n"
0044     config_txt = mylogo1 + mylogo2 + mylogo3 + mylogo4 + mylogo5 + config_txt
0045
0046     i=0
0047     while i < len(config_txt) :
0048         if config_txt[i:i+15]=='XXX_NEVENTS_XXX':
0049             Skip = INDEX*NEVENTS+Job_SEvents
0050             MaxEvent = NEVENTS
0051             print("job #%d" %INDEX + "\t\tNumber of Events fixed to \t\t%d"%MaxEvent)
0052             newconfig_file=open(newconfig_path,'w')
0053             newconfig_file.write("%s" % config_txt[0:i])
0054             newconfig_file.write("%d" % MaxEvent)
0055             newconfig_file.write("%s" % config_txt[i+15:len(config_txt)])
0056             newconfig_file.close()
0057             newconfig_file=open(newconfig_path,'r')
0058             config_txt   = newconfig_file.read()
0059             newconfig_file.close()
0060             i = 0
0061         if config_txt[i:i+14]=='XXX_OUTPUT_XXX':
0062             print("job #%d" %INDEX + "\tOutput file fixed to\t\t%s"%OUTPUTFILE)
0063             newconfig_file=open(newconfig_path,'w')
0064             newconfig_file.write("%s" % config_txt[0:i])
0065             newconfig_file.write("%s"% OUTPUTFILE)
0066             newconfig_file.write("_%04i.root" % INDEX)
0067             newconfig_file.write("%s" % config_txt[i+14:len(config_txt)])
0068             newconfig_file.close()
0069             newconfig_file=open(newconfig_path,'r')
0070             config_txt   = newconfig_file.read()
0071             newconfig_file.close()
0072             i = 0
0073                 if config_txt[i:i+17]=='XXX_SKIPEVENT_XXX':
0074             Skip = INDEX*NEVENTS+Job_SEvents
0075                         print("job #%d" %INDEX + "\tNumber of Event to skip is fixed to\t\t%i"%Skip)
0076                         newconfig_file=open(newconfig_path,'w')
0077                         newconfig_file.write("%s" % config_txt[0:i])
0078                         newconfig_file.write("%i"%Skip)
0079                         newconfig_file.write("%s" % config_txt[i+17:len(config_txt)])
0080                         newconfig_file.close()
0081                         newconfig_file=open(newconfig_path,'r')
0082                         config_txt   = newconfig_file.read()
0083                         newconfig_file.close()
0084                         i = 0
0085                 if config_txt[i:i+9]=='XXX_I_XXX':
0086                         newconfig_file=open(newconfig_path,'w')
0087                         newconfig_file.write("%s" % config_txt[0:i])
0088                         newconfig_file.write("%04i"%INDEX)
0089                         newconfig_file.write("%s" % config_txt[i+9:len(config_txt)])
0090                         newconfig_file.close()
0091                         newconfig_file=open(newconfig_path,'r')
0092                         config_txt   = newconfig_file.read()
0093                         newconfig_file.close()
0094                         i = 0
0095                 if config_txt[i:i+13]=='XXX_INPUT_XXX':
0096                         print("job #%d" %INDEX + "\tInput file fixed to\t\t%s"%INPUTFILE)
0097                         newconfig_file=open(newconfig_path,'w')
0098                         newconfig_file.write("%s" % config_txt[0:i])
0099                         newconfig_file.write("%s" % GetInputFiles(PATH,Input_CffFile,NEVENTS,OUTPUTFILE,INDEX))
0100                         newconfig_file.write("%s" % config_txt[i+13:len(config_txt)])
0101                         newconfig_file.close()
0102                         newconfig_file=open(newconfig_path,'r')
0103                         config_txt   = newconfig_file.read()
0104                         newconfig_file.close()
0105                         i = 0
0106
0107
0108
0109         i = i+1
0110
0111 def GetInputFiles(PATH,INPUT_FILE,NEVENTS,OUTPUTFILE,INDEX):
0112
0113         config_file=open(INPUT_FILE,'r')
0114         config_txt = ""
0115     i=0
0116         iMin = (INDEX+0)*Input_CffN
0117     iMax = (INDEX+1)*Input_CffN-1
0118         for line in config_file:
0119                 if(line[0:1]!='\''):
0120             continue
0121
0122                 if( (i>=iMin) and (i<=iMax) ):
0123             config_txt = config_txt + line
0124                 i = i+1
0125
0126         if(iMax>=i):
0127         return 0
0128         config_file.close()
0129
0130
0131         if(config_txt[len(config_txt)-2:len(config_txt)-1]==','):
0132         config_txt = config_txt[0:len(config_txt)-2]
0133         newconfig_path = PATH + "/"+FarmDirectory+"/InputFile/%04i_" % INDEX
0134         newconfig_path = newconfig_path + Output_RootFile + "_cff.py"
0135
0136         return config_txt
0137
0138
0139
0140 def CreateTheShellFile(PATH,INDEX):
0141     shell_path = "./"+FarmDirectory+"/InputFile/%04i_" % INDEX
0142         shell_path = shell_path + Output_RootFile + ".sh"
0143
0144         cfg_path = PATH + "/" + FarmDirectory + "/InputFile/%04i_" % INDEX
0145         cfg_path = cfg_path + Output_RootFile + "_cfg.py"
0146
0147     shell_file=open(shell_path,'w')
0148     shell_file.write("#! /bin/sh\n")
0149     shell_file.write("#  ----------------------------------------------- \n")
0150     shell_file.write("# |   Script created by the LaunchOnFarm Script   |\n")
0151     shell_file.write("# |   Created by Loic Quertenmont                 |\n")
0152     shell_file.write("# |   Loic.quertenmont@cern.ch                    |\n")
0153     shell_file.write("#  ----------------------------------------------- \n\n\n\n")
0154         shell_file.write("%s" % "cd " + PATH + "/" + FarmDirectory + "\n")
0155     shell_file.write("%s\n" % "eval `scramv1 runtime -sh`")
0156 #        shell_file.write("%s\n" % "export STAGE_SVCCLASS=cmscaf")
0157 #        shell_file.write("%s\n" % "export STAGER_TRACE=3")
0158         shell_file.write("%s" % "cmsRun " + cfg_path +"\n")
0159     shell_file.close()
0160     chmod_path = "chmod 777 "+shell_path
0161     os.system(chmod_path)
0162
0163
0164 path = os.getcwd()  #Get the current path
0165 os.system('mkdir '+FarmDirectory)
0166 os.system('mkdir '+FarmDirectory+'/RootFiles')
0167 os.system('mkdir '+FarmDirectory+'/Log')
0168 os.system('mkdir '+FarmDirectory+'/InputFile')
0169
0170 for i in range(Job_Start,Job_End):
0171     print('Submitting job number %d' %i)
0172
0173         input_path = FarmDirectory + ".InputFile.%04i_" % i
0174         input_path = input_path + Output_RootFile + "_cff.py"
0175
0176
0177 #        if( GetInputFiles(path,Input_CffFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,i) == 0)
0178 #               print('error during the _cfg.py file creation --> are you sure InputFile_cff.py contains enough lines? \n')
0179 #               continue
0180
0181
0182 #   cff_created = 0
0183 #        if(len(Input_CffFile)>3):
0184 #       cff_created = CreateTheInputFile(path,Input_CffFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,i)
0185 #        if(cff_created==0):
0186 #                print('error during the cff file creation --> are you sure it contains enough lines? \n')
0187 #       continue
0188     CreateTheConfigFile(path,Input_ConfigFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,input_path,i)
0189     CreateTheShellFile(path,i)
0190
0191     condor_path = "./"+FarmDirectory+"/InputFile/%04i_" % i
0192         condor_path = condor_path + Output_RootFile + ".cmd"
0193
0194         shell_path = path + "/" + FarmDirectory + "/InputFile/%04i_" % i
0195         shell_path = shell_path + Output_RootFile + ".sh"
0196
0197         JobName = "'" + Output_RootFile + "%04i'" % i
0198 #        OutputPath = "Log/%04i_" % i
0199 #        OutputPath = OutputPath + Output_RootFile + "/"
0200         OutputPath = "'out/'"
0201
0202 #        batchSubmit = "bsub -q" + QUEUE + " -J" + JobName  + "'" + shell_path + " 0 ele'"
0203 #        batchSubmit = "bsub -q " + QUEUE + " -J " + JobName  + " -oo " + OutputPath + " -eo " + OutputPath + " '" + shell_path + " 0 ele'"
0204         batchSubmit = "bsub -q " + QUEUE + " -J " + JobName  + " '" + shell_path + " 0 ele'"
0205     os.system (batchSubmit)
0206
0207     print('\n')
0208 NJobs = Job_End - Job_Start
0209 print("\n\n")
0210 print("\t\t\t%i Jobs submitted by the LaunchOnFarm script" % NJobs)
0211 print("\t\t\t         Created by Loic Quertenmont")
0212 print("\t\t\t           Loic.quertenmont@cern.ch")
0213 print("\n\n")