test/Cosmic_B38/MergeOnBatch.py

0001 #!/usr/bin/env python
0002
0003 from __future__ import print_function
0004 import urllib
0005 import string
0006 import os
0007 import sys
0008
0009 Input_ConfigFile = "MergeJob_cfg.py"
0010                     # The name of your config file
0011                     # where you have to replace the OutputAdresse by XXX_OUTPUT_XXX
0012                     # and the Number of Events by XXX_NEVENTS_XXX
0013                     # and the Number of Event to skip is XXX_SKIPEVENT_XXX
0014
0015 Output_RootFile = "MERGE" # The name of your output file  (will replace XXX_OUTPUT_XXX)
0016
0017 Job_NEvents = 10000         # Number of Events by job (will replace XXX_NEVENTS_XXX)
0018 Job_Start   = 0             # The Index of your first job
0019 Job_End     = 1             # The Index of your last  job
0020 Job_SEvents = 0             # Event that you want to skip
0021                     # The first event will be Job_SEvents+1
0022
0023 FarmDirectory = "FARM_Merge"
0024 QUEUE         = "cmscaf"
0025
0026
0027 def CreateTheConfigFile(PATH,CONFIG_FILE,NEVENTS,OUTPUTFILE,INDEX):
0028
0029     config_file=open(CONFIG_FILE,'r')
0030     config_txt   = config_file.read()               # Read all data
0031     config_file.close()
0032     newconfig_path = PATH + "/"+FarmDirectory+"/InputFile/%04i_" % INDEX
0033         newconfig_path = newconfig_path + Output_RootFile + ".cfg"
0034
0035     mylogo1 = "#  -----------------------------------------------\n"
0036     mylogo2 = "# |   cfg modified by the LaunchOnFarm Script     |\n"
0037     mylogo3 = "# |   Created by Loic Quertenmont                 |\n"
0038     mylogo4 = "# |   Loic.quertenmont@cern.ch                    |\n"
0039     mylogo5 = "#  -----------------------------------------------\n\n\n\n"
0040     config_txt = mylogo1 + mylogo2 + mylogo3 + mylogo4 + mylogo5 + config_txt
0041
0042     i=0
0043     while i < len(config_txt) :
0044         if config_txt[i:i+15]=='XXX_NEVENTS_XXX':
0045             Skip = INDEX*NEVENTS+Job_SEvents
0046             MaxEvent = NEVENTS
0047             print("job #%d" %INDEX + "\t\tNumber of Events fixed to \t\t%d"%MaxEvent)
0048             newconfig_file=open(newconfig_path,'w')
0049             newconfig_file.write("%s" % config_txt[0:i])
0050             newconfig_file.write("%d" % MaxEvent)
0051             newconfig_file.write("%s" % config_txt[i+15:len(config_txt)])
0052             newconfig_file.close()
0053             newconfig_file=open(newconfig_path,'r')
0054             config_txt   = newconfig_file.read()
0055             newconfig_file.close()
0056             i = 0
0057         if config_txt[i:i+14]=='XXX_OUTPUT_XXX':
0058             print("job #%d" %INDEX + "\tOutput file fixed to\t\t%s"%OUTPUTFILE)
0059             newconfig_file=open(newconfig_path,'w')
0060             newconfig_file.write("%s" % config_txt[0:i])
0061             newconfig_file.write("%s"% OUTPUTFILE)
0062             newconfig_file.write("_%04i.root" % INDEX)
0063             newconfig_file.write("%s" % config_txt[i+14:len(config_txt)])
0064             newconfig_file.close()
0065             newconfig_file=open(newconfig_path,'r')
0066             config_txt   = newconfig_file.read()
0067             newconfig_file.close()
0068             i = 0
0069                 if config_txt[i:i+17]=='XXX_SKIPEVENT_XXX':
0070             Skip = INDEX*NEVENTS+Job_SEvents
0071                         print("job #%d" %INDEX + "\tNumber of Event to skip is fixed to\t\t%i"%Skip)
0072                         newconfig_file=open(newconfig_path,'w')
0073                         newconfig_file.write("%s" % config_txt[0:i])
0074                         newconfig_file.write("%i"%Skip)
0075                         newconfig_file.write("%s" % config_txt[i+17:len(config_txt)])
0076                         newconfig_file.close()
0077                         newconfig_file=open(newconfig_path,'r')
0078                         config_txt   = newconfig_file.read()
0079                         newconfig_file.close()
0080                         i = 0
0081                 if config_txt[i:i+9]=='XXX_I_XXX':
0082                         newconfig_file=open(newconfig_path,'w')
0083                         newconfig_file.write("%s" % config_txt[0:i])
0084                         newconfig_file.write("%04i"%INDEX)
0085                         newconfig_file.write("%s" % config_txt[i+9:len(config_txt)])
0086                         newconfig_file.close()
0087                         newconfig_file=open(newconfig_path,'r')
0088                         config_txt   = newconfig_file.read()
0089                         newconfig_file.close()
0090                         i = 0
0091
0092
0093         i = i+1
0094
0095 def CreateTheShellFile(PATH,INDEX):
0096     shell_path = "./"+FarmDirectory+"/InputFile/%04i_" % INDEX
0097         shell_path = shell_path + Output_RootFile + ".sh"
0098
0099         cfg_path = PATH + "/" + FarmDirectory + "/InputFile/%04i_" % INDEX
0100         cfg_path = cfg_path + Output_RootFile + ".cfg"
0101
0102     shell_file=open(shell_path,'w')
0103     shell_file.write("#! /bin/sh\n")
0104     shell_file.write("#  ----------------------------------------------- \n")
0105     shell_file.write("# |   Script created by the LaunchOnFarm Script   |\n")
0106     shell_file.write("# |   Created by Loic Quertenmont                 |\n")
0107     shell_file.write("# |   Loic.quertenmont@cern.ch                    |\n")
0108     shell_file.write("#  ----------------------------------------------- \n\n\n\n")
0109         shell_file.write("%s" % "cd " + PATH + "/" + FarmDirectory + "\n")
0110     shell_file.write("%s\n" % "eval `scramv1 runtime -sh`")
0111         shell_file.write("%s" % "cmsRun " + cfg_path +"\n")
0112     shell_file.close()
0113     chmod_path = "chmod 777 "+shell_path
0114     os.system(chmod_path)
0115
0116
0117 path = os.getcwd()  #Get the current path
0118 os.system('mkdir '+FarmDirectory)
0119 os.system('mkdir '+FarmDirectory+'/RootFiles')
0120 os.system('mkdir '+FarmDirectory+'/Log')
0121 os.system('mkdir '+FarmDirectory+'/InputFile')
0122
0123 for i in range(Job_Start,Job_End):
0124     print('Submitting job number %d' %i)
0125
0126     CreateTheConfigFile(path,Input_ConfigFile,Job_NEvents,path+"/"+FarmDirectory+"/RootFiles/"+Output_RootFile,i)
0127     CreateTheShellFile(path,i)
0128
0129     condor_path = "./"+FarmDirectory+"/InputFile/%04i_" % i
0130         condor_path = condor_path + Output_RootFile + ".cmd"
0131
0132         shell_path = path + "/" + FarmDirectory + "/InputFile/%04i_" % i
0133         shell_path = shell_path + Output_RootFile + ".sh"
0134
0135         cdPath = "cd " + path
0136         cdFarm = "cd " + path + "/" + FarmDirectory
0137         os.system(cdFarm)
0138
0139         JobName = "'" + Output_RootFile + "%04i'" % i
0140 #        OutputPath = "Log/%04i_" % i
0141 #        OutputPath = OutputPath + Output_RootFile + "/"
0142         OutputPath = "'out/'"
0143
0144 #        batchSubmit = "bsub -q" + QUEUE + " -J" + JobName  + "'" + shell_path + " 0 ele'"
0145 #        batchSubmit = "bsub -q " + QUEUE + " -J " + JobName  + " -oo " + OutputPath + " -eo " + OutputPath + " '" + shell_path + " 0 ele'"
0146         batchSubmit = "bsub -q " + QUEUE + " -J " + JobName  + " '" + shell_path + " 0 ele'"
0147     os.system (batchSubmit)
0148         os.system(cdPath)
0149
0150
0151     print('\n')
0152 NJobs = Job_End - Job_Start
0153 print("\n\n")
0154 print("\t\t\t%i Jobs submitted by the LaunchOnFarm script" % NJobs)
0155 print("\t\t\t         Created by Loic Quertenmont")
0156 print("\t\t\t           Loic.quertenmont@cern.ch")
0157 print("\n\n")