MakeCalibrationTrees/submitCalibTree/runJob.py

0001 #!/usr/bin/env python3
0002 import os
0003 import sys
0004 import subprocess
0005 import time
0006 import optparse
0007 from . import Config
0008
0009 usage = 'usage: %prog [options]'
0010 parser = optparse.OptionParser(usage)
0011 parser.add_option('-r', '--run'        ,    dest='runNumber'          , help='Run number to process (-1 --> Submit all)', default='-1')
0012 parser.add_option('-f', '--files'      ,    dest='files'              , help='files to process'                         , default='-1')
0013 parser.add_option('-n', '--firstFile'  ,    dest='firstFile'          , help='first File to process'                    , default='-1')
0014 parser.add_option('-a', '--aag'        ,    dest='aag'                , help='Dataset type (is Aag)'                    , default=False, action="store_true")
0015 parser.add_option('-c', '--corrupted'  ,    dest='corrupted'          , help='Check for corrupted runs'                 , default=False, action="store_true")
0016 parser.add_option('-d', '--dataset'    ,    dest='dataset'            , help='dataset'                                  , default='')
0017 parser.add_option('-s', '--stageout'   ,    dest='stageout'           , help='stageout produced files'                  , default="True")
0018
0019 (opt, args) = parser.parse_args()
0020
0021
0022 print("Running calib tree production...")
0023
0024 files     = opt.files
0025 nFiles    = len(files.split(","))-1
0026 firstFile = int(opt.firstFile)
0027 checkCorrupted = opt.corrupted
0028 AAG      = opt.aag
0029 stageout = (opt.stageout=="True")
0030 print("After the abort gap : %s"%AAG)
0031 conf           = Config.configuration(opt.aag)
0032 run            = int(opt.runNumber)
0033 dataset        = opt.dataset
0034 print(conf.checkIntegrity())
0035 print(conf)
0036 print(files)
0037
0038 PWDDIR  =os.getcwd() #Current Dir
0039 print("Running on %s"%PWDDIR)
0040 os.chdir(conf.RUNDIR);
0041 if firstFile<0:firstFile=0
0042
0043 print("Processing files %i to %i of run %i" % (firstFile,firstFile+nFiles,run))
0044
0045
0046 if(firstFile==0):outfile = 'calibTree_%i.root' % (run)
0047 else:
0048    outfile = 'calibTree_%i_%i.root' % (run, firstFile)
0049 #reinitialize the afs token, to make sure that the job isn't kill after a few hours of running
0050 os.system('/usr/sue/bin/kinit -R')
0051
0052
0053 #BUILD CMSSW CONFIG, START CMSRUN, COPY THE OUTPUT AND CLEAN THE PROJECT
0054 cmd='cmsRun produceCalibrationTree_template_cfg.py'
0055 cmd+=' outputFile="'+PWDDIR+'/'+outfile+'"'
0056 cmd+=' conditionGT="'+conf.globalTag+'"'
0057 cmd+=' inputCollection="'+conf.collection+'"'
0058 if files[-1]==",":files=files[:-1]
0059 cmd+=' inputFiles="'+files.replace("'","")+'"'
0060 cmd+=' runNumber=%s'%run
0061 print(cmd)
0062
0063 exit_code = os.system(conf.initEnv+cmd)
0064 stageOutCode = True
0065 if(int(exit_code)!=0):
0066    print("Job Failed with ExitCode "+str(exit_code))
0067    os.system('echo %i %i %i >> FailledRun%s.txt' % (run, firstFile, firstFile+nFiles,'_Aag' if AAG else ''))
0068 else:
0069    FileSizeInKBytes = subprocess.getstatusoutput('ls  -lth --block-size=1024 '+PWDDIR+'/'+outfile)[1].split()[4]
0070    if(int(FileSizeInKBytes)>10 and stageout):
0071       print("Preparing for stageout of " + PWDDIR+'/'+outfile + ' on ' + conf.CASTORDIR+'/'+outfile + '.  The file size is %d KB' % int(FileSizeInKBytes))
0072       cpCmd = "eos cp %s/%s "%(PWDDIR,outfile)
0073       cpCmd+= "root://eoscms.cern.ch//eos/cms/%s/%s"%(conf.CASTORDIR,outfile)
0074       stageOutCode&= not os.system(conf.initEnv+" "+cpCmd)
0075       print(conf.eosLs + conf.CASTORDIR+'/'+outfile)
0076       stageOutCode&= not os.system("eos ls " + conf.CASTORDIR+'/'+outfile)
0077    else:
0078       print('File size is %d KB, this is under the threshold --> the file will not be transfered on EOS' % int(FileSizeInKBytes))
0079       print("Stageout status = %s"%stageout)
0080 if not stageOutCode:
0081    print("WARNING WARNING WARNING STAGE OUT FAILED BUT NOT RELAUNCHED")
0082
0083 os.system('ls -lth '+PWDDIR+'/'+outfile)
0084 if stageout:
0085    os.system('rm -f '+PWDDIR+'/'+outfile)
0086    os.system('rm -f ConfigFile_'+str(run)+'_'+str(firstFile)+'_cfg.py')
0087    os.system('cd ' + conf.RUNDIR)
0088    os.system('rm -rf LSFJOB_${LSB_JOBID}')