MakeCalibrationTrees/submitCalibTree/runJob.py

0001 #!/usr/bin/env python3
0002 from __future__ import absolute_import
0003 from __future__ import print_function
0004 import os
0005 import sys
0006 import subprocess
0007 import time
0008 import optparse
0009 from . import Config
0010
0011 usage = 'usage: %prog [options]'
0012 parser = optparse.OptionParser(usage)
0013 parser.add_option('-r', '--run'        ,    dest='runNumber'          , help='Run number to process (-1 --> Submit all)', default='-1')
0014 parser.add_option('-f', '--files'      ,    dest='files'              , help='files to process'                         , default='-1')
0015 parser.add_option('-n', '--firstFile'  ,    dest='firstFile'          , help='first File to process'                    , default='-1')
0016 parser.add_option('-a', '--aag'        ,    dest='aag'                , help='Dataset type (is Aag)'                    , default=False, action="store_true")
0017 parser.add_option('-c', '--corrupted'  ,    dest='corrupted'          , help='Check for corrupted runs'                 , default=False, action="store_true")
0018 parser.add_option('-d', '--dataset'    ,    dest='dataset'            , help='dataset'                                  , default='')
0019 parser.add_option('-s', '--stageout'   ,    dest='stageout'           , help='stageout produced files'                  , default="True")
0020
0021 (opt, args) = parser.parse_args()
0022
0023
0024 print("Running calib tree production...")
0025
0026 files     = opt.files
0027 nFiles    = len(files.split(","))-1
0028 firstFile = int(opt.firstFile)
0029 checkCorrupted = opt.corrupted
0030 AAG      = opt.aag
0031 stageout = (opt.stageout=="True")
0032 print("After the abort gap : %s"%AAG)
0033 conf           = Config.configuration(opt.aag)
0034 run            = int(opt.runNumber)
0035 dataset        = opt.dataset
0036 print(conf.checkIntegrity())
0037 print(conf)
0038 print(files)
0039
0040 PWDDIR  =os.getcwd() #Current Dir
0041 print("Running on %s"%PWDDIR)
0042 os.chdir(conf.RUNDIR);
0043 if firstFile<0:firstFile=0
0044
0045 print("Processing files %i to %i of run %i" % (firstFile,firstFile+nFiles,run))
0046
0047
0048 if(firstFile==0):outfile = 'calibTree_%i.root' % (run)
0049 else:
0050    outfile = 'calibTree_%i_%i.root' % (run, firstFile)
0051 #reinitialize the afs token, to make sure that the job isn't kill after a few hours of running
0052 os.system('/usr/sue/bin/kinit -R')
0053
0054
0055 #BUILD CMSSW CONFIG, START CMSRUN, COPY THE OUTPUT AND CLEAN THE PROJECT
0056 cmd='cmsRun produceCalibrationTree_template_cfg.py'
0057 cmd+=' outputFile="'+PWDDIR+'/'+outfile+'"'
0058 cmd+=' conditionGT="'+conf.globalTag+'"'
0059 cmd+=' inputCollection="'+conf.collection+'"'
0060 if files[-1]==",":files=files[:-1]
0061 cmd+=' inputFiles="'+files.replace("'","")+'"'
0062 cmd+=' runNumber=%s'%run
0063 print(cmd)
0064
0065 exit_code = os.system(conf.initEnv+cmd)
0066 stageOutCode = True
0067 if(int(exit_code)!=0):
0068    print("Job Failed with ExitCode "+str(exit_code))
0069    os.system('echo %i %i %i >> FailledRun%s.txt' % (run, firstFile, firstFile+nFiles,'_Aag' if AAG else ''))
0070 else:
0071    FileSizeInKBytes = subprocess.getstatusoutput('ls  -lth --block-size=1024 '+PWDDIR+'/'+outfile)[1].split()[4]
0072    if(int(FileSizeInKBytes)>10 and stageout):
0073       print("Preparing for stageout of " + PWDDIR+'/'+outfile + ' on ' + conf.CASTORDIR+'/'+outfile + '.  The file size is %d KB' % int(FileSizeInKBytes))
0074       cpCmd = "eos cp %s/%s "%(PWDDIR,outfile)
0075       cpCmd+= "root://eoscms.cern.ch//eos/cms/%s/%s"%(conf.CASTORDIR,outfile)
0076       stageOutCode&= not os.system(conf.initEnv+" "+cpCmd)
0077       print(conf.eosLs + conf.CASTORDIR+'/'+outfile)
0078       stageOutCode&= not os.system("eos ls " + conf.CASTORDIR+'/'+outfile)
0079    else:
0080       print('File size is %d KB, this is under the threshold --> the file will not be transfered on EOS' % int(FileSizeInKBytes))
0081       print("Stageout status = %s"%stageout)
0082 if not stageOutCode:
0083    print("WARNING WARNING WARNING STAGE OUT FAILED BUT NOT RELAUNCHED")
0084
0085 os.system('ls -lth '+PWDDIR+'/'+outfile)
0086 if stageout:
0087    os.system('rm -f '+PWDDIR+'/'+outfile)
0088    os.system('rm -f ConfigFile_'+str(run)+'_'+str(firstFile)+'_cfg.py')
0089    os.system('cd ' + conf.RUNDIR)
0090    os.system('rm -rf LSFJOB_${LSB_JOBID}')