Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2022-03-03 02:23:11

0001 #!/usr/bin/env python3
0002 
0003 from __future__ import print_function
0004 import sys
0005 import imp
0006 import copy
0007 import os
0008 import shutil
0009 import pickle
0010 import math
0011 import pprint
0012 import subprocess
0013 from datetime import date
0014 from optparse import OptionParser
0015 
0016 
0017 class MyBatchManager:
0018    '''Batch manager specific to cmsRun processes.'''
0019 
0020    def __init__(self):
0021       # define options and arguments ====================================
0022       self.parser = OptionParser()
0023       self.parser.add_option("-o", "--outdir", dest="outputdir", type="string",
0024                                 help="Name of the local output directory for your jobs. This directory will be created automatically.",
0025                                 default="./")
0026       self.parser.add_option("--commoncfg", dest="commoncfg", type="string",
0027                                 help="Name of the common config file.",
0028                                 default="python/common_cff_py.txt")
0029       self.parser.add_option("--aligncfg", dest="aligncfg", type="string",
0030                                 help="Name of the align. config file.",
0031                                 default="python/align_tpl_py.txt")
0032       self.parser.add_option("--niter", dest="niter", type="int",
0033                                 help="Number of iterations",
0034                                 default="15")
0035       self.parser.add_option("--lst", "--listfile", "--lstfile", dest="lstfile", type="string",
0036                                 help="lst file to read",
0037                                 default=None)
0038       self.parser.add_option("--iovs", "--iovfile", dest="iovfile", type="string",
0039                                 help="IOV list to read",
0040                                 default=None)
0041       self.parser.add_option("--trkselcfg", "--trackselectionconfig", dest="trkselcfg", type="string",
0042                                 help="Track selection config location",
0043                                 default="python")
0044       self.parser.add_option("--notify", "--sendto", dest="sendto", type="string",
0045                                 help="Email addresses (comma-separated) to notify when job is complete.",
0046                                 default=None)
0047       self.parser.add_option("--deform", action="store_true",
0048                                 dest="useSD", default=False,
0049                                 help="Include surface deformations in alignment")
0050       self.parser.add_option("-f", "--force", action="store_true",
0051                                 dest="force", default=False,
0052                                 help="Don't ask any questions, just over-write")
0053       self.parser.add_option("--resubmit", action="store_true",
0054                                 dest="resubmit", default=False,
0055                                 help="Resubmit a job from the last iteration")
0056       self.parser.add_option("--redirectproxy", action="store_true",
0057                                 dest="redirectproxy", default=False,
0058                                 help="Redirect the proxy to a path visible in batch")
0059       self.parser.add_option("--dry", dest="dryRun", type="int",
0060                                 default=0,
0061                                 help="Do not submit jobs, just set up the cfg files")
0062       (self.opt,self.args) = self.parser.parse_args()
0063 
0064       self.checkProxy() # Check if Grid proxy initialized
0065 
0066       self.mkdir(self.opt.outputdir)
0067 
0068       if self.opt.lstfile is None:
0069          print("Unspecified lst file.")
0070          sys.exit(1)
0071       if self.opt.iovfile is None:
0072          print("Unspecified IOV list.")
0073          sys.exit(1)
0074 
0075       self.jobname = self.opt.outputdir.split('/')[-1]
0076 
0077       if self.opt.redirectproxy:
0078          print("Job {} is configured to redirect its Grid proxy.".format(self.jobname))
0079          self.redirectProxy()
0080 
0081       if self.opt.sendto is not None:
0082          self.opt.sendto.strip()
0083          self.opt.sendto.replace(","," ")
0084          print("Job {} is configured to notify {}.".format(self.jobname, self.opt.sendto))
0085 
0086       # Set numerical flags for iterator_py
0087       self.SDflag = 1 if self.opt.useSD else 0
0088       self.redirectproxyflag = 1 if self.opt.redirectproxy else 0
0089 
0090 
0091    def mkdir(self, dirname):
0092       mkdir = 'mkdir -p %s' % dirname
0093       ret = os.system( mkdir )
0094       if( ret != 0 ):
0095          print('Please remove or rename directory: ', dirname)
0096          sys.exit(4)
0097 
0098    def notify(self, desc):
0099       print(desc)
0100       if self.opt.sendto is not None:
0101          strcmd = "mail -s {1} {0} <<< \"{2}\"".format(self.opt.sendto, self.jobname, desc)
0102          os.system(strcmd)
0103 
0104    def checkLastIteration(self):
0105       lastIter=self.opt.niter
0106       doesExist=os.system("test -s {}/alignments_iter{}.db".format(self.opt.outputdir, lastIter))
0107       while (doesExist != 0):
0108          lastIter -= 1
0109          if lastIter < 0:
0110             break
0111          doesExist=os.system("test -s {}/alignments_iter{}.db".format(self.opt.outputdir, lastIter))
0112       return lastIter
0113 
0114    def finalize(self, ret):
0115       strresult=""
0116       exitCode=0
0117       if( ret != 0 ):
0118          strresult = "Jobs cannot be submitted for {}. Exiting...".format(self.jobname)
0119          exitCode=1
0120       elif self.opt.dryRun > 0:
0121          strresult = "Dry run setup is complete for {}.".format(self.jobname)
0122       else:
0123          lastIter=self.checkLastIteration()
0124          if lastIter == self.opt.niter:
0125             strresult = "The final iteration {}/alignments_iter{}.db is recorded successfully.".format(self.jobname, lastIter)
0126          elif lastIter>0:
0127             strresult = "The last successful iteration was {}/alignments_iter{}.db out of the {} requested iterations.".format(self.jobname, lastIter, self.opt.niter)
0128             exitCode=1
0129          else:
0130             strresult = "None of the {} iterations were successful in job {}.".format(self.opt.niter, self.jobname)
0131             exitCode=1
0132       self.notify(strresult)
0133       if exitCode!=0:
0134          sys.exit(strresult)
0135 
0136    def submitJobs(self):
0137       jobcmd=""
0138       if self.opt.resubmit:
0139          jobcmd = 'scripts/reiterator_py {} {} {} {} {} {}'.format(
0140          self.opt.niter,
0141          self.opt.outputdir,
0142          self.opt.iovfile
0143          )
0144       else:
0145          if self.opt.dryRun > 0:
0146             print('Dry run option is enabled. Will not submit jobs to the queue')
0147          jobcmd = 'scripts/iterator_py {} {} {} {} {} {} {} {} {} {}'.format(
0148          self.opt.niter,
0149          self.opt.outputdir,
0150          self.opt.lstfile,
0151          self.opt.iovfile,
0152          self.opt.commoncfg,
0153          self.opt.aligncfg,
0154          self.opt.trkselcfg,
0155          self.SDflag,
0156          self.redirectproxyflag,
0157          self.opt.dryRun
0158          )
0159       ret = os.system( jobcmd )
0160       self.finalize(ret)
0161 
0162    def checkProxy(self):
0163       try:
0164          subprocess.check_call(["voms-proxy-info", "--exists"])
0165       except subprocess.CalledProcessError:
0166          print("Please initialize your proxy before submitting.")
0167          sys.exit(1)
0168 
0169    def redirectProxy(self):
0170       local_proxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
0171       new_proxy_path = os.path.join(self.opt.outputdir,".user_proxy")
0172       print("Copying local proxy {} to the job directory as {}.".format(local_proxy,new_proxy_path))
0173       shutil.copyfile(local_proxy, new_proxy_path)
0174 
0175 
0176 
0177 if __name__ == '__main__':
0178    batchManager = MyBatchManager()
0179    batchManager.submitJobs()