Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-25 02:29:02

0001 #!/usr/bin/env python3
0002 
0003 import sys
0004 import imp
0005 import copy
0006 import os
0007 import shutil
0008 import pickle
0009 import math
0010 import pprint
0011 import subprocess
0012 from datetime import date
0013 from optparse import OptionParser
0014 
0015 
0016 class MyBatchManager:
0017    '''Batch manager specific to cmsRun processes.'''
0018 
0019    def __init__(self):
0020       # define options and arguments ====================================
0021       self.parser = OptionParser()
0022       self.parser.add_option("-o", "--outdir", dest="outputdir", type="string",
0023                                 help="Name of the local output directory for your jobs. This directory will be created automatically.",
0024                                 default="./")
0025       self.parser.add_option("--commoncfg", dest="commoncfg", type="string",
0026                                 help="Name of the common config file.",
0027                                 default="python/common_cff_py.txt")
0028       self.parser.add_option("--aligncfg", dest="aligncfg", type="string",
0029                                 help="Name of the align. config file.",
0030                                 default="python/align_tpl_py.txt")
0031       self.parser.add_option("--niter", dest="niter", type="int",
0032                                 help="Number of iterations",
0033                                 default="15")
0034       self.parser.add_option("--lst", "--listfile", "--lstfile", dest="lstfile", type="string",
0035                                 help="lst file to read",
0036                                 default=None)
0037       self.parser.add_option("--iovs", "--iovfile", dest="iovfile", type="string",
0038                                 help="IOV list to read",
0039                                 default=None)
0040       self.parser.add_option("--trkselcfg", "--trackselectionconfig", dest="trkselcfg", type="string",
0041                                 help="Track selection config location",
0042                                 default="python")
0043       self.parser.add_option("--notify", "--sendto", dest="sendto", type="string",
0044                                 help="Email addresses (comma-separated) to notify when job is complete.",
0045                                 default=None)
0046       self.parser.add_option("--deform", action="store_true",
0047                                 dest="useSD", default=False,
0048                                 help="Include surface deformations in alignment")
0049       self.parser.add_option("-f", "--force", action="store_true",
0050                                 dest="force", default=False,
0051                                 help="Don't ask any questions, just over-write")
0052       self.parser.add_option("--resubmit", action="store_true",
0053                                 dest="resubmit", default=False,
0054                                 help="Resubmit a job from the last iteration")
0055       self.parser.add_option("--redirectproxy", action="store_true",
0056                                 dest="redirectproxy", default=False,
0057                                 help="Redirect the proxy to a path visible in batch")
0058       self.parser.add_option("--dry", dest="dryRun", type="int",
0059                                 default=0,
0060                                 help="Do not submit jobs, just set up the cfg files")
0061       (self.opt,self.args) = self.parser.parse_args()
0062 
0063       self.checkProxy() # Check if Grid proxy initialized
0064 
0065       self.mkdir(self.opt.outputdir)
0066 
0067       if self.opt.lstfile is None:
0068          print("Unspecified lst file.")
0069          sys.exit(1)
0070       if self.opt.iovfile is None:
0071          print("Unspecified IOV list.")
0072          sys.exit(1)
0073 
0074       self.jobname = self.opt.outputdir.split('/')[-1]
0075 
0076       if self.opt.redirectproxy:
0077          print("Job {} is configured to redirect its Grid proxy.".format(self.jobname))
0078          self.redirectProxy()
0079 
0080       if self.opt.sendto is not None:
0081          self.opt.sendto.strip()
0082          self.opt.sendto.replace(","," ")
0083          print("Job {} is configured to notify {}.".format(self.jobname, self.opt.sendto))
0084 
0085       # Set numerical flags for iterator_py
0086       self.SDflag = 1 if self.opt.useSD else 0
0087       self.redirectproxyflag = 1 if self.opt.redirectproxy else 0
0088 
0089 
0090    def mkdir(self, dirname):
0091       mkdir = 'mkdir -p %s' % dirname
0092       ret = os.system( mkdir )
0093       if( ret != 0 ):
0094          print('Please remove or rename directory: ', dirname)
0095          sys.exit(4)
0096 
0097    def notify(self, desc):
0098       print(desc)
0099       if self.opt.sendto is not None:
0100          strcmd = "mail -s {1} {0} <<< \"{2}\"".format(self.opt.sendto, self.jobname, desc)
0101          os.system(strcmd)
0102 
0103    def checkLastIteration(self):
0104       lastIter=self.opt.niter
0105       doesExist=os.system("test -s {}/alignments_iter{}.db".format(self.opt.outputdir, lastIter))
0106       while (doesExist != 0):
0107          lastIter -= 1
0108          if lastIter < 0:
0109             break
0110          doesExist=os.system("test -s {}/alignments_iter{}.db".format(self.opt.outputdir, lastIter))
0111       return lastIter
0112 
0113    def finalize(self, ret):
0114       strresult=""
0115       exitCode=0
0116       if( ret != 0 ):
0117          strresult = "Jobs cannot be submitted for {}. Exiting...".format(self.jobname)
0118          exitCode=1
0119       elif self.opt.dryRun > 0:
0120          strresult = "Dry run setup is complete for {}.".format(self.jobname)
0121       else:
0122          lastIter=self.checkLastIteration()
0123          if lastIter == self.opt.niter:
0124             strresult = "The final iteration {}/alignments_iter{}.db is recorded successfully.".format(self.jobname, lastIter)
0125          elif lastIter>0:
0126             strresult = "The last successful iteration was {}/alignments_iter{}.db out of the {} requested iterations.".format(self.jobname, lastIter, self.opt.niter)
0127             exitCode=1
0128          else:
0129             strresult = "None of the {} iterations were successful in job {}.".format(self.opt.niter, self.jobname)
0130             exitCode=1
0131       self.notify(strresult)
0132       if exitCode!=0:
0133          sys.exit(strresult)
0134 
0135    def submitJobs(self):
0136       jobcmd=""
0137       if self.opt.resubmit:
0138          jobcmd = 'scripts/reiterator_py {} {} {} {} {} {}'.format(
0139          self.opt.niter,
0140          self.opt.outputdir,
0141          self.opt.iovfile
0142          )
0143       else:
0144          if self.opt.dryRun > 0:
0145             print('Dry run option is enabled. Will not submit jobs to the queue')
0146          jobcmd = 'scripts/iterator_py {} {} {} {} {} {} {} {} {} {}'.format(
0147          self.opt.niter,
0148          self.opt.outputdir,
0149          self.opt.lstfile,
0150          self.opt.iovfile,
0151          self.opt.commoncfg,
0152          self.opt.aligncfg,
0153          self.opt.trkselcfg,
0154          self.SDflag,
0155          self.redirectproxyflag,
0156          self.opt.dryRun
0157          )
0158       ret = os.system( jobcmd )
0159       self.finalize(ret)
0160 
0161    def checkProxy(self):
0162       try:
0163          subprocess.check_call(["voms-proxy-info", "--exists"])
0164       except subprocess.CalledProcessError:
0165          print("Please initialize your proxy before submitting.")
0166          sys.exit(1)
0167 
0168    def redirectProxy(self):
0169       local_proxy = subprocess.check_output(["voms-proxy-info", "--path"]).strip()
0170       new_proxy_path = os.path.join(self.opt.outputdir,".user_proxy")
0171       print("Copying local proxy {} to the job directory as {}.".format(local_proxy,new_proxy_path))
0172       shutil.copyfile(local_proxy, new_proxy_path)
0173 
0174 
0175 
0176 if __name__ == '__main__':
0177    batchManager = MyBatchManager()
0178    batchManager.submitJobs()