Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-06-16 03:19:21

0001 #!/usr/bin/env python3
0002 
0003 '''Script that submits CMS Tracker Alignment Primary Vertex Validation workflows
0004 '''
0005 from __future__ import print_function
0006 
0007 from builtins import range
0008 __author__ = 'Marco Musich'
0009 __copyright__ = 'Copyright 2015, CERN CMS'
0010 __credits__ = ['Ernesto Migliore', 'Salvatore Di Guida', 'Javier Duarte']
0011 __license__ = 'Unknown'
0012 __maintainer__ = 'Marco Musich'
0013 __email__ = 'marco.musich@cern.ch'
0014 __version__ = 1
0015 
0016 import datetime,time
0017 import os,sys
0018 import copy
0019 import string, re
0020 import configparser as ConfigParser, json
0021 from optparse import OptionParser
0022 from subprocess import Popen, PIPE
0023 
0024 CopyRights  = '##################################\n'
0025 CopyRights += '#      submitAllJobs Script      #\n'
0026 CopyRights += '#      marco.musich@cern.ch      #\n'
0027 CopyRights += '#         December 2015          #\n'
0028 CopyRights += '##################################\n'
0029 
0030 ##############################################
0031 def drawProgressBar(percent, barLen=40):
0032 ##############################################
0033     sys.stdout.write("\r")
0034     progress = ""
0035     for i in range(barLen):
0036         if i < int(barLen * percent):
0037             progress += "="
0038         else:
0039             progress += " "
0040     sys.stdout.write("[ %s ] %.2f%%" % (progress, percent * 100))
0041     sys.stdout.flush()
0042 
0043 ##############################################
0044 def getCommandOutput(command):
0045 ##############################################
0046     """This function executes `command` and returns it output.
0047     Arguments:
0048     - `command`: Shell command to be invoked by this function.
0049     """
0050     child = os.popen(command)
0051     data = child.read()
0052     err = child.close()
0053     if err:
0054         print('%s failed w/ exit code %d' % (command, err))
0055     return data
0056 
0057 ##############################################
0058 def to_bool(value):
0059 ##############################################
0060     """
0061        Converts 'something' to boolean. Raises exception for invalid formats
0062            Possible True  values: 1, True, "1", "TRue", "yes", "y", "t"
0063            Possible False values: 0, False, None, [], {}, "", "0", "faLse", "no", "n", "f", 0.0, ...
0064     """
0065     if str(value).lower() in ("yes", "y", "true",  "t", "1"): return True
0066     if str(value).lower() in ("no",  "n", "false", "f", "0", "0.0", "", "none", "[]", "{}"): return False
0067     raise Exception('Invalid value for boolean conversion: ' + str(value))
0068 
0069 ####################--- Classes ---############################
0070 class BetterConfigParser(ConfigParser.ConfigParser):
0071 
0072     ##############################################
0073     def optionxform(self, optionstr):
0074         return optionstr
0075 
0076     ##############################################
0077     def exists( self, section, option):
0078          try:
0079              items = self.items(section) 
0080          except ConfigParser.NoSectionError:
0081              return False
0082          for item in items:
0083              if item[0] == option:
0084                  return True
0085          return False
0086 
0087     ##############################################
0088     def __updateDict( self, dictionary, section ):
0089         result = dictionary
0090         try:
0091             for option in self.options( section ):
0092                 result[option] = self.get( section, option )
0093             if "local"+section.title() in self.sections():
0094                 for option in self.options( "local"+section.title() ):
0095                     result[option] = self.get( "local"+section.title(),option )
0096         except ConfigParser.NoSectionError as section:
0097             msg = ("%s in configuration files. This section is mandatory."
0098                    %(str(section).replace(":", "", 1)))
0099             #raise AllInOneError(msg)
0100         return result     
0101 
0102     ##############################################
0103     def getResultingSection( self, section, defaultDict = {}, demandPars = [] ):
0104         result = copy.deepcopy(defaultDict)
0105         for option in demandPars:
0106             try:
0107                 result[option] = self.get( section, option )
0108             except ConfigParser.NoOptionError as globalSectionError:
0109                 globalSection = str( globalSectionError ).split( "'" )[-2]
0110                 splittedSectionName = section.split( ":" )
0111                 if len( splittedSectionName ) > 1:
0112                     localSection = ("local"+section.split( ":" )[0].title()+":"
0113                                     +section.split(":")[1])
0114                 else:
0115                     localSection = ("local"+section.split( ":" )[0].title())
0116                 if self.has_section( localSection ):
0117                     try:
0118                         result[option] = self.get( localSection, option )
0119                     except ConfigParser.NoOptionError as option:
0120                         msg = ("%s. This option is mandatory."
0121                                %(str(option).replace(":", "", 1).replace(
0122                                    "section",
0123                                    "section '"+globalSection+"' or", 1)))
0124                         #raise AllInOneError(msg)
0125                 else:
0126                     msg = ("%s. This option is mandatory."
0127                            %(str(globalSectionError).replace(":", "", 1)))
0128                     #raise AllInOneError(msg)
0129         result = self.__updateDict( result, section )
0130         #print result
0131         return result
0132 
0133 ##### method to parse the input file ################################
0134 def ConfigSectionMap(config, section):
0135     the_dict = {}
0136     options = config.options(section)
0137     for option in options:
0138         try:
0139             the_dict[option] = config.get(section, option)
0140             if the_dict[option] == -1:
0141                 DebugPrint("skip: %s" % option)
0142         except:
0143             print("exception on %s!" % option)
0144             the_dict[option] = None
0145     return the_dict
0146 
0147 ###### method to create recursively directories on EOS #############
0148 def mkdir_eos(out_path):
0149     newpath='/'
0150     for dir in out_path.split('/'):
0151         newpath=os.path.join(newpath,dir)
0152         # do not issue mkdir from very top of the tree
0153         if newpath.find('test_out') > 0:
0154             p = subprocess.Popen(["eos", "mkdir", newpath], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0155             (out, err) = p.communicate()
0156             p.wait()
0157 
0158     # now check that the directory exists
0159     p = subprocess.Popen(["eos", "ls", out_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0160     (out, err) = p.communicate()
0161     p.wait()
0162     if p.returncode !=0:
0163         print(out)
0164 
0165 def split(sequence, size):
0166 ##########################    
0167 # aux generator function to split lists
0168 # based on http://sandrotosi.blogspot.com/2011/04/python-group-list-in-sub-lists-of-n.html
0169 # about generators see also http://stackoverflow.com/questions/231767/the-python-yield-keyword-explained
0170 ##########################
0171     for i in range(0, len(sequence), size):
0172         yield sequence[i:i+size] 
0173 
0174 #############
0175 class Job:
0176 #############
0177 
0178     def __init__(self, job_id, job_name, isDA, isMC, applyBOWS, applyEXTRACOND, extraconditions, runboundary, lumilist, maxevents, gt, allFromGT, alignmentDB, alignmentTAG, apeDB, apeTAG, bowDB, bowTAG, vertextype, tracktype, applyruncontrol, ptcut, CMSSW_dir ,the_dir):
0179 ###############################
0180         self.job_id=job_id    
0181         self.batch_job_id = None 
0182         self.job_name=job_name
0183         
0184         self.isDA              = isDA             
0185         self.isMC              = isMC             
0186         self.applyBOWS         = applyBOWS
0187         self.applyEXTRACOND    = applyEXTRACOND
0188         self.extraCondVect     = extraconditions
0189         self.runboundary       = runboundary         
0190         self.lumilist          = lumilist         
0191         self.maxevents         = maxevents
0192         self.gt                = gt
0193         self.allFromGT         = allFromGT
0194         self.alignmentDB       = alignmentDB      
0195         self.alignmentTAG      = alignmentTAG     
0196         self.apeDB             = apeDB            
0197         self.apeTAG            = apeTAG           
0198         self.bowDB             = bowDB            
0199         self.bowTAG            = bowTAG           
0200         self.vertextype        = vertextype       
0201         self.tracktype         = tracktype        
0202         self.applyruncontrol   = applyruncontrol  
0203         self.ptcut             = ptcut            
0204 
0205         self.the_dir=the_dir
0206         self.CMSSW_dir=CMSSW_dir
0207 
0208         self.output_full_name=self.getOutputBaseName()+"_"+str(self.job_id)
0209 
0210         self.cfg_dir=None
0211         self.outputCfgName=None
0212         
0213         # LSF variables        
0214         self.LSF_dir=None
0215         self.output_LSF_name=None
0216 
0217         self.lfn_list=list()      
0218 
0219         #self.OUTDIR = "" # TODO: write a setter method
0220         #self.OUTDIR = self.createEOSout()
0221 
0222     def __del__(self):
0223 ###############################
0224         del self.lfn_list
0225 
0226     def setEOSout(self,theEOSdir):    
0227 ###############################
0228         self.OUTDIR = theEOSdir
0229           
0230     def getOutputBaseName(self):
0231 ########################    
0232         return "PVValidation_"+self.job_name
0233         
0234     def createTheCfgFile(self,lfn):
0235 ###############################
0236 
0237         global CopyRights
0238 
0239         # write the cfg file 
0240         self.cfg_dir = os.path.join(self.the_dir,"cfg")
0241         if not os.path.exists(self.cfg_dir):
0242             os.makedirs(self.cfg_dir)
0243 
0244         self.outputCfgName=self.output_full_name+"_cfg.py"
0245         fout=open(os.path.join(self.cfg_dir,self.outputCfgName),'w+b')
0246 
0247         # decide which template according to data/mc
0248         if self.isMC:
0249             template_cfg_file = os.path.join(self.the_dir,"PVValidation_TEMPL_cfg.py")
0250         else:
0251             template_cfg_file = os.path.join(self.the_dir,"PVValidation_TEMPL_cfg.py")
0252 
0253         fin = open(template_cfg_file)
0254 
0255         config_txt = '\n\n' + CopyRights + '\n\n'
0256         config_txt += fin.read()
0257 
0258         config_txt=config_txt.replace("ISDATEMPLATE",self.isDA)                 
0259         config_txt=config_txt.replace("ISMCTEMPLATE",self.isMC)                  
0260         config_txt=config_txt.replace("APPLYBOWSTEMPLATE",self.applyBOWS)       
0261         config_txt=config_txt.replace("EXTRACONDTEMPLATE",self.applyEXTRACOND)                  
0262         config_txt=config_txt.replace("USEFILELISTTEMPLATE","True")             
0263         config_txt=config_txt.replace("RUNBOUNDARYTEMPLATE",self.runboundary)   
0264         config_txt=config_txt.replace("LUMILISTTEMPLATE",self.lumilist)         
0265         config_txt=config_txt.replace("MAXEVENTSTEMPLATE",self.maxevents)       
0266         config_txt=config_txt.replace("GLOBALTAGTEMPLATE",self.gt)              
0267         config_txt=config_txt.replace("ALLFROMGTTEMPLATE",self.allFromGT)       
0268         config_txt=config_txt.replace("ALIGNOBJTEMPLATE",self.alignmentDB)      
0269         config_txt=config_txt.replace("GEOMTAGTEMPLATE",self.alignmentTAG)      
0270         config_txt=config_txt.replace("APEOBJTEMPLATE",self.apeDB)              
0271         config_txt=config_txt.replace("ERRORTAGTEMPLATE",self.apeTAG)           
0272         config_txt=config_txt.replace("BOWSOBJECTTEMPLATE",self.bowDB)          
0273         config_txt=config_txt.replace("BOWSTAGTEMPLATE",self.bowTAG)            
0274         config_txt=config_txt.replace("VERTEXTYPETEMPLATE",self.vertextype)     
0275         config_txt=config_txt.replace("TRACKTYPETEMPLATE",self.tracktype)       
0276         config_txt=config_txt.replace("PTCUTTEMPLATE",self.ptcut)               
0277         config_txt=config_txt.replace("RUNCONTROLTEMPLATE",self.applyruncontrol)
0278         lfn_with_quotes = map(lambda x: "\'"+x+"\'",lfn)                           
0279         config_txt=config_txt.replace("FILESOURCETEMPLATE","["+",".join(lfn_with_quotes)+"]")         
0280         config_txt=config_txt.replace("OUTFILETEMPLATE",self.output_full_name+".root")         
0281 
0282         fout.write(config_txt)
0283         
0284         for line in fin.readlines():
0285 
0286             if 'END OF EXTRA CONDITIONS' in line:
0287                 for element in self.extraCondVect :
0288                     if("Rcd" in element):
0289                         params = self.extraCondVect[element].split(',')
0290 
0291                         fout.write(" \n")
0292                         fout.write("     process.conditionsIn"+element+"= CalibTracker.Configuration.Common.PoolDBESSource_cfi.poolDBESSource.clone( \n")
0293                         fout.write("          connect = cms.string('"+params[0]+"'), \n")
0294                         fout.write("          toGet = cms.VPSet(cms.PSet(record = cms.string('"+element+"'), \n")
0295                         fout.write("                                     tag = cms.string('"+params[1]+"'), \n")
0296                         if (len(params)>2):
0297                             fout.write("                                     label = cms.string('"+params[2]+"') \n")
0298                         fout.write("                                     ) \n")
0299                         fout.write("                            ) \n")
0300                         fout.write("          ) \n")
0301                         fout.write("     process.prefer_conditionsIn"+element+" = cms.ESPrefer(\"PoolDBESSource\", \"conditionsIn"+element[0]+"\") \n \n") 
0302                     
0303             fout.write(line)    
0304       
0305         fout.close()                
0306                           
0307     def createTheLSFFile(self):
0308 ###############################
0309 
0310        # directory to store the LSF to be submitted
0311         self.LSF_dir = os.path.join(self.the_dir,"LSF")
0312         if not os.path.exists(self.LSF_dir):
0313             os.makedirs(self.LSF_dir)
0314 
0315         self.output_LSF_name=self.output_full_name+".lsf"
0316         fout=open(os.path.join(self.LSF_dir,self.output_LSF_name),'w')
0317     
0318         job_name = self.output_full_name
0319 
0320         log_dir = os.path.join(self.the_dir,"log")
0321         if not os.path.exists(log_dir):
0322             os.makedirs(log_dir)
0323 
0324         fout.write("#!/bin/sh \n") 
0325         fout.write("#BSUB -L /bin/sh\n")       
0326         fout.write("#BSUB -J "+job_name+"\n")
0327         fout.write("#BSUB -o "+os.path.join(log_dir,job_name+".log")+"\n")
0328         fout.write("#BSUB -q cmscaf1nd \n")
0329         fout.write("JobName="+job_name+" \n")
0330         fout.write("OUT_DIR="+self.OUTDIR+" \n")
0331         fout.write("LXBATCH_DIR=`pwd` \n") 
0332         fout.write("cd "+os.path.join(self.CMSSW_dir,"src")+" \n")
0333         fout.write("eval `scram runtime -sh` \n")
0334         fout.write("cd $LXBATCH_DIR \n") 
0335         fout.write("cmsRun "+os.path.join(self.cfg_dir,self.outputCfgName)+" \n")
0336         fout.write("ls -lh . \n")
0337         fout.write("for RootOutputFile in $(ls *root ); do xrdcp -f ${RootOutputFile}  root://eoscms//eos/cms${OUT_DIR}/${RootOutputFile} ; done \n")
0338         fout.write("for TxtOutputFile in $(ls *txt ); do xrdcp -f ${TxtOutputFile}  root://eoscms//eos/cms${OUT_DIR}/${TxtOutputFile} ; done \n")
0339 
0340         fout.close()
0341 
0342     def getOutputFileName(self):
0343 ############################################
0344         return os.path.join(self.OUTDIR,self.output_full_name+".root")
0345         
0346     def submit(self):
0347 ###############################        
0348         print("submit job", self.job_id)
0349         job_name = self.output_full_name
0350         submitcommand1 = "chmod u+x " + os.path.join(self.LSF_dir,self.output_LSF_name)
0351         child1  = os.system(submitcommand1)
0352         #submitcommand2 = "bsub < "+os.path.join(self.LSF_dir,self.output_LSF_name)
0353         #child2  = os.system(submitcommand2)
0354         self.batch_job_id = getCommandOutput("bsub < "+os.path.join(self.LSF_dir,self.output_LSF_name))
0355 
0356     def getBatchjobId(self):    
0357 ############################################
0358        return self.batch_job_id.split("<")[1].split(">")[0] 
0359 
0360 ##############################################
0361 def main():
0362 ##############################################
0363 
0364     global CopyRights
0365     print('\n'+CopyRights)
0366 
0367     # CMSSW section
0368     input_CMSSW_BASE = os.environ.get('CMSSW_BASE')
0369     AnalysisStep_dir = os.path.join(input_CMSSW_BASE,"src/Alignment/OfflineValidation/test")
0370     sourceModule     = os.path.join(input_CMSSW_BASE,"src/Alignment/OfflineValidation/test","PVValidation_HLTPhysics2015B_TkAlMinBias_cff.py")
0371     lib_path = os.path.abspath(AnalysisStep_dir)
0372     sys.path.append(lib_path)
0373 
0374     ## N.B.: this is dediced here once and for all
0375     srcFiles        = []
0376 
0377     desc="""This is a description of %prog."""
0378     parser = OptionParser(description=desc,version='%prog version 0.1')
0379     parser.add_option('-s','--submit',    help='job submitted',    dest='submit',     action='store_true',  default=False)
0380     parser.add_option('-j','--jobname',   help='task name',        dest='taskname',   action='store',       default='')
0381     parser.add_option('-D','--dataset',   help='selected dataset', dest='data',       action='store'      , default='')
0382     parser.add_option('-r','--doRunBased',help='selected dataset', dest='doRunBased', action='store_true' , default=False)
0383     parser.add_option('-i','--input',     help='set input configuration (overrides default)', dest='inputconfig',action='store',default=None)
0384    
0385     (opts, args) = parser.parse_args()
0386 
0387     now = datetime.datetime.now()
0388     t = now.strftime("test_%Y_%m_%d_%H_%M_%S_DATA_")
0389     t+=opts.taskname
0390     
0391     USER = os.environ.get('USER')
0392     eosdir=os.path.join("/store/caf/user",USER,"test_out",t)
0393     #mkdir_eos(eosdir)
0394 
0395     #### Initialize all the variables
0396 
0397     jobName         = None
0398     isMC            = None
0399     isDA            = None
0400     doRunBased      = False
0401     maxevents       = None
0402 
0403     gt              = None
0404     allFromGT       = None
0405     applyEXTRACOND  = None
0406     extraCondVect   = None      
0407     alignmentDB     = None
0408     alignmentTAG    = None
0409     apeDB           = None
0410     apeTAG          = None
0411     applyBOWS       = None
0412     bowDB           = None
0413     bowTAG          = None
0414 
0415     vertextype      = None
0416     tracktype       = None
0417 
0418     applyruncontrol = None
0419     ptcut           = None
0420     runboundary     = None
0421     lumilist        = None
0422       
0423     ConfigFile = opts.inputconfig
0424     
0425     if ConfigFile is not None:
0426 
0427         print("********************************************************")
0428         print("* Parsing from input file:", ConfigFile," ")
0429         
0430         #config = ConfigParser.ConfigParser()
0431         #config.read(ConfigFile)
0432 
0433         config = BetterConfigParser()
0434         config.read(ConfigFile)
0435 
0436         #print  config.sections()
0437 
0438         # please notice: since in principle one wants to run on several different samples simultaneously,
0439         # all these inputs are vectors
0440 
0441         jobName          = [ConfigSectionMap(config,"Job")['jobname']]
0442         isDA             = [ConfigSectionMap(config,"Job")['isda']]
0443         isMC             = [ConfigSectionMap(config,"Job")['ismc']]
0444         doRunBased       = opts.doRunBased
0445         maxevents        = [ConfigSectionMap(config,"Job")['maxevents']]
0446 
0447         gt               = [ConfigSectionMap(config,"Conditions")['gt']]
0448         allFromGT        = [ConfigSectionMap(config,"Conditions")['allFromGT']]
0449         applyEXTRACOND   = [ConfigSectionMap(config,"Conditions")['applyextracond']]
0450         conditions       = [config.getResultingSection("ExtraConditions")]
0451 
0452         alignmentDB      = [ConfigSectionMap(config,"Conditions")['alignmentdb']]
0453         alignmentTAG     = [ConfigSectionMap(config,"Conditions")['alignmenttag']]
0454         apeDB            = [ConfigSectionMap(config,"Conditions")['apedb']]
0455         apeTAG           = [ConfigSectionMap(config,"Conditions")['apetag']]
0456         applyBOWS        = [ConfigSectionMap(config,"Conditions")['applybows']]
0457         bowDB            = [ConfigSectionMap(config,"Conditions")['bowdb']]
0458         bowTAG           = [ConfigSectionMap(config,"Conditions")['bowtag']]
0459         
0460         vertextype       = [ConfigSectionMap(config,"Type")['vertextype']]      
0461         tracktype        = [ConfigSectionMap(config,"Type")['tracktype']]
0462     
0463         applyruncontrol  = [ConfigSectionMap(config,"Selection")['applyruncontrol']]
0464         ptcut            = [ConfigSectionMap(config,"Selection")['ptcut']]
0465         runboundary      = [ConfigSectionMap(config,"Selection")['runboundary']]
0466         lumilist         = [ConfigSectionMap(config,"Selection")['lumilist']]
0467                       
0468     else :
0469 
0470         print("********************************************************")
0471         print("* Parsing from command line                            *")
0472         print("********************************************************")
0473           
0474         jobName         = ['MinBiasQCD_CSA14Ali_CSA14APE']
0475         isDA            = ['True']   
0476         isMC            = ['True']
0477         doRunBased      = opts.doRunBased
0478         maxevents       = ['10000']
0479         
0480         gt              = ['START53_V7A::All']       
0481         allFromGT       = ['False']
0482         applyEXTRACOND  = ['False']
0483         conditions      = [[('SiPixelTemplateDBObjectRcd','frontier://FrontierProd/CMS_COND_31X_PIXEL','SiPixelTemplates38T_2010_2011_mc'),
0484                             ('SiPixelQualityFromDBRcd','frontier://FrontierProd/CMS_COND_31X_PIXEL','SiPixelQuality_v20_mc')]]
0485         alignmentDB     = ['sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/PayLoads/TkAl-14-02_CSA14/Alignments_CSA14_v1.db']
0486         alignmentTAG    = ['TrackerCSA14Scenario']  
0487         apeDB           = ['sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/PayLoads/TkAl-14-02_CSA14/AlignmentErrors_CSA14_v1.db']  
0488         apeTAG          = ['TrackerCSA14ScenarioErrors']
0489         applyBOWS       = ['True']  
0490         bowDB           = ['frontier://FrontierProd/CMS_COND_310X_ALIGN']  
0491         bowTAG          = ['TrackerSurfaceDeformations_2011Realistic_v2_mc']  
0492         
0493         vertextype      = ['offlinePrimaryVertices']  
0494         tracktype       = ['ALCARECOTkAlMinBias']  
0495         
0496         applyruncontrol = ['False']  
0497         ptcut           = ['3'] 
0498         runboundary     = ['1']  
0499         lumilist        = ['']  
0500  
0501     # start loop on samples
0502 
0503     # print some of the configuration
0504     
0505     print("********************************************************")
0506     print("* Configuration info *")
0507     print("********************************************************")
0508     print("- submitted   : ",opts.submit)
0509     print("- Jobname     : ",jobName)           
0510     print("- use DA      : ",isDA)            
0511     print("- is MC       : ",isMC)            
0512     print("- is run-based: ",doRunBased)
0513     print("- evts/job    : ",maxevents)                    
0514     print("- GlobatTag   : ",gt)      
0515     print("- allFromGT?  : ",allFromGT)
0516     print("- extraCond?  : ",applyEXTRACOND)
0517     print("- extraCond   : ",conditions)                 
0518     print("- Align db    : ",alignmentDB)     
0519     print("- Align tag   : ",alignmentTAG)    
0520     print("- APE db      : ",apeDB)           
0521     print("- APE tag     : ",apeTAG)          
0522     print("- use bows?   : ",applyBOWS)       
0523     print("- K&B db      : ",bowDB)
0524     print("- K&B tag     : ",bowTAG)                        
0525     print("- VertexColl  : ",vertextype)      
0526     print("- TrackColl   : ",tracktype)                       
0527     print("- RunControl? : ",applyruncontrol) 
0528     print("- Pt>           ",ptcut)           
0529     print("- run=          ",runboundary)     
0530     print("- JSON        : ",lumilist)        
0531     print("********************************************************")
0532 
0533     sublogging_dir = os.path.join(AnalysisStep_dir,"submissions")
0534     if not os.path.exists(sublogging_dir):
0535         os.makedirs(sublogging_dir)
0536     submission_log_file = os.path.join(sublogging_dir,"sub"+t+".log")
0537     log_fout = open(submission_log_file,'w')
0538     for iConf in range(len(jobName)):
0539         log_fout.write("============================================================ \n")
0540         log_fout.write("- timestamp   : "+t.strip("test_")+"\n")
0541         log_fout.write("- submitted   : "+str(opts.submit)+"\n")
0542         log_fout.write("- Jobname     : "+jobName[iConf]+"\n")           
0543         log_fout.write("- use DA      : "+isDA[iConf]+"\n")            
0544         log_fout.write("- is MC       : "+isMC[iConf]+"\n")            
0545         log_fout.write("- is run-based: "+str(doRunBased)+"\n")
0546         log_fout.write("- evts/job    : "+maxevents[iConf]+"\n")                    
0547         log_fout.write("- GlobatTag   : "+gt[iConf]+"\n")      
0548         log_fout.write("- allFromGT?  : "+allFromGT[iConf]+"\n")
0549         log_fout.write("- extraCond?  : "+applyEXTRACOND[iConf]+"\n")
0550         for x in conditions:
0551             for attribute,value in x.items():
0552                      log_fout.write('   - {} : {}'.format(attribute, value)+"\n")
0553         log_fout.write("- Align db    : "+alignmentDB[iConf]+"\n")     
0554         log_fout.write("- Align tag   : "+alignmentTAG[iConf]+"\n")    
0555         log_fout.write("- APE db      : "+apeDB[iConf]+"\n")           
0556         log_fout.write("- APE tag     : "+apeTAG[iConf]+"\n")          
0557         log_fout.write("- use bows?   : "+applyBOWS[iConf]+"\n")       
0558         log_fout.write("- K&B db      : "+bowDB[iConf]+"\n")
0559         log_fout.write("- K&B tag     : "+bowTAG[iConf]+"\n")                        
0560         log_fout.write("- VertexColl  : "+vertextype[iConf]+"\n")      
0561         log_fout.write("- TrackColl   : "+tracktype[iConf]+"\n")                       
0562         log_fout.write("- RunControl? : "+applyruncontrol[iConf]+"\n") 
0563         log_fout.write("- Pt>           "+ptcut[iConf]+"\n")           
0564         log_fout.write("- run=          "+runboundary[iConf]+"\n")     
0565         log_fout.write("- JSON        : "+lumilist[iConf]+"\n")
0566         log_fout.write("- output EOS  : "+eosdir+"\n")
0567 
0568     print("Will run on ",len(jobName),"workflows")
0569 
0570     for iConf in range(len(jobName)):
0571         print("Preparing",iConf," configurtion to run")
0572 
0573         # for hadd script
0574         scripts_dir = os.path.join(AnalysisStep_dir,"scripts")
0575         if not os.path.exists(scripts_dir):
0576             os.makedirs(scripts_dir)
0577         hadd_script_file = os.path.join(scripts_dir,jobName[iConf]+".sh")
0578         fout = open(hadd_script_file,'w')
0579 
0580         output_file_list1=list()      
0581         output_file_list2=list()
0582         output_file_list2.append("hadd ")
0583             
0584         inputFiles = []
0585         myRuns = []
0586         
0587         if (to_bool(isMC[iConf]) or (not to_bool(doRunBased))):
0588             if(to_bool(isMC[iConf])):
0589                 print("this is MC")
0590                 cmd = 'dasgoclient -query \'file dataset='+opts.data+'\''
0591                 s = Popen(cmd , shell=True, stdout=PIPE, stderr=PIPE)
0592                 out,err = s.communicate()
0593                 mylist = out.split('\n')
0594                 mylist.pop()
0595                 #print mylist
0596            
0597                 splitList = split(mylist,10)
0598                 for files in splitList:
0599                     inputFiles.append(files)
0600                     myRuns.append(str(1))
0601             else:
0602                 print("this is DATA (not doing full run-based selection)")
0603                 cmd = 'dasgoclient -query \'file dataset='+opts.data+' run='+runboundary[iConf]+'\''
0604                 #print cmd
0605                 s = Popen(cmd , shell=True, stdout=PIPE, stderr=PIPE)
0606                 out,err = s.communicate()
0607                 mylist = out.split('\n')
0608                 mylist.pop()
0609                 #print "len(mylist):",len(mylist)
0610                 print("mylist:",mylist)
0611                 inputFiles.append(mylist)
0612                 myRuns.append(str(runboundary[iConf]))
0613 
0614         else:
0615             print("this is Data")
0616             print("doing run based selection")
0617             cmd = 'dasgoclient -query \'run dataset='+opts.data+'\''
0618             p = Popen(cmd , shell=True, stdout=PIPE, stderr=PIPE)
0619             out, err = p.communicate()
0620             listOfRuns=out.split('\n')
0621             listOfRuns.pop()
0622             listOfRuns.sort()
0623             myRuns = listOfRuns
0624             print("Will run on ",len(listOfRuns), " runs")
0625             print(listOfRuns)
0626 
0627             procs = []
0628 
0629             for run in listOfRuns:
0630                 #print "preparing run",run
0631                 cmd2 = ' dasgoclient -query \'file run='+run+' dataset='+opts.data+'\''
0632                 q = Popen(cmd2 , shell=True, stdout=PIPE, stderr=PIPE)
0633                 procs.append(q)
0634                 #out2, err2 = q.communicate()
0635                 #mylist = out2.split('\n')
0636                 #mylist.pop()
0637                 #inputFiles.append(mylist)
0638                 
0639             toolbar_width = len(listOfRuns)
0640             # setup toolbar
0641             print("********************************************************")
0642             print(" Retrieving run info")
0643             
0644             for i,p in enumerate(procs):
0645                 out2,err2 = p.communicate()
0646                 mylist = out2.split('\n')
0647                 mylist.pop()
0648                 inputFiles.append(mylist)
0649                 #sys.stdout.write("-")
0650                 #sys.stdout.flush()
0651                 percent = float(i)/len(procs)
0652                 #print percent
0653                 drawProgressBar(percent)
0654 
0655             sys.stdout.write("\n") 
0656 
0657         for jobN,theSrcFiles in enumerate(inputFiles):
0658             print(jobN,"run",myRuns[jobN],theSrcFiles)
0659             thejobIndex=None
0660             batchJobIds = []
0661 
0662             #if(to_bool(isMC[iConf]) and (not to_bool(doRunBased))):
0663             if(to_bool(isMC[iConf])):
0664                 thejobIndex=jobN
0665             else:
0666                 thejobIndex=myRuns[jobN]
0667 
0668             aJob = Job(thejobIndex,
0669                        jobName[iConf],isDA[iConf],isMC[iConf],
0670                        applyBOWS[iConf],applyEXTRACOND[iConf],conditions[iConf],
0671                        myRuns[jobN], lumilist[iConf], maxevents[iConf],
0672                        gt[iConf],allFromGT[iConf],
0673                        alignmentDB[iConf], alignmentTAG[iConf],
0674                        apeDB[iConf], apeTAG[iConf],
0675                        bowDB[iConf], bowTAG[iConf],
0676                        vertextype[iConf], tracktype[iConf],
0677                        applyruncontrol[iConf],
0678                        ptcut[iConf],input_CMSSW_BASE,AnalysisStep_dir)
0679             
0680             aJob.setEOSout(eosdir)
0681             aJob.createTheCfgFile(theSrcFiles)
0682             aJob.createTheLSFFile()
0683 
0684             output_file_list1.append("xrdcp root://eoscms//eos/cms"+aJob.getOutputFileName()+" /tmp/$USER/"+opts.taskname+" \n")
0685             if jobN == 0:
0686                 output_file_list2.append("/tmp/$USER/"+opts.taskname+"/"+aJob.getOutputBaseName()+".root ")
0687             output_file_list2.append("/tmp/$USER/"+opts.taskname+"/"+os.path.split(aJob.getOutputFileName())[1]+" ")    
0688    
0689             if opts.submit:
0690                 aJob.submit()
0691                 batchJobIds.append(ajob.getBatchjobId())
0692             del aJob
0693 
0694         if opts.submit:
0695             print("********************************************************")
0696             for theBatchJobId in batchJobIds:
0697                 print("theBatchJobId is: ",theBatchJobId)
0698 
0699         fout.write("#!/bin/bash \n")
0700         fout.write("MAIL = $USER@mail.cern.ch \n")
0701         fout.write("OUT_DIR = "+eosdir+ "\n")
0702         fout.write("echo $HOST | mail -s \"Harvesting job started\" $USER@mail.cern.ch \n")
0703         fout.write("cd "+os.path.join(input_CMSSW_BASE,"src")+"\n")
0704         fout.write("eval `scram r -sh` \n")
0705         fout.write("mkdir -p /tmp/$USER/"+opts.taskname+" \n")
0706         fout.writelines(output_file_list1)
0707         fout.writelines(output_file_list2)
0708         fout.write("\n")
0709         fout.write("echo \"xrdcp -f $FILE root://eoscms//eos/cms$OUT_DIR\" \n")
0710         fout.write("xrdcp -f root://eoscms//eos/cms$FILE $OUT_DIR \n")
0711         fout.write("echo \"Harvesting for complete; please find output at $OUT_DIR \" | mail -s \"Harvesting for" +opts.taskname +" compled\" $MAIL \n")
0712 
0713         os.system("chmod u+x "+hadd_script_file)
0714 
0715         conditions = '"' + " && ".join(["ended(" + jobId + ")" for jobId in batchJobIds]) + '"'
0716         print(conditions)
0717         lastJobCommand = "bsub -o harvester"+opts.taskname+".tmp -q 1nh -w "+conditions+" "+hadd_script_file
0718         print(lastJobCommand)
0719         if opts.submit:
0720             lastJobOutput = getCommandOutput(lastJobCommand)
0721             print(lastJobOutput)
0722 
0723         fout.close()
0724         del output_file_list1
0725         
0726 if __name__ == "__main__":        
0727     main()