test/MakeCalibrationTrees/SubmitJobs.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 import urllib
0004 import string
0005 import os
0006 import sys
0007 import subprocess
0008 import time
0009 import optparse
0010 import submitCalibTree.Config
0011 import submitCalibTree.launchJobs
0012
0013 mailAdd = ""
0014 start = time.strftime("%D %H:%M")
0015
0016 def mail(STDruns,AAGruns,cleanUpLog):
0017    if mailAdd=="":
0018       print("No email address specified.")
0019       return
0020
0021    message  = "Production started at %s\n"%start
0022    message += "             ended at %s\n"%time.strftime("%D %H:%M")
0023    message += "\n\n\n"
0024    message += "Std bunch : processed the following runs :\n"
0025    previousRun=0
0026    nJobs=1
0027    runs = {}
0028    for run in STDruns:
0029       if not run[0] in runs.keys():
0030          runs[run[0]]=1
0031       else:
0032          runs[run[0]]+=1
0033
0034    runsOrdered = sorted(runs.keys())
0035
0036    for r in runsOrdered:
0037       message+=" Run %s (%s jobs) \n"%(r,runs[r])
0038    message +="\n\n"
0039    message += "Aag bunch : processed the following runs :\n"
0040    runs={}
0041    for run in AAGruns:
0042       if not run[0] in runs.keys():
0043          runs[run[0]]=1
0044       else:
0045          runs[run[0]]+=1
0046    runsOrdered = runs.keys()
0047    runsOrdered.sort()
0048
0049    for r in runsOrdered:
0050       message+=" Run %s (%s jobs) \n"%(r,runs[r])
0051
0052    message+="\n\n **** Cleaning report **** \n\n"
0053    message+=cleanUpLog.replace("\"","").replace("'","")
0054    os.system('echo "%s" | mail -s "CalibTree production status" '%message + mailAdd)
0055
0056 debug = False
0057 for arg in sys.argv:
0058    debug +="debug" in arg.lower()
0059 if debug:
0060    print("DEBUG MODE DETECTED")
0061    print("--> Will not submit jobs")
0062    print("--> Will not modify status files")
0063    print()
0064
0065
0066
0067 #Std bunch:
0068 print("Processing Std bunch")
0069 config = submitCalibTree.Config.configuration(debug=debug)
0070 cleanUpMessage = "Unable to clean up folder. Bad clean-up integrity?"
0071
0072 if config.integrity:
0073    print("Cleaning up directory...")
0074    cleanUpMessage = subprocess.getstatusoutput("cd %s; python cleanFolders.py; cd -"%config.RUNDIR)[1]
0075    print(cleanUpMessage)
0076    with open("LastRun.txt","r") as lastRun:
0077       for line in lastRun:
0078          line = line.replace("\n","").replace("\r","")
0079          if line.isdigit():
0080             config.firstRun = int(line)
0081
0082    with open("FailledRun.txt","r") as failled:
0083       for line in failled:
0084          line = line.split()
0085          if len(line)==1:
0086             if line[0].isdigit() and len(line[0])==6:
0087                config.relaunchList.append(line)
0088          elif len(line)==3:
0089             if line[0].isdigit() and line[1].isdigit() and line[2].isdigit and len(line[0])==6:
0090                config.relaunchList.append(line)
0091    if not debug:
0092       with open("FailledRun.txt","w") as failled:
0093          failled.write("")
0094
0095    lastRunProcessed = submitCalibTree.launchJobs.generateJobs(config)
0096
0097    print(config.launchedRuns)
0098
0099    if not debug:
0100       with open("LastRun.txt","w") as lastRun:
0101          lastRun.write(str(lastRunProcessed))
0102
0103
0104
0105 print("Processing AAG")
0106
0107
0108
0109 configAAG = submitCalibTree.Config.configuration(True,debug=debug)
0110 if configAAG.integrity:
0111    with open("LastRun_Aag.txt","r") as lastRun:
0112       for line in lastRun:
0113          line = line.replace("\n","").replace("\r","")
0114          if line.isdigit():
0115             configAAG.firstRun = int(line)
0116
0117
0118    with open("FailledRun_Aag.txt","r") as failled:
0119       for line in failled:
0120          line = line.split()
0121          if len(line)==1:
0122             if line[0].isdigit() and len(line[0])==6:
0123                configAAG.relaunchList.append(line)
0124          elif len(line)==3:
0125             if line[0].isdigit() and line[1].isdigit() and line[2].isdigit and len(line[0])==6:
0126                configAAG.relaunchList.append(line)
0127    if not debug:
0128       with open("FailledRun_Aag.txt","w") as failled:
0129          failled.write("")
0130
0131    lastRunProcessed = submitCalibTree.launchJobs.generateJobs(configAAG)
0132
0133    if not debug:
0134       with open("LastRun_Aag.txt","w") as lastRun:
0135          lastRun.write(str(lastRunProcessed))
0136
0137
0138 mail(config.launchedRuns,configAAG.launchedRuns,cleanUpMessage)