BeamSpotProducer/scripts/beamvalidation.py

0001 #!/usr/bin/env python3
0002 #____________________________________________________________
0003 #
0004 #
0005 # A very simple
0006 #
0007 # Francisco Yumiceva
0008 # yumiceva@fnal.gov
0009 #
0010 # Fermilab, 2010
0011 #
0012 #____________________________________________________________
0013
0014 """
0015    beam spot validation
0016
0017    A very simple script
0018
0019    usage: %prog -t <tag name>
0020    -o, --output    = OUTPUT: filename of output html file.
0021    -p, --path      = PATH: path to beam spot scripts.
0022    -w, --web       = WEB: html path to website.
0023
0024    Francisco Yumiceva (yumiceva@fnal.gov)
0025    Fermilab 2010
0026
0027 """
0028 from __future__ import print_function
0029
0030
0031 from builtins import range
0032 import os, string, re, sys, math
0033 import subprocess, time
0034
0035 #_______________OPTIONS________________
0036 import optparse
0037
0038 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
0039
0040 def nonzero(self): # will become the nonzero method of optparse.Values
0041     "True if options were given"
0042     for v in self.__dict__.values():
0043         if v is not None: return True
0044     return False
0045
0046 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
0047
0048 class ParsingError(Exception): pass
0049
0050 optionstring=""
0051
0052 def exit(msg=""):
0053     raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
0054
0055 def parse(docstring, arglist=None):
0056     global optionstring
0057     optionstring = docstring
0058     match = USAGE.search(optionstring)
0059     if not match: raise ParsingError("Cannot find the option string")
0060     optlines = match.group(1).splitlines()
0061     try:
0062         p = optparse.OptionParser(optlines[0])
0063         for line in optlines[1:]:
0064             opt, help=line.split(':')[:2]
0065             short,long=opt.split(',')[:2]
0066             if '=' in opt:
0067                 action='store'
0068                 long=long.split('=')[0]
0069             else:
0070                 action='store_true'
0071             p.add_option(short.strip(),long.strip(),
0072                          action = action, help = help.strip())
0073     except (IndexError,ValueError):
0074         raise ParsingError("Cannot parse the option string correctly")
0075     return p.parse_args(arglist)
0076
0077 #_______________________________
0078
0079 def cmp_tags(a,b):
0080
0081     yeara = int(a.split('_')[1])
0082     yearb = int(b.split('_')[1])
0083
0084     if yeara < yearb: return -1
0085     if yeara > yearb: return 1
0086
0087     suffix = "_offline"
0088     if a.find("_express") != -1:
0089         suffix = "_express"
0090     if a.find("_prompt") != -1:
0091         suffix = "_prompt"
0092
0093     tmpa = a.replace("BeamSpotObjects_2009_v","")
0094     tmpa = tmpa.replace(suffix,"")
0095
0096     tmpb = b.replace("BeamSpotObjects_2009_v","")
0097     tmpb = tmpb.replace(suffix,"")
0098
0099     na = int(tmpa)
0100     nb = int(tmpb)
0101     if na < nb: return -1
0102     if na == nb: return 0
0103     if na > nb: return 1
0104
0105 #___
0106
0107 def dump_header(lines):
0108
0109     lines.append('''
0110 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html>
0111 <head><title>Beam Spot Calibration Status</title></head>
0112
0113 <body>
0114
0115 <H1>Status of the Beam Spot Calibration</H1>
0116 <BR>
0117 <BR>
0118 This page is updated every 10 minutes:
0119 <BR>
0120 <strong><script src="datemod.js"
0121 type="text/javascript"></script></strong>
0122
0123 ''')
0124
0125 #____
0126
0127 def dump_footer(lines):
0128
0129     lines.append('</body>\n</html>\n')
0130
0131 #______________
0132 def write_tags(tags, lines):
0133
0134     end = '\n'
0135     br = '<BR>'+end
0136
0137     lines.append('<tr>'+end)
0138     for i in tags.keys():
0139
0140         lines.append('<th>'+i)
0141         lines.append('</th>'+end)
0142     lines.append('</tr>'+end)
0143
0144     for ntags in range(0,len(tags['offline'])):
0145         lines.append('<tr>'+end)
0146         for i in tags.keys():
0147             alist = tags[i]
0148             if ntags < len(tags[i]):
0149                 lines.append('<td> '+alist[ntags]+' </td>'+end)
0150             else:
0151                 lines.append('<td> </td>')
0152         lines.append('</tr>'+end)
0153
0154 #______________
0155 def write_iovs(iovs, lines):
0156
0157     end = '\n'
0158     br = '<BR>'+end
0159     # hearder
0160     lines.append('<tr>'+end)
0161     for i in iovs.keys():
0162         lines.append('<th>'+i)
0163         lines.append('</th>'+end)
0164     lines.append('</tr>'+end)
0165     # lumi type
0166     lines.append('<tr>'+end)
0167     for i in iovs.keys():
0168         aIOVlist = iovs[i]
0169         aIOV = IOV()
0170         if len(aIOVlist) > 0:
0171             aIOV = aIOVlist[0]
0172         lines.append('<td> '+aIOV.type+' </td>'+end)
0173     lines.append('</tr>'+end)
0174     # print iovs
0175     for niovs in range(0,len(iovs[iovs.keys()[0]])):
0176         lines.append('<tr>'+end)
0177         for i in iovs.keys():
0178             aIOVlist = iovs[i]
0179             aIOV = IOV()
0180             if len(aIOVlist) > niovs:
0181                 aIOV = aIOVlist[niovs]
0182             lines.append('<td> '+aIOV.IOVfirst +' - '+aIOV.IOVlast+' </td>'+end)
0183         lines.append('</tr>'+end)
0184
0185 #______________
0186 def get_listoftags(dest, auth,):
0187
0188     queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
0189     print(queryTags_cmd)
0190     outcmd = subprocess.getstatusoutput( queryTags_cmd )
0191     print(outcmd[1])
0192
0193     listtags = outcmd[1].split()
0194
0195     listtags_offline = []
0196     for itag in listtags:
0197         if itag[len(itag)-7:len(itag)] == "offline":
0198             listtags_offline.append(itag)
0199     listtags_express = []
0200     for itag in listtags:
0201         if itag[len(itag)-7:len(itag)] == "express":
0202             listtags_express.append(itag)
0203     listtags_prompt = []
0204     for itag in listtags:
0205         if itag[len(itag)-6:len(itag)] == "prompt":
0206             listtags_prompt.append(itag)
0207
0208     listtags_offline.sort( cmp = cmp_tags )
0209     listtags_offline.reverse()
0210     listtags_express.sort( cmp = cmp_tags )
0211     listtags_express.reverse()
0212     listtags_prompt.sort( cmp = cmp_tags )
0213     listtags_prompt.reverse()
0214
0215     result = {}
0216     result['offline'] = listtags_offline
0217     result['express'] = listtags_express
0218     result['prompt'] = listtags_prompt
0219
0220     return result
0221
0222 #______________________
0223 class IOV:
0224     def __init__(self):
0225         self.type = ''
0226         self.IOVfirst = ''
0227         self.IOVlast  = ''
0228
0229 #_________________
0230 def get_lastIOVs( listoftags, dest, auth ):
0231
0232     dbtags = ['offline','express','prompt']
0233
0234     results = {}
0235     for itag in dbtags:
0236
0237         lasttag = listoftags[itag][0]
0238         #fix for the moment to read old tags
0239         if itag != "offline":
0240             lasttag = listoftags[itag][1]
0241
0242         queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
0243         print(queryIOVs_cmd)
0244
0245         outcmd = subprocess.getstatusoutput( queryIOVs_cmd )
0246
0247         tmparr = outcmd[1].split('\n')
0248
0249         TimeType = tmparr[1].split()[1]
0250         listIOVs = []
0251
0252         # look at number of payloads
0253         lastline =  tmparr[len(tmparr)-1].split()
0254         npayloads = int( lastline[len(lastline)-1] )
0255
0256         maxIOVs = 3
0257         if npayloads < 3:
0258             maxIOVs = npayloads
0259         # pick the last three IOVs
0260         for i in range(0,maxIOVs):
0261             tmpline = tmparr[len(tmparr) -2 -i]
0262             aIOV = IOV()
0263             aIOV.IOVfirst = tmpline.split()[0]
0264             aIOV.IOVlast =  tmpline.split()[1]
0265             aIOV.type = TimeType
0266             listIOVs.append( aIOV )
0267
0268         results[lasttag] = listIOVs
0269
0270     return results
0271
0272 #
0273 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
0274 def pack(high,low):
0275     """pack high,low 32bit unsigned int to one unsigned 64bit long long
0276        Note:the print value of result number may appear signed, if the sign bit is used.
0277     """
0278     h=high<<32
0279     return (h|low)
0280
0281 def unpack(i):
0282     """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
0283     """
0284     high=i>>32
0285     low=i&0xFFFFFFFF
0286     return(high,low)
0287
0288 def unpackLumiid(i):
0289     """unpack 64bit lumiid to dictionary {'run','lumisection'}
0290     """
0291     j=unpack(i)
0292     return {'run':j[0],'lumisection':j[1]}
0293
0294 #____________________________
0295 def get_plots(path,output, iovs, tag):
0296
0297     initial=iovs[len(iovs)-1].IOVfirst
0298     final =iovs[0].IOVfirst
0299     if iovs[0].type == "lumiid":
0300         initial = str(unpack(initial)[0])+":"+str(unpack(initial)[1])
0301         final =  str(unpack(final)[0])+":"+str(unpack(final)[1])
0302
0303     initial = str(int(initial) -100 )
0304     cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
0305     print(cmd)
0306     outcmd = subprocess.getstatusoutput( cmd )
0307     print(outcmd[1])
0308
0309     cmd = "ls *.png"
0310     outcmd = subprocess.getstatusoutput( cmd )
0311
0312     pngfiles = outcmd[1].split('\n')
0313     print(pngfiles)
0314
0315     cmd = "cp *.png "+os.path.dirname(output)
0316     outcmd = subprocess.getstatusoutput( cmd )
0317     cmd = "rm *.png"
0318     outcmd = subprocess.getstatusoutput( cmd )
0319
0320     pngfiles.sort()
0321     return pngfiles
0322
0323 #_______________________________
0324 def write_plots(lines, plots,web):
0325
0326     end = '\n'
0327     br = '<BR>'+end
0328
0329     lines.append(br)
0330     lines.append('''
0331 <table border="1">
0332
0333 ''')
0334     for i in range(0,len(plots)):
0335         plot = plots[i]
0336         plot = os.path.basename(plot)
0337         if i%2 == 0:
0338             lines.append("<tr>"+end)
0339         lines.append("<td> <a href=\""+web+"/"+plot+"\"> <img src="+plot+" alt="+plot+" width='700' height='250' /> </a> </td>"+end)
0340         if i%2 == 1:
0341             lines.append("</tr>"+end)
0342
0343     lines.append('</table>'+end)
0344
0345 #________________________________
0346 def get_productionFiles( directory ):
0347
0348     list = subprocess.getstatusoutput('ls -t '+directory)
0349     list = list[1].split()
0350     newlist = []
0351     for i in list:
0352         if i.find('BeamFit_')!=-1:
0353             newlist.append(i)
0354
0355     return newlist
0356 #_______________________________
0357 def get_productionIOVs( directory ):
0358
0359     files = get_productionFiles( directory )
0360     listofruns = []
0361     for f in files:
0362         ii = f.find('Run')
0363         arun = f[ii:len(f)-4]
0364         listofruns.append(arun)
0365     return listofruns
0366
0367 #______________________________
0368 if __name__ == '__main__':
0369
0370
0371     # COMMAND LINE OPTIONS
0372     #################################
0373     option,args = parse(__doc__)
0374     if not args and not option: exit()
0375
0376     #htmlwebsite = "http://cmsrocstor.fnal.gov/lpc1/cmsroc/yumiceva/tmp/"
0377     htmlwebsite = "https://yumiceva.web.cern.ch/yumiceva/beamspot/"
0378     if option.web: htmlwebsite = option.web
0379
0380     ## Get the latest tags
0381     #dest = "frontier://cmsfrontier.cern.ch:8000/Frontier/CMS_COND_31X_BEAMSPOT"
0382     dest = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
0383     auth = "/afs/cern.ch/cms/DB/conddb"
0384     #cmscond_list_iov -c oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT -P /afs/cern.ch/cms/DB/conddb  -a
0385
0386     list_tags = get_listoftags( dest, auth)
0387
0388     # Get the latest IOVs from last tag
0389     list_lastIOVs = get_lastIOVs( list_tags, dest, auth)
0390
0391     # get latest processed runs
0392     processedruns = get_productionIOVs('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_BSPOT/yumiceva/tmp_lumi_workflow/')
0393
0394     # create web page
0395     lines = []
0396     end = '\n'
0397     br = '<BR>'+end
0398
0399     dump_header(lines)
0400
0401     lines.append('<h2>The three latest IOVs</h2>'+end)
0402     lines.append(br)
0403     lines.append('''
0404 <table border="1">
0405 ''')
0406     write_iovs( list_lastIOVs, lines )
0407     lines.append('</table>'+end)
0408
0409     lines.append('<h2>Latest data processed</h2>'+end)
0410     lines.append(br)
0411     #lines.append('to be written'+end)
0412     lines.append(processedruns[0]+', '+processedruns[1]+', '+processedruns[2])
0413     print(processedruns)
0414     lines.append(br)
0415
0416     lines.append('<h2>The Latest Tags</h2>'+end)
0417     lines.append(br)
0418     lines.append('''
0419 <table border="1">
0420 ''')
0421     write_tags( list_tags, lines)
0422     lines.append('</table>'+end)
0423
0424     lasttag = list_lastIOVs.keys()[0]
0425     lines.append('<h2>Plots of the latest IOVs from condDB tag: '+lasttag+' </h2>'+end)
0426     lines.append(br)
0427
0428     pngfiles = get_plots(option.path,option.output, list_lastIOVs[lasttag], lasttag)
0429     write_plots( lines, pngfiles, htmlwebsite )
0430
0431     dump_footer(lines)
0432
0433     outfile = open(option.output,'w')
0434     #print lines
0435     outfile.writelines( lines )
0436
0437
0438
0439
0440
0441
0442
0443