BeamSpotProducer/scripts/beamvalidation.py

0001 #!/usr/bin/env python3
0002 #____________________________________________________________
0003 #
0004 #
0005 # A very simple
0006 #
0007 # Francisco Yumiceva
0008 # yumiceva@fnal.gov
0009 #
0010 # Fermilab, 2010
0011 #
0012 #____________________________________________________________
0013
0014 """
0015    beam spot validation
0016
0017    A very simple script
0018
0019    usage: %prog -t <tag name>
0020    -o, --output    = OUTPUT: filename of output html file.
0021    -p, --path      = PATH: path to beam spot scripts.
0022    -w, --web       = WEB: html path to website.
0023
0024    Francisco Yumiceva (yumiceva@fnal.gov)
0025    Fermilab 2010
0026
0027 """
0028
0029
0030 from builtins import range
0031 import os, string, re, sys, math
0032 import subprocess, time
0033
0034 #_______________OPTIONS________________
0035 import optparse
0036
0037 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
0038
0039 def nonzero(self): # will become the nonzero method of optparse.Values
0040     "True if options were given"
0041     for v in self.__dict__.values():
0042         if v is not None: return True
0043     return False
0044
0045 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
0046
0047 class ParsingError(Exception): pass
0048
0049 optionstring=""
0050
0051 def exit(msg=""):
0052     raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
0053
0054 def parse(docstring, arglist=None):
0055     global optionstring
0056     optionstring = docstring
0057     match = USAGE.search(optionstring)
0058     if not match: raise ParsingError("Cannot find the option string")
0059     optlines = match.group(1).splitlines()
0060     try:
0061         p = optparse.OptionParser(optlines[0])
0062         for line in optlines[1:]:
0063             opt, help=line.split(':')[:2]
0064             short,long=opt.split(',')[:2]
0065             if '=' in opt:
0066                 action='store'
0067                 long=long.split('=')[0]
0068             else:
0069                 action='store_true'
0070             p.add_option(short.strip(),long.strip(),
0071                          action = action, help = help.strip())
0072     except (IndexError,ValueError):
0073         raise ParsingError("Cannot parse the option string correctly")
0074     return p.parse_args(arglist)
0075
0076 #_______________________________
0077
0078 def cmp_tags(a,b):
0079
0080     yeara = int(a.split('_')[1])
0081     yearb = int(b.split('_')[1])
0082
0083     if yeara < yearb: return -1
0084     if yeara > yearb: return 1
0085
0086     suffix = "_offline"
0087     if a.find("_express") != -1:
0088         suffix = "_express"
0089     if a.find("_prompt") != -1:
0090         suffix = "_prompt"
0091
0092     tmpa = a.replace("BeamSpotObjects_2009_v","")
0093     tmpa = tmpa.replace(suffix,"")
0094
0095     tmpb = b.replace("BeamSpotObjects_2009_v","")
0096     tmpb = tmpb.replace(suffix,"")
0097
0098     na = int(tmpa)
0099     nb = int(tmpb)
0100     if na < nb: return -1
0101     if na == nb: return 0
0102     if na > nb: return 1
0103
0104 #___
0105
0106 def dump_header(lines):
0107
0108     lines.append('''
0109 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html>
0110 <head><title>Beam Spot Calibration Status</title></head>
0111
0112 <body>
0113
0114 <H1>Status of the Beam Spot Calibration</H1>
0115 <BR>
0116 <BR>
0117 This page is updated every 10 minutes:
0118 <BR>
0119 <strong><script src="datemod.js"
0120 type="text/javascript"></script></strong>
0121
0122 ''')
0123
0124 #____
0125
0126 def dump_footer(lines):
0127
0128     lines.append('</body>\n</html>\n')
0129
0130 #______________
0131 def write_tags(tags, lines):
0132
0133     end = '\n'
0134     br = '<BR>'+end
0135
0136     lines.append('<tr>'+end)
0137     for i in tags.keys():
0138
0139         lines.append('<th>'+i)
0140         lines.append('</th>'+end)
0141     lines.append('</tr>'+end)
0142
0143     for ntags in range(0,len(tags['offline'])):
0144         lines.append('<tr>'+end)
0145         for i in tags.keys():
0146             alist = tags[i]
0147             if ntags < len(tags[i]):
0148                 lines.append('<td> '+alist[ntags]+' </td>'+end)
0149             else:
0150                 lines.append('<td> </td>')
0151         lines.append('</tr>'+end)
0152
0153 #______________
0154 def write_iovs(iovs, lines):
0155
0156     end = '\n'
0157     br = '<BR>'+end
0158     # hearder
0159     lines.append('<tr>'+end)
0160     for i in iovs.keys():
0161         lines.append('<th>'+i)
0162         lines.append('</th>'+end)
0163     lines.append('</tr>'+end)
0164     # lumi type
0165     lines.append('<tr>'+end)
0166     for i in iovs.keys():
0167         aIOVlist = iovs[i]
0168         aIOV = IOV()
0169         if len(aIOVlist) > 0:
0170             aIOV = aIOVlist[0]
0171         lines.append('<td> '+aIOV.type+' </td>'+end)
0172     lines.append('</tr>'+end)
0173     # print iovs
0174     for niovs in range(0,len(iovs[iovs.keys()[0]])):
0175         lines.append('<tr>'+end)
0176         for i in iovs.keys():
0177             aIOVlist = iovs[i]
0178             aIOV = IOV()
0179             if len(aIOVlist) > niovs:
0180                 aIOV = aIOVlist[niovs]
0181             lines.append('<td> '+aIOV.IOVfirst +' - '+aIOV.IOVlast+' </td>'+end)
0182         lines.append('</tr>'+end)
0183
0184 #______________
0185 def get_listoftags(dest, auth,):
0186
0187     queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
0188     print(queryTags_cmd)
0189     outcmd = subprocess.getstatusoutput( queryTags_cmd )
0190     print(outcmd[1])
0191
0192     listtags = outcmd[1].split()
0193
0194     listtags_offline = []
0195     for itag in listtags:
0196         if itag[len(itag)-7:len(itag)] == "offline":
0197             listtags_offline.append(itag)
0198     listtags_express = []
0199     for itag in listtags:
0200         if itag[len(itag)-7:len(itag)] == "express":
0201             listtags_express.append(itag)
0202     listtags_prompt = []
0203     for itag in listtags:
0204         if itag[len(itag)-6:len(itag)] == "prompt":
0205             listtags_prompt.append(itag)
0206
0207     listtags_offline.sort( cmp = cmp_tags )
0208     listtags_offline.reverse()
0209     listtags_express.sort( cmp = cmp_tags )
0210     listtags_express.reverse()
0211     listtags_prompt.sort( cmp = cmp_tags )
0212     listtags_prompt.reverse()
0213
0214     result = {}
0215     result['offline'] = listtags_offline
0216     result['express'] = listtags_express
0217     result['prompt'] = listtags_prompt
0218
0219     return result
0220
0221 #______________________
0222 class IOV:
0223     def __init__(self):
0224         self.type = ''
0225         self.IOVfirst = ''
0226         self.IOVlast  = ''
0227
0228 #_________________
0229 def get_lastIOVs( listoftags, dest, auth ):
0230
0231     dbtags = ['offline','express','prompt']
0232
0233     results = {}
0234     for itag in dbtags:
0235
0236         lasttag = listoftags[itag][0]
0237         #fix for the moment to read old tags
0238         if itag != "offline":
0239             lasttag = listoftags[itag][1]
0240
0241         queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
0242         print(queryIOVs_cmd)
0243
0244         outcmd = subprocess.getstatusoutput( queryIOVs_cmd )
0245
0246         tmparr = outcmd[1].split('\n')
0247
0248         TimeType = tmparr[1].split()[1]
0249         listIOVs = []
0250
0251         # look at number of payloads
0252         lastline =  tmparr[len(tmparr)-1].split()
0253         npayloads = int( lastline[len(lastline)-1] )
0254
0255         maxIOVs = 3
0256         if npayloads < 3:
0257             maxIOVs = npayloads
0258         # pick the last three IOVs
0259         for i in range(0,maxIOVs):
0260             tmpline = tmparr[len(tmparr) -2 -i]
0261             aIOV = IOV()
0262             aIOV.IOVfirst = tmpline.split()[0]
0263             aIOV.IOVlast =  tmpline.split()[1]
0264             aIOV.type = TimeType
0265             listIOVs.append( aIOV )
0266
0267         results[lasttag] = listIOVs
0268
0269     return results
0270
0271 #
0272 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
0273 def pack(high,low):
0274     """pack high,low 32bit unsigned int to one unsigned 64bit long long
0275        Note:the print value of result number may appear signed, if the sign bit is used.
0276     """
0277     h=high<<32
0278     return (h|low)
0279
0280 def unpack(i):
0281     """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
0282     """
0283     high=i>>32
0284     low=i&0xFFFFFFFF
0285     return(high,low)
0286
0287 def unpackLumiid(i):
0288     """unpack 64bit lumiid to dictionary {'run','lumisection'}
0289     """
0290     j=unpack(i)
0291     return {'run':j[0],'lumisection':j[1]}
0292
0293 #____________________________
0294 def get_plots(path,output, iovs, tag):
0295
0296     initial=iovs[len(iovs)-1].IOVfirst
0297     final =iovs[0].IOVfirst
0298     if iovs[0].type == "lumiid":
0299         initial = str(unpack(initial)[0])+":"+str(unpack(initial)[1])
0300         final =  str(unpack(final)[0])+":"+str(unpack(final)[1])
0301
0302     initial = str(int(initial) -100 )
0303     cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
0304     print(cmd)
0305     outcmd = subprocess.getstatusoutput( cmd )
0306     print(outcmd[1])
0307
0308     cmd = "ls *.png"
0309     outcmd = subprocess.getstatusoutput( cmd )
0310
0311     pngfiles = outcmd[1].split('\n')
0312     print(pngfiles)
0313
0314     cmd = "cp *.png "+os.path.dirname(output)
0315     outcmd = subprocess.getstatusoutput( cmd )
0316     cmd = "rm *.png"
0317     outcmd = subprocess.getstatusoutput( cmd )
0318
0319     pngfiles.sort()
0320     return pngfiles
0321
0322 #_______________________________
0323 def write_plots(lines, plots,web):
0324
0325     end = '\n'
0326     br = '<BR>'+end
0327
0328     lines.append(br)
0329     lines.append('''
0330 <table border="1">
0331
0332 ''')
0333     for i in range(0,len(plots)):
0334         plot = plots[i]
0335         plot = os.path.basename(plot)
0336         if i%2 == 0:
0337             lines.append("<tr>"+end)
0338         lines.append("<td> <a href=\""+web+"/"+plot+"\"> <img src="+plot+" alt="+plot+" width='700' height='250' /> </a> </td>"+end)
0339         if i%2 == 1:
0340             lines.append("</tr>"+end)
0341
0342     lines.append('</table>'+end)
0343
0344 #________________________________
0345 def get_productionFiles( directory ):
0346
0347     list = subprocess.getstatusoutput('ls -t '+directory)
0348     list = list[1].split()
0349     newlist = []
0350     for i in list:
0351         if i.find('BeamFit_')!=-1:
0352             newlist.append(i)
0353
0354     return newlist
0355 #_______________________________
0356 def get_productionIOVs( directory ):
0357
0358     files = get_productionFiles( directory )
0359     listofruns = []
0360     for f in files:
0361         ii = f.find('Run')
0362         arun = f[ii:len(f)-4]
0363         listofruns.append(arun)
0364     return listofruns
0365
0366 #______________________________
0367 if __name__ == '__main__':
0368
0369
0370     # COMMAND LINE OPTIONS
0371     #################################
0372     option,args = parse(__doc__)
0373     if not args and not option: exit()
0374
0375     #htmlwebsite = "http://cmsrocstor.fnal.gov/lpc1/cmsroc/yumiceva/tmp/"
0376     htmlwebsite = "https://yumiceva.web.cern.ch/yumiceva/beamspot/"
0377     if option.web: htmlwebsite = option.web
0378
0379     ## Get the latest tags
0380     #dest = "frontier://cmsfrontier.cern.ch:8000/Frontier/CMS_COND_31X_BEAMSPOT"
0381     dest = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
0382     auth = "/afs/cern.ch/cms/DB/conddb"
0383     #cmscond_list_iov -c oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT -P /afs/cern.ch/cms/DB/conddb  -a
0384
0385     list_tags = get_listoftags( dest, auth)
0386
0387     # Get the latest IOVs from last tag
0388     list_lastIOVs = get_lastIOVs( list_tags, dest, auth)
0389
0390     # get latest processed runs
0391     processedruns = get_productionIOVs('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_BSPOT/yumiceva/tmp_lumi_workflow/')
0392
0393     # create web page
0394     lines = []
0395     end = '\n'
0396     br = '<BR>'+end
0397
0398     dump_header(lines)
0399
0400     lines.append('<h2>The three latest IOVs</h2>'+end)
0401     lines.append(br)
0402     lines.append('''
0403 <table border="1">
0404 ''')
0405     write_iovs( list_lastIOVs, lines )
0406     lines.append('</table>'+end)
0407
0408     lines.append('<h2>Latest data processed</h2>'+end)
0409     lines.append(br)
0410     #lines.append('to be written'+end)
0411     lines.append(processedruns[0]+', '+processedruns[1]+', '+processedruns[2])
0412     print(processedruns)
0413     lines.append(br)
0414
0415     lines.append('<h2>The Latest Tags</h2>'+end)
0416     lines.append(br)
0417     lines.append('''
0418 <table border="1">
0419 ''')
0420     write_tags( list_tags, lines)
0421     lines.append('</table>'+end)
0422
0423     lasttag = list_lastIOVs.keys()[0]
0424     lines.append('<h2>Plots of the latest IOVs from condDB tag: '+lasttag+' </h2>'+end)
0425     lines.append(br)
0426
0427     pngfiles = get_plots(option.path,option.output, list_lastIOVs[lasttag], lasttag)
0428     write_plots( lines, pngfiles, htmlwebsite )
0429
0430     dump_footer(lines)
0431
0432     outfile = open(option.output,'w')
0433     #print lines
0434     outfile.writelines( lines )
0435
0436
0437
0438
0439
0440
0441
0442