RelMon/python/directories2html.py

0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 ################################################################################
0004 # RelMon: a tool for automatic Release Comparison
0005 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0006 #
0007
0008 #
0009 #
0010 # Danilo Piparo CERN - danilo.piparo@cern.ch
0011 #
0012 ################################################################################
0013
0014 from builtins import range
0015 from os import chdir,getcwd,listdir,makedirs
0016 from os.path import basename,join,exists
0017 import html
0018
0019 import sys
0020 theargv=sys.argv
0021 sys.argv=[]
0022 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
0023 sys.argv=theargv
0024
0025 import os
0026 import hashlib
0027 import random
0028 from matplotlib.colors import to_hex
0029
0030 if "RELMON_SA" in os.environ:
0031   from .dirstructure import Comparison,Directory
0032   from .definitions import *
0033   from .utils import unpickler
0034 else:
0035   from Utilities.RelMon.dirstructure import Comparison,Directory
0036   from Utilities.RelMon.definitions import *
0037   from Utilities.RelMon.utils import unpickler
0038
0039 #-------------------------------------------------------------------------------
0040
0041 def encode_obj_url(url):
0042   for old,new in url_encode_dict.items():
0043     url=url.replace(old,new)
0044   return url
0045
0046 def plot_size(h=250,w=200):
0047   return "w=%s;h=%s" %(h,w)
0048
0049 def build_obj_addr(run,sample,version,plot_path,tier):
0050   slash="/"
0051   obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
0052   obj_url+=plot_path
0053   while obj_url.endswith(slash):
0054     obj_url=obj_url[:-1]
0055   while slash*2 in obj_url:
0056     obj_url=obj_url.replace(slash*2,slash)
0057   return obj_url
0058
0059 def build_obj(run,sample,version,plot_path,tier):
0060   obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
0061   return encode_obj_url(obj_url)
0062
0063 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
0064   fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
0065   fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
0066   fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
0067   if len(draw_opts)>0:
0068     fairy_url+="drawopts=%s;" %draw_opts
0069   fairy_url+= plot_size(h,w)
0070   fairy_url += ";ref=ratiooverlay"
0071   return fairy_url
0072
0073 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
0074   fairy_url = "%s/%s/plotfairy/" %(server,base_url)
0075   fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
0076   fairy_url+= "?%s"%plot_size(h,w)
0077   if len(draw_opts)>0:
0078     fairy_url+="drawopts=%s;" %draw_opts
0079   return fairy_url
0080
0081 #-------------------------------------------------------------------------------
0082 style_location="/cms-service-reldqm"
0083 def get_page_header(directory=None, standalone=False, additional_header=""):
0084   style_location="/cms-service-reldqm"
0085   if standalone:
0086     style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2
0087   javascripts=''
0088   style=''
0089   tablestyle=''
0090   thead_h = 400
0091   wrapper_h = 1500
0092   if directory!=None and len(directory.comparisons)>0:
0093     meta=directory.meta
0094     style='img.fail {border:1px solid #ff0000;}\n'+\
0095           'img.succes {border:1px solid #00ff00;}\n'+\
0096           'img.null {border:1px solid #ffff00;}\n'+\
0097           'img.skiped {border:1px solid #7a7a7a;}\n'+\
0098           'a.black_link:link {color: #333333}\n'+\
0099           'a.black_link:hover {color: #737373}\n'+\
0100           'a.black_link:visited {color: #333333}\n'+\
0101           'a.black_link:active {color: #333333}\n'
0102     ## fixed first row and first column table
0103
0104     wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)
0105
0106   tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
0107   tablestyle += 'table {  position: relative; border-collapse: separate; border-spacing: 0;} \n'
0108   tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
0109   tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
0110   tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
0111   tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
0112   tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1  ; width: 200px; left:0}\n'
0113   tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
0114   tablestyle += '.vertical_head {top: 60%; -webkit-transform:  translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
0115   javascripts=""
0116
0117
0118
0119   html='<html>'+\
0120        '<head>'+\
0121        '<title>RelMon Summary</title>'+\
0122        '<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' + \
0123        '<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%style_location+\
0124        '<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%style_location+\
0125        '<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%style_location+\
0126        '<style type="text/css">'+\
0127        '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
0128        '%s'%style+\
0129        '%s'%tablestyle +\
0130        '</style>'+\
0131        '%s'%javascripts+\
0132        '%s'%additional_header+\
0133        '</head>'+\
0134        '<body>'+\
0135        '<div class="container">'
0136
0137   return html
0138
0139 #-------------------------------------------------------------------------------
0140
0141 def get_page_footer():
0142     return '</div></body></html>'
0143
0144 #-------------------------------------------------------------------------------
0145
0146 def get_title_section(directory, hashing_flag, standalone, depth=2):
0147
0148   mother_name=basename(directory.mother_dir)
0149   mother_file_name=""
0150   if depth==1:
0151     mother_file_name="../RelMonSummary.html"
0152     if mother_name!="":
0153       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0154   elif depth==2:
0155     mother_file_name="RelMonSummary.html"
0156     #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
0157     if mother_name!="":
0158       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0159   else:
0160       if hashing_flag:
0161           files = directory.mother_dir.split("/")
0162           if len(files) != 1:
0163               dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
0164           else:
0165               dir_name = files[-1]
0166           dir_name = directory.mother_dir
0167           mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
0168       else:
0169           mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
0170           mother_file_name=mother_file_name.strip("_")
0171
0172   link_to_mother='<a href="%s">..</a>' %mother_file_name
0173   html= '<div class="span-20">'+\
0174         '<h1>%s</h1>'%directory.name+\
0175         '</div>'+\
0176         '<div class="span-1">'+\
0177         '<h1>%s</h1>'%link_to_mother+\
0178         '</div>'+\
0179         '<div class="span-3 last">'+\
0180         '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
0181         '</div>'+\
0182         '<hr>'
0183   if len(mother_name)>0:
0184     html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
0185           '<hr>'
0186   return html
0187
0188 #-------------------------------------------------------------------------------
0189 def get_dir_stats(directory):
0190   html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
0191   html+='<ul>'
0192   if directory.n_successes>0:
0193     html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
0194   if directory.n_nulls>0:
0195     html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
0196   if directory.n_fails>0:
0197     html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
0198   if directory.n_skiped>0:
0199     html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
0200   if directory.n_missing_objs>0:
0201     html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
0202   html+='</ul>'
0203   return html
0204
0205 #-------------------------------------------------------------------------------
0206
0207 def get_subdirs_section(directory, hashing_flag):
0208   if len(directory.subdirs)==0:
0209     return ""
0210   html= '<div class="span-20 colborder">'
0211   html+='<h2 class="alt">Sub-Directories</h2>'
0212   # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
0213   # so reverse :)
0214   sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
0215   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
0216   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
0217   for subdir in sorted_subdirs:
0218     name=subdir.name
0219     if hashing_flag:
0220         link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
0221     else:
0222         link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
0223         link=link.strip("_")
0224     html+='<div class="span-4 prepend-2 colborder">'
0225     html+='<h3>%s</h3>'%name
0226     html+='</div>'
0227
0228     html+='<div class="span-7">'
0229     html+=get_dir_stats(subdir)
0230     html+='</div>'
0231
0232     html+='<div class="span-6 last">'
0233     html+= subdir.get_piechart_js(120,link)
0234     html+='</div>'
0235
0236     html+='<hr>'
0237   return html+'</div>'
0238
0239
0240 #-------------------------------------------------------------------------------
0241
0242 def get_summary_section(directory,matrix_page=True):
0243
0244   # Hack find the first comparison and fill test and threshold
0245   # shall this be put in meta?
0246   test_name=""
0247   test_threshold=0
0248   for comparison in directory.comparisons:
0249       test_name=comparison.test_name
0250       test_threshold=comparison.test_thr
0251       break
0252   if len(test_name)==0:
0253     for subdir in directory.subdirs:
0254       for comparison in subdir.comparisons:
0255         test_name=comparison.test_name
0256         test_threshold=comparison.test_thr
0257         break
0258       if len(test_name)!=0:break
0259       if len(test_name)==0:
0260         for subsubdir in subdir.subdirs:
0261           for comparison in subsubdir.comparisons:
0262             test_name=comparison.test_name
0263             test_threshold=comparison.test_thr
0264             break
0265           if len(test_name)!=0:break
0266         if len(test_name)==0:
0267           for subsubsubdir in subsubdir.subdirs:
0268             for comparison in subsubsubdir.comparisons:
0269               test_name=comparison.test_name
0270               test_threshold=comparison.test_thr
0271               break
0272             if len(test_name)!=0:break
0273
0274
0275   meta=directory.meta
0276
0277   html= '<div class="span-6">'+\
0278         '<h3>Summary</h3>'
0279   html+=get_dir_stats(directory)
0280   html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
0281   html+='</div>'
0282
0283   html+='<div class="span-7 colborder">'+\
0284         '%s'%(directory.get_piechart_js(150)) +\
0285         '</div>'+\
0286         '<div class="span-9 last">'
0287   if matrix_page:
0288     html+='<h3>Sample:</h3>'+\
0289           '<p class="caps">%s</p>'%meta.sample1+\
0290           '<h3>Run1 and Run2:</h3>'+\
0291           '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
0292   html+='<h3>Releases:</h3>'+\
0293         '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
0294         '<h3>Statistical Test (Pvalue threshold):</h3>'+\
0295         '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
0296         '</div>'+\
0297         '<hr>'
0298   return html
0299
0300 #-------------------------------------------------------------------------------
0301
0302 def get_comparisons(category,directory):
0303   """Prepare the comparisons between histograms and organise them in the page.
0304   Moreover create separate pages with the overlay and the single plots.
0305   """
0306   counter=1
0307   tot_counter=1
0308
0309   # get the right ones
0310   comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
0311   n_comparisons=len(comparisons)
0312
0313   is_reverse=True
0314   if category == FAIL:
0315     is_reverse=False
0316   comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
0317
0318
0319   dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
0320   html_comparisons=""
0321   for comparison in comparisons:
0322     plot_name=comparison.img_name
0323     if "http://" not in plot_name:
0324       plot_name= basename(comparison.img_name)
0325     class_type="colborder"
0326     if counter==3 or tot_counter==n_comparisons:
0327       class_type=" colborder last"
0328     comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
0329
0330
0331     if directory.do_pngs:
0332       png_link=comparison.img_name
0333       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0334                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0335                       '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
0336     else:
0337       big_fairy=fairy_url(directory.meta.run1,
0338                         directory.meta.run2,
0339                         directory.meta.sample1,
0340                         directory.meta.sample2,
0341                         directory.meta.release1,
0342                         directory.meta.release2,
0343                         comp_abs_path,
0344                         directory.meta.tier1,
0345                         directory.meta.tier2,
0346                         "",600,600)
0347       small_fairy=fairy_url(directory.meta.run1,
0348                         directory.meta.run2,
0349                         directory.meta.sample1,
0350                         directory.meta.sample2,
0351                         directory.meta.release1,
0352                         directory.meta.release2,
0353                         comp_abs_path,
0354                         directory.meta.tier1,
0355                         directory.meta.tier2)
0356
0357       single_fairy1=fairy_url_single(directory.meta.run1,
0358                                    directory.meta.sample1,
0359                                    directory.meta.release1,
0360                                    comp_abs_path,
0361                                    directory.meta.tier1,
0362                                    "",500,500)
0363       single_fairy2=fairy_url_single(directory.meta.run2,
0364                                    directory.meta.sample2,
0365                                    directory.meta.release2,
0366                                    comp_abs_path,
0367                                    directory.meta.tier2,
0368                                    "",500,500)
0369
0370       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0371                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0372                       '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
0373                       '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
0374                       '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
0375
0376     if counter==3:
0377       html_comparisons+="<hr>"
0378       counter=0
0379     counter+=1
0380     tot_counter+=1
0381
0382   if len(html_comparisons)!=0:
0383     html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
0384     html+=html_comparisons
0385     html+='<hr>'
0386     return html
0387   return ""
0388
0389 #-------------------------------------------------------------------------------
0390
0391 def get_rank_section(directory):
0392 # do Rank histo png
0393     imgname="RankSummary.png"
0394     gStyle.SetPadTickY(0)
0395     c=TCanvas("ranks","ranks",500,400)
0396     #gStyle.SetOptStat(0)
0397     c.cd()
0398
0399     h=directory.rank_histo
0400     rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
0401     rank_histof.SetLineWidth(2)
0402     for i in range(0,h.GetNbinsX()+1):
0403       rank_histof.SetBinContent(i,h.GetBinContent(i))
0404     h.SetTitle("Ranks Summary;Rank;Frequency")
0405     h.Draw("Hist")
0406     c.Update()
0407     rank_histof.ComputeIntegral()
0408     integral = rank_histof.GetIntegral()
0409     rank_histof.SetContent(integral)
0410
0411     rightmax = 1.1*rank_histof.GetMaximum()
0412     scale = gPad.GetUymax()/rightmax
0413     rank_histof.SetLineColor(kRed)
0414     rank_histof.Scale(scale)
0415     rank_histof.Draw("same")
0416
0417     #draw an axis on the right side
0418     axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
0419     axis.SetTitle("Cumulative")
0420     axis.SetTitleColor(kRed)
0421     axis.SetLineColor(kRed)
0422     axis.SetLabelColor(kRed)
0423     axis.Draw()
0424
0425     rank_histof.Draw("Same");
0426
0427
0428     c.Print(imgname)
0429
0430     page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
0431     page_html+='<div class="span-19"><img src="%s"></div>' %imgname
0432     page_html+='</div> <a href="#top">Top...</a><hr>'
0433
0434     return page_html
0435
0436 #-------------------------------------------------------------------------------
0437
0438 #-------------------------------------------------------------------------------
0439 def get_missing_objs_section(directory):
0440     """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
0441     """
0442     page_html = "Unpaired in %s</br>"%(directory.filename1)
0443     for elem in directory.different_histograms['file1']:
0444         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
0445     page_html +="</br>"
0446     page_html += "Unpaired in %s</br>"%(directory.filename2)
0447     for elem in directory.different_histograms['file2']:
0448         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
0449     return page_html
0450 #-------------------------------------------------------------------------------
0451
0452 def directory2html(directory, hashing, standalone, depth=0):
0453   """Converts a directory tree into html pages, very nice ones.
0454   """
0455   #print "d2html: depth", str(depth)," dir ",directory.name
0456   depth+=1
0457   #old_cwd=getcwd()
0458   #if not exists(directory.name) and len(directory.name)>0:
0459     #makedirs(directory.name)
0460
0461   #if len(directory.name)>0:
0462     #chdir(directory.name)
0463
0464   for subdir in directory.subdirs:
0465     directory2html(subdir,hashing,standalone, depth)
0466
0467   page_html=get_page_header(directory, standalone)+\
0468             get_title_section(directory,hashing, standalone, depth)+\
0469             get_summary_section(directory)+\
0470             get_subdirs_section(directory, hashing)
0471
0472   for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
0473                      (directory.n_comp_nulls >0,NULL ),
0474
0475                      (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
0476                      (directory.n_comp_skiped >0,SKIPED)):
0477     if do_cat:
0478       page_html+=get_comparisons(cat,directory)
0479
0480   if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
0481     page_html += get_missing_objs_section(directory)
0482
0483   # Distribution of ranks
0484
0485   if depth==1:
0486     page_html+=get_rank_section(directory)
0487
0488
0489   page_html+=get_page_footer()
0490
0491   page_name=directory.name
0492
0493   if len(page_name)==0:
0494     page_name="RelMonSummary"
0495   if hashing:
0496       if page_name != "RelMonSummary":
0497           #print " ## oFile path: %s"%(join(directory.full_path))
0498           #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
0499           ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
0500       else:
0501           ofilename = "RelMonSummary.html"
0502   else:
0503       ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
0504       ofilename=ofilename.strip("_")
0505   #print "Writing on %s" %ofilename
0506   ofile=open(ofilename,"w")
0507   ofile.write(page_html)
0508   ofile.close()
0509
0510   #chdir(old_cwd)
0511
0512 def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):
0513
0514   color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
0515   font_size = int(w/9)
0516   gauge_max = 1. - rate
0517
0518   name = random.getrandbits(64) # just a random has for the canvas
0519   html = ""
0520   html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)
0521
0522   # "gauge" chart
0523   html += '<script> new Chart("%s",'%(name)
0524   html += '{ type: "doughnut",'
0525
0526   # data
0527   html += 'data: {'
0528   html += 'labels: ["Success", "Failure"],'
0529   html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
0530   html += 'data: [%f,%f]}] },'%(rate,gauge_max)
0531
0532   #options
0533   html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' ## in radiants
0534   html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
0535   html += '}); </script>'
0536   if add_rate:
0537     html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
0538     html += '<span style="color: %s; font-family: courier; font-size:  %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)
0539
0540   return html
0541 #-------------------------------------------------------------------------------
0542
0543 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
0544   # Let's make a summary for an overview in categories act on the global dir
0545   aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
0546
0547   list_of_names=[]
0548   if the_aggr_pairs==[]:
0549     for samplename,sampledir in dir_dict.items():
0550       for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
0551         if not subsysdirname in list_of_names:
0552           list_of_names.append(subsysdirname)
0553           the_aggr_pairs.append((subsysdirname,[subsysdirname]))
0554
0555   #print the_aggr_pairs
0556   for cat_name, subdir_list in the_aggr_pairs:
0557     total_successes=0.
0558     total_directory_successes=0
0559     total_weight=0.
0560     present_subdirs={}
0561     total_ndirs=0
0562     # Loop on samples
0563     for dirname, sampledir in dir_dict.items():
0564       # Loop on directories
0565       for subdirname,subdir in sampledir.get_subdirs_dict().items():
0566         if subdirname in subdir_list:
0567           nsucc=subdir.n_successes
0568           total_successes+=nsucc
0569           weight=subdir.weight
0570           total_weight+=weight
0571           total_ndirs+=1
0572
0573           total_directory_successes+= float(nsucc)/weight
0574           if subdirname in present_subdirs:
0575             this_dir_dict=present_subdirs[subdirname]
0576             this_dir_dict["nsucc"]+=nsucc
0577             this_dir_dict["weight"]+=weight
0578           else:
0579             present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
0580         # Make it usable also for subdirectories
0581         for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
0582           for pathname in [name for name in subdir_list if "/" in name]:
0583             selected_subdirname,selected_subsubdirname = pathname.split("/")
0584             if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
0585               #print "Studying directory ",subsubdirname," in directory ",subdirname
0586               nsucc=subsubdir.n_successes
0587               total_successes+=nsucc
0588               weight=subsubdir.weight
0589               total_weight+=weight
0590               total_ndirs+=1
0591               total_directory_successes+= float(nsucc)/weight
0592
0593               if subsubdirname in present_subdirs:
0594                 this_dir_dict=present_subdirs[subsubdirname]
0595                 this_dir_dict["nsucc"]+=nsucc
0596                 this_dir_dict["weight"]+=weight
0597               else:
0598                 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
0599
0600     if total_ndirs == 0:
0601       print("No directory of the category %s is present in the samples: skipping." %cat_name)
0602       continue
0603
0604     average_success_rate=total_directory_successes/(total_ndirs)
0605     aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
0606
0607   return aggr_pairs_info
0608
0609 #-------------------------------------------------------------------------------
0610
0611 def make_categories_summary(dir_dict,aggregation_rules):
0612
0613   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0614
0615   #print aggr_pairs_info
0616
0617   # Now Let's build the HTML
0618
0619   html= '<div class="span-20 colborder">'
0620   html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
0621
0622   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0623     #print cat_name,present_subdirs,total_weight,average_success_rate
0624     html+='<div class="span-3 prepend-0 colborder">'
0625     html+='<h3>%s</h3>'%cat_name
0626     html+='<div><span class="alt">Avg. Success rate:</span></div>'
0627     html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
0628     html+='</div>'
0629     html+='<div class="span-9">'
0630
0631     html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
0632     html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
0633     html+='<ul>'
0634     for subdirname in sorted(present_subdirs.keys()):
0635       this_dir_dict=present_subdirs[subdirname]
0636       nsucc=this_dir_dict["nsucc"]
0637       weight=this_dir_dict["weight"]
0638       html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
0639     html+='</ul>'
0640     html+='</div>'
0641
0642     html+='<div class="span-6 last">'
0643     html+=build_gauge_js(average_success_rate,add_rate=True)
0644     html+='</div>'
0645
0646     html+='<hr>'
0647   return html+'<br><a href="#top">Top...</a> </div><hr>'
0648
0649  #-------------------------------------------------------------------------------
0650
0651 def make_twiki_table(dir_dict,aggregation_rules):
0652
0653   # decide the release
0654   meta= list(dir_dict.items())[0][1].meta
0655   releases=sorted([meta.release1,meta.release2])
0656   latest_release=releases[1].split("-")[0]
0657
0658
0659   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0660
0661   # Now Let's build the HTML
0662
0663   html= '<div class="span-20 colborder">'
0664   html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
0665   html+='<div>| Release | Comparison |'
0666   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0667     html+=cat_name
0668     html+=" | "
0669   html+='</div>'
0670
0671   html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release
0672
0673   # Now add all the line with small gauges
0674
0675   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0676     #print cat_name,present_subdirs,total_weight,average_success_rate
0677     html+=build_gauge_js(average_success_rate,w=40)
0678     html+=" | "
0679
0680   html+='</div> <a href="#top">Top...</a>'
0681   html+='<hr>'
0682   return html+'</div>'
0683
0684 #-------------------------------------------------------------------------------
0685
0686 def get_pie_tooltip(directory):
0687   tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
0688   return tooltip
0689
0690 #-------------------------------------------------------------------------------
0691
0692 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
0693
0694   aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
0695
0696   script="""
0697     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
0698     <script type="text/javascript">
0699       google.load("visualization", "1", {packages:["corechart"]});
0700       google.setOnLoadCallback(drawChart);
0701       function drawChart() {
0702         var data = new google.visualization.DataTable();
0703         data.addColumn('string', 'DQM Directory');
0704         data.addColumn('number', 'Success Rate');
0705         """
0706   script+="data.addRows(%i);\n"%len(aggr_pairs_info)
0707   counter=0
0708   for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
0709     #print subsystname,present_directories
0710     script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
0711     script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
0712     counter+=1
0713   script+="""
0714         var chart = new google.visualization.BarChart(document.getElementById('%s'));
0715         chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
0716                           vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
0717                          });
0718       }
0719     </script>
0720     """%(name,35*counter,title)
0721   return script
0722
0723
0724 #-------------------------------------------------------------------------------
0725
0726 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
0727   """Create a table, with as rows the directories and as columns the samples.
0728   Each box in the table will contain a pie chart linking to the directory.
0729   """
0730   #aggregation_rules={}
0731   #aggregation_rules_twiki={}
0732
0733   chdir(indir)
0734   if os.path.isabs(indir):
0735       title = basename(indir)
0736   else:
0737       title=indir
0738   title=title.strip(".")
0739   title=title.strip("/")
0740
0741
0742   # Get the list of pickles
0743   sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
0744
0745   # Load directories, build a list of all first level subdirs
0746   dir_unpicklers=[]
0747   n_unpicklers=0
0748   for sample_pkl in sample_pkls:
0749     dir_unpickler=unpickler(sample_pkl)
0750     dir_unpickler.start()
0751     n_unpicklers+=1
0752     dir_unpicklers.append(dir_unpickler)
0753     if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
0754       n_unpicklers=0
0755       for dir_unpickler in dir_unpicklers:
0756         dir_unpickler.join()
0757
0758   dir_dict={}
0759
0760   # create a fake global directory
0761   global_dir=Directory("global","")
0762   for dir_unpickler in dir_unpicklers:
0763     dir_unpickler.join()
0764     directory=dir_unpickler.directory
0765     #directory.prune("Run summary")
0766     #directory.calcStats()
0767     global_dir.meta=directory.meta
0768     dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
0769     global_dir.subdirs.append(directory)
0770
0771   global_dir.calcStats()
0772
0773   directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
0774   categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
0775
0776   page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
0777   rel1=""
0778   rel2=""
0779   try:
0780     rel1,rel2=title.split("VS")
0781   except:
0782     rel1=global_dir.meta.release1.split("-")[0]
0783     rel2=global_dir.meta.release2.split("-")[0]
0784     global_dir.meta.release1=rel1
0785     global_dir.meta.release2=rel2
0786
0787   # union of all subdirs names
0788   all_subdirs=[]
0789   for directory in dir_dict.values():
0790     for subdir_name in directory.get_subdirs_names():
0791       all_subdirs.append(subdir_name)
0792   all_subdirs=sorted(list(set(all_subdirs)))
0793
0794   #print " $$ all_subdirs: %s" %(all_subdirs)
0795
0796   # Get The title
0797   page_html+= '<div class="span-20">'+\
0798               '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
0799               '</div>'+\
0800               '<div class="span-1">'+\
0801               '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
0802               '</div>'+\
0803               '<hr>'
0804   page_html+='<div class="span-24"><p></p></div>\n'*3
0805
0806   # Get The summary section
0807   page_html+= get_summary_section(global_dir,False)
0808
0809   # Make the anchor sections
0810   page_html+= '<div class="span-24">'
0811   page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
0812               '<ul>'+\
0813               '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
0814               '<li><a href="#categories">Categories</a></li>'+\
0815               '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
0816           '<li><a href="#summary_table">Summary Table</a></li>'+\
0817               '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
0818               '<li><a href="#twiki_table">Twiki Table</a></li>'+\
0819               '</ul>'+\
0820               '</div><hr>'
0821
0822
0823   # Make the CategoriesBar chart
0824   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
0825   page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
0826
0827   # Make the gauges per categories
0828   page_html+=make_categories_summary(dir_dict,aggregation_rules)
0829
0830   # Make the Directories chart
0831   page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
0832
0833   # Barbarian vertical space. Suggestions are welcome
0834   for i in range(2):
0835     page_html+='<div class="span-24"><p></p></div>\n'
0836
0837
0838   # Prepare the table
0839   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'
0840
0841   div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
0842   page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
0843   page_html+="""
0844         <table>
0845         <thead>
0846           <tr>
0847           <th> <p class = "vertical_head center_head"></p> </th>
0848   """
0849
0850   # First row with samples
0851   page_html+="""
0852           <th> <p class = "vertical_head center_head">Summary</p></th>"""
0853
0854   sorted_samples=sorted(dir_dict.keys())
0855   for sample in sorted_samples:
0856     if "_" in sample and "Data" not in sample:
0857       sample_nick="_".join(sample.split("X_")[0].split("_")[:-1])
0858       # Cleaning for MC: just the fragment
0859     elif "Data" in sample and "RelVal" in sample:
0860       sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
0861       # Cleaning for Data: PD + Era + Run
0862     else:
0863       sample_nick = sample
0864
0865     page_html+="""
0866           <th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
0867   page_html+="</tr> \n </thead> \n </tbody> \n"
0868
0869
0870  # FIRST ROW
0871  # Now the summaries  at the beginning of the table
0872   page_html+="<tr>"
0873   page_html+='<td class="sticky-col">'
0874
0875   page_html+='<b>Summary</b></td>'
0876   page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
0877   for sample in sorted_samples:
0878     col=dir_dict[sample]
0879     # check if the directory was a top one or not
0880     summary_page_name="RelMonSummary.html"
0881     if col.name!="":
0882       summary_page_name=hash_name(col.name, hashing_flag)+".html"
0883     title = get_pie_tooltip(col)
0884     chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
0885     page_html+='<td>'
0886     page_html+='%s </a></td>' %(chart)
0887   page_html+="</tr>"
0888
0889   # Now the content
0890   for subdir_name in all_subdirs:
0891
0892     page_html+='          <tr>\n'
0893     page_html+='          <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name
0894
0895     row_summary=Directory("row_summary","")
0896     sample_counter=0
0897     n_samples=len(sorted_samples)
0898
0899     for sample in sorted_samples:
0900       subdirs_dict=directory.get_subdirs_dict()
0901       directory=dir_dict[sample]
0902       dir_is_there=subdir_name in subdirs_dict
0903       if dir_is_there:
0904         row_summary.subdirs.append(subdirs_dict[subdir_name])
0905
0906     # one first row for the summary!
0907     row_summary.calcStats()
0908     title = get_pie_tooltip(col)
0909     chart = row_summary.get_piechart_js(50)
0910     page_html+='<td><div>'#<div class="span-1">'
0911     page_html+= chart + '</div></td>'
0912
0913     for sample in sorted_samples:
0914       sample_counter+=1
0915
0916       directory=dir_dict[sample]
0917       subdirs_dict=directory.get_subdirs_dict()
0918
0919       # Check if the directory is the top one
0920       if directory.name!="":
0921         # We did not run on the topdir
0922         #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
0923         #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
0924         if hashing_flag:
0925           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
0926         else:
0927           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
0928       else:
0929         #print "   ## summary_page: %s"%(directory.name+subdir_name)
0930         #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
0931         summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
0932       dir_is_there=subdir_name in subdirs_dict
0933
0934       img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
0935       page_html+='<td><div class="span-1">'
0936
0937       if dir_is_there:
0938         #row_summary.subdirs.append(subdirs_dict[subdir_name])
0939         chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
0940         page_html+='%s'%chart
0941       else:
0942         page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
0943       page_html+='</div></td>'
0944
0945     page_html+="          </tr>\n"
0946
0947
0948
0949   page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'
0950
0951   page_html+=get_rank_section(global_dir)
0952
0953   #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
0954   # ^ commenting out for the moment, not really useful nor used
0955
0956   page_html+=get_page_footer()
0957   return page_html
0958
0959
0960 #-----------UPDATES------
0961 def hash_name(file_name, flag):
0962     if flag: #if hashing flag is ON then return
0963         if (3,0,0) <= sys.version_info:
0964             return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
0965         return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
0966     else:
0967         return file_name #return standart name