RelMon/python/directories2html.py

0001 ################################################################################
0002 # RelMon: a tool for automatic Release Comparison
0003 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0004 #
0005
0006 #
0007 #
0008 # Danilo Piparo CERN - danilo.piparo@cern.ch
0009 #
0010 ################################################################################
0011
0012 from builtins import range
0013 from os import chdir,getcwd,listdir,makedirs
0014 from os.path import basename,join,exists
0015 import html
0016
0017 import sys
0018 theargv=sys.argv
0019 sys.argv=[]
0020 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
0021 sys.argv=theargv
0022
0023 import os
0024 import hashlib
0025 import random
0026 from matplotlib.colors import to_hex
0027
0028 if "RELMON_SA" in os.environ:
0029   from .dirstructure import Comparison,Directory
0030   from .definitions import *
0031   from .utils import unpickler
0032 else:
0033   from Utilities.RelMon.dirstructure import Comparison,Directory
0034   from Utilities.RelMon.definitions import *
0035   from Utilities.RelMon.utils import unpickler
0036
0037 #-------------------------------------------------------------------------------
0038
0039 def encode_obj_url(url):
0040   for old,new in url_encode_dict.items():
0041     url=url.replace(old,new)
0042   return url
0043
0044 def plot_size(h=250,w=200):
0045   return "w=%s;h=%s" %(h,w)
0046
0047 def build_obj_addr(run,sample,version,plot_path,tier):
0048   slash="/"
0049   obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
0050   obj_url+=plot_path
0051   while obj_url.endswith(slash):
0052     obj_url=obj_url[:-1]
0053   while slash*2 in obj_url:
0054     obj_url=obj_url.replace(slash*2,slash)
0055   return obj_url
0056
0057 def build_obj(run,sample,version,plot_path,tier):
0058   obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
0059   return encode_obj_url(obj_url)
0060
0061 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
0062   fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
0063   fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
0064   fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
0065   if len(draw_opts)>0:
0066     fairy_url+="drawopts=%s;" %draw_opts
0067   fairy_url+= plot_size(h,w)
0068   fairy_url += ";ref=ratiooverlay"
0069   return fairy_url
0070
0071 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
0072   fairy_url = "%s/%s/plotfairy/" %(server,base_url)
0073   fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
0074   fairy_url+= "?%s"%plot_size(h,w)
0075   if len(draw_opts)>0:
0076     fairy_url+="drawopts=%s;" %draw_opts
0077   return fairy_url
0078
0079 #-------------------------------------------------------------------------------
0080 def get_page_header(directory=None, standalone=False, additional_header=""):
0081
0082   javascripts=''
0083   style=''
0084   tablestyle=''
0085   thead_h = 400
0086   wrapper_h = 1500
0087   style_location="https://cms-pdmv-prod.web.cern.ch/relmon//cms-service-reldqm/"
0088
0089   if directory!=None and len(directory.comparisons)>0:
0090     meta=directory.meta
0091     style='img.fail {border:1px solid #ff0000;}\n'+\
0092           'img.succes {border:1px solid #00ff00;}\n'+\
0093           'img.null {border:1px solid #ffff00;}\n'+\
0094           'img.skiped {border:1px solid #7a7a7a;}\n'+\
0095           'a.black_link:link {color: #333333}\n'+\
0096           'a.black_link:hover {color: #737373}\n'+\
0097           'a.black_link:visited {color: #333333}\n'+\
0098           'a.black_link:active {color: #333333}\n'
0099     ## fixed first row and first column table
0100
0101     wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)
0102
0103   tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
0104   tablestyle += 'table {  position: relative; border-collapse: separate; border-spacing: 0;} \n'
0105   tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
0106   tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
0107   tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
0108   tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
0109   tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1  ; width: 200px; left:0}\n'
0110   tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
0111   tablestyle += '.vertical_head {top: 60%; -webkit-transform:  translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
0112   javascripts=""
0113
0114
0115   html='<html>'+\
0116        '<head>'+\
0117        '<title>RelMon Summary</title>'+\
0118        '<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' +\
0119        '<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%(style_location) +\
0120        '<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%(style_location) +\
0121        '<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%(style_location) +\
0122        '<style type="text/css">'+\
0123        '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
0124        '%s'%style+\
0125        '%s'%tablestyle +\
0126        '</style>'+\
0127        '%s'%javascripts+\
0128        '%s'%additional_header+\
0129        '</head>'+\
0130        '<body>'+\
0131        '<div class="container">'
0132
0133   return html
0134
0135 #-------------------------------------------------------------------------------
0136
0137 def get_page_footer():
0138     return '</div></body></html>'
0139
0140 #-------------------------------------------------------------------------------
0141
0142 def get_title_section(directory, hashing_flag, standalone, depth=2):
0143
0144   mother_name=basename(directory.mother_dir)
0145   mother_file_name=""
0146   if depth==1:
0147     mother_file_name="../RelMonSummary.html"
0148     if mother_name!="":
0149       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0150   elif depth==2:
0151     mother_file_name="RelMonSummary.html"
0152     #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
0153     if mother_name!="":
0154       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0155   else:
0156       if hashing_flag:
0157           files = directory.mother_dir.split("/")
0158           if len(files) != 1:
0159               dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
0160           else:
0161               dir_name = files[-1]
0162           dir_name = directory.mother_dir
0163           mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
0164       else:
0165           mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
0166           mother_file_name=mother_file_name.strip("_")
0167
0168   link_to_mother='<a href="%s">..</a>' %mother_file_name
0169   html= '<div class="span-20">'+\
0170         '<h1>%s</h1>'%directory.name+\
0171         '</div>'+\
0172         '<div class="span-1">'+\
0173         '<h1>%s</h1>'%link_to_mother+\
0174         '</div>'+\
0175         '<div class="span-3 last">'+\
0176         '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
0177         '</div>'+\
0178         '<hr>'
0179   if len(mother_name)>0:
0180     html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
0181           '<hr>'
0182   return html
0183
0184 #-------------------------------------------------------------------------------
0185 def get_dir_stats(directory):
0186   html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
0187   html+='<ul>'
0188   if directory.n_successes>0:
0189     html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
0190   if directory.n_nulls>0:
0191     html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
0192   if directory.n_fails>0:
0193     html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
0194   if directory.n_skiped>0:
0195     html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
0196   if directory.n_missing_objs>0:
0197     html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
0198   html+='</ul>'
0199   return html
0200
0201 #-------------------------------------------------------------------------------
0202
0203 def get_subdirs_section(directory, hashing_flag):
0204   if len(directory.subdirs)==0:
0205     return ""
0206   html= '<div class="span-20 colborder">'
0207   html+='<h2 class="alt">Sub-Directories</h2>'
0208   # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
0209   # so reverse :)
0210   sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
0211   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
0212   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
0213   for subdir in sorted_subdirs:
0214     name=subdir.name
0215     if hashing_flag:
0216         link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
0217     else:
0218         link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
0219         link=link.strip("_")
0220     html+='<div class="span-4 prepend-2 colborder">'
0221     html+='<h3>%s</h3>'%name
0222     html+='</div>'
0223
0224     html+='<div class="span-7">'
0225     html+=get_dir_stats(subdir)
0226     html+='</div>'
0227
0228     html+='<div class="span-6 last">'
0229     html+= subdir.get_piechart_js(120,link)
0230     html+='</div>'
0231
0232     html+='<hr>'
0233   return html+'</div>'
0234
0235
0236 #-------------------------------------------------------------------------------
0237
0238 def get_summary_section(directory,matrix_page=True):
0239
0240   # Hack find the first comparison and fill test and threshold
0241   # shall this be put in meta?
0242   test_name=""
0243   test_threshold=0
0244   for comparison in directory.comparisons:
0245       test_name=comparison.test_name
0246       test_threshold=comparison.test_thr
0247       break
0248   if len(test_name)==0:
0249     for subdir in directory.subdirs:
0250       for comparison in subdir.comparisons:
0251         test_name=comparison.test_name
0252         test_threshold=comparison.test_thr
0253         break
0254       if len(test_name)!=0:break
0255       if len(test_name)==0:
0256         for subsubdir in subdir.subdirs:
0257           for comparison in subsubdir.comparisons:
0258             test_name=comparison.test_name
0259             test_threshold=comparison.test_thr
0260             break
0261           if len(test_name)!=0:break
0262         if len(test_name)==0:
0263           for subsubsubdir in subsubdir.subdirs:
0264             for comparison in subsubsubdir.comparisons:
0265               test_name=comparison.test_name
0266               test_threshold=comparison.test_thr
0267               break
0268             if len(test_name)!=0:break
0269
0270
0271   meta=directory.meta
0272
0273   html= '<div class="span-6">'+\
0274         '<h3>Summary</h3>'
0275   html+=get_dir_stats(directory)
0276   html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
0277   html+='</div>'
0278
0279   html+='<div class="span-7 colborder">'+\
0280         '%s'%(directory.get_piechart_js(150)) +\
0281         '</div>'+\
0282         '<div class="span-9 last">'
0283   if matrix_page:
0284     html+='<h3>Sample:</h3>'+\
0285           '<p class="caps">%s</p>'%meta.sample1+\
0286           '<h3>Run1 and Run2:</h3>'+\
0287           '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
0288   html+='<h3>Releases:</h3>'+\
0289         '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
0290         '<h3>Statistical Test (Pvalue threshold):</h3>'+\
0291         '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
0292         '</div>'+\
0293         '<hr>'
0294   return html
0295
0296 #-------------------------------------------------------------------------------
0297
0298 def get_comparisons(category,directory):
0299   """Prepare the comparisons between histograms and organise them in the page.
0300   Moreover create separate pages with the overlay and the single plots.
0301   """
0302   counter=1
0303   tot_counter=1
0304
0305   # get the right ones
0306   comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
0307   n_comparisons=len(comparisons)
0308
0309   is_reverse=True
0310   if category == FAIL:
0311     is_reverse=False
0312   comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
0313
0314
0315   dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
0316   html_comparisons=""
0317   for comparison in comparisons:
0318     plot_name=comparison.img_name
0319     if "http://" not in plot_name:
0320       plot_name= basename(comparison.img_name)
0321     class_type="colborder"
0322     if counter==3 or tot_counter==n_comparisons:
0323       class_type=" colborder last"
0324     comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
0325
0326
0327     if directory.do_pngs:
0328       png_link=comparison.img_name
0329       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0330                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0331                       '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
0332     else:
0333       big_fairy=fairy_url(directory.meta.run1,
0334                         directory.meta.run2,
0335                         directory.meta.sample1,
0336                         directory.meta.sample2,
0337                         directory.meta.release1,
0338                         directory.meta.release2,
0339                         comp_abs_path,
0340                         directory.meta.tier1,
0341                         directory.meta.tier2,
0342                         "",600,600)
0343       small_fairy=fairy_url(directory.meta.run1,
0344                         directory.meta.run2,
0345                         directory.meta.sample1,
0346                         directory.meta.sample2,
0347                         directory.meta.release1,
0348                         directory.meta.release2,
0349                         comp_abs_path,
0350                         directory.meta.tier1,
0351                         directory.meta.tier2)
0352
0353       single_fairy1=fairy_url_single(directory.meta.run1,
0354                                    directory.meta.sample1,
0355                                    directory.meta.release1,
0356                                    comp_abs_path,
0357                                    directory.meta.tier1,
0358                                    "",500,500)
0359       single_fairy2=fairy_url_single(directory.meta.run2,
0360                                    directory.meta.sample2,
0361                                    directory.meta.release2,
0362                                    comp_abs_path,
0363                                    directory.meta.tier2,
0364                                    "",500,500)
0365
0366       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0367                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0368                       '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
0369                       '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
0370                       '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
0371
0372     if counter==3:
0373       html_comparisons+="<hr>"
0374       counter=0
0375     counter+=1
0376     tot_counter+=1
0377
0378   if len(html_comparisons)!=0:
0379     html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
0380     html+=html_comparisons
0381     html+='<hr>'
0382     return html
0383   return ""
0384
0385 #-------------------------------------------------------------------------------
0386
0387 def get_rank_section(directory):
0388 # do Rank histo png
0389     imgname="RankSummary.png"
0390     gStyle.SetPadTickY(0)
0391     c=TCanvas("ranks","ranks",500,400)
0392     #gStyle.SetOptStat(0)
0393     c.cd()
0394
0395     h=directory.rank_histo
0396     rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
0397     rank_histof.SetLineWidth(2)
0398     for i in range(0,h.GetNbinsX()+1):
0399       rank_histof.SetBinContent(i,h.GetBinContent(i))
0400     h.SetTitle("Ranks Summary;Rank;Frequency")
0401     h.Draw("Hist")
0402     c.Update()
0403     rank_histof.ComputeIntegral()
0404     integral = rank_histof.GetIntegral()
0405     rank_histof.SetContent(integral)
0406
0407     rightmax = 1.1*rank_histof.GetMaximum()
0408     scale = gPad.GetUymax()/rightmax
0409     rank_histof.SetLineColor(kRed)
0410     rank_histof.Scale(scale)
0411     rank_histof.Draw("same")
0412
0413     #draw an axis on the right side
0414     axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
0415     axis.SetTitle("Cumulative")
0416     axis.SetTitleColor(kRed)
0417     axis.SetLineColor(kRed)
0418     axis.SetLabelColor(kRed)
0419     axis.Draw()
0420
0421     rank_histof.Draw("Same");
0422
0423
0424     c.Print(imgname)
0425
0426     page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
0427     page_html+='<div class="span-19"><img src="%s"></div>' %imgname
0428     page_html+='</div> <a href="#top">Top...</a><hr>'
0429
0430     return page_html
0431
0432 #-------------------------------------------------------------------------------
0433
0434 #-------------------------------------------------------------------------------
0435 def get_missing_objs_section(directory):
0436     """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
0437     """
0438     page_html = "Unpaired in %s</br>"%(directory.filename1)
0439     for elem in directory.different_histograms['file1']:
0440         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
0441     page_html +="</br>"
0442     page_html += "Unpaired in %s</br>"%(directory.filename2)
0443     for elem in directory.different_histograms['file2']:
0444         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
0445     return page_html
0446 #-------------------------------------------------------------------------------
0447
0448 def directory2html(directory, hashing, standalone, depth=0):
0449   """Converts a directory tree into html pages, very nice ones.
0450   """
0451   #print "d2html: depth", str(depth)," dir ",directory.name
0452   depth+=1
0453   #old_cwd=getcwd()
0454   #if not exists(directory.name) and len(directory.name)>0:
0455     #makedirs(directory.name)
0456
0457   #if len(directory.name)>0:
0458     #chdir(directory.name)
0459
0460   for subdir in directory.subdirs:
0461     directory2html(subdir,hashing,standalone, depth)
0462
0463   page_html=get_page_header(directory, standalone)+\
0464             get_title_section(directory,hashing, standalone, depth)+\
0465             get_summary_section(directory)+\
0466             get_subdirs_section(directory, hashing)
0467
0468   for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
0469                      (directory.n_comp_nulls >0,NULL ),
0470
0471                      (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
0472                      (directory.n_comp_skiped >0,SKIPED)):
0473     if do_cat:
0474       page_html+=get_comparisons(cat,directory)
0475
0476   if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
0477     page_html += get_missing_objs_section(directory)
0478
0479   # Distribution of ranks
0480
0481   if depth==1:
0482     page_html+=get_rank_section(directory)
0483
0484
0485   page_html+=get_page_footer()
0486
0487   page_name=directory.name
0488
0489   if len(page_name)==0:
0490     page_name="RelMonSummary"
0491   if hashing:
0492       if page_name != "RelMonSummary":
0493           #print " ## oFile path: %s"%(join(directory.full_path))
0494           #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
0495           ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
0496       else:
0497           ofilename = "RelMonSummary.html"
0498   else:
0499       ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
0500       ofilename=ofilename.strip("_")
0501   #print "Writing on %s" %ofilename
0502   ofile=open(ofilename,"w")
0503   ofile.write(page_html)
0504   ofile.close()
0505
0506   #chdir(old_cwd)
0507
0508 def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):
0509
0510   color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
0511   font_size = int(w/9)
0512   gauge_max = 1. - rate
0513
0514   name = random.getrandbits(64) # just a random has for the canvas
0515   html = ""
0516   html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)
0517
0518   # "gauge" chart
0519   html += '<script> new Chart("%s",'%(name)
0520   html += '{ type: "doughnut",'
0521
0522   # data
0523   html += 'data: {'
0524   html += 'labels: ["Success", "Failure"],'
0525   html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
0526   html += 'data: [%f,%f]}] },'%(rate,gauge_max)
0527
0528   #options
0529   html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' ## in radiants
0530   html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
0531   html += '}); </script>'
0532   if add_rate:
0533     html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
0534     html += '<span style="color: %s; font-family: courier; font-size:  %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)
0535
0536   return html
0537 #-------------------------------------------------------------------------------
0538
0539 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
0540   # Let's make a summary for an overview in categories act on the global dir
0541   aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
0542
0543   list_of_names=[]
0544   if the_aggr_pairs==[]:
0545     for samplename,sampledir in dir_dict.items():
0546       for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
0547         if not subsysdirname in list_of_names:
0548           list_of_names.append(subsysdirname)
0549           the_aggr_pairs.append((subsysdirname,[subsysdirname]))
0550
0551   #print the_aggr_pairs
0552   for cat_name, subdir_list in the_aggr_pairs:
0553     total_successes=0.
0554     total_directory_successes=0
0555     total_weight=0.
0556     present_subdirs={}
0557     total_ndirs=0
0558     # Loop on samples
0559     for dirname, sampledir in dir_dict.items():
0560       # Loop on directories
0561       for subdirname,subdir in sampledir.get_subdirs_dict().items():
0562         if subdirname in subdir_list:
0563           nsucc=subdir.n_successes
0564           total_successes+=nsucc
0565           weight=subdir.weight
0566           total_weight+=weight
0567           total_ndirs+=1
0568
0569           total_directory_successes+= float(nsucc)/weight
0570           if subdirname in present_subdirs:
0571             this_dir_dict=present_subdirs[subdirname]
0572             this_dir_dict["nsucc"]+=nsucc
0573             this_dir_dict["weight"]+=weight
0574           else:
0575             present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
0576         # Make it usable also for subdirectories
0577         for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
0578           for pathname in [name for name in subdir_list if "/" in name]:
0579             selected_subdirname,selected_subsubdirname = pathname.split("/")
0580             if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
0581               #print "Studying directory ",subsubdirname," in directory ",subdirname
0582               nsucc=subsubdir.n_successes
0583               total_successes+=nsucc
0584               weight=subsubdir.weight
0585               total_weight+=weight
0586               total_ndirs+=1
0587               total_directory_successes+= float(nsucc)/weight
0588
0589               if subsubdirname in present_subdirs:
0590                 this_dir_dict=present_subdirs[subsubdirname]
0591                 this_dir_dict["nsucc"]+=nsucc
0592                 this_dir_dict["weight"]+=weight
0593               else:
0594                 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
0595
0596     if total_ndirs == 0:
0597       print("No directory of the category %s is present in the samples: skipping." %cat_name)
0598       continue
0599
0600     average_success_rate=total_directory_successes/(total_ndirs)
0601     aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
0602
0603   return aggr_pairs_info
0604
0605 #-------------------------------------------------------------------------------
0606
0607 def make_categories_summary(dir_dict,aggregation_rules):
0608
0609   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0610
0611   #print aggr_pairs_info
0612
0613   # Now Let's build the HTML
0614
0615   html= '<div class="span-20 colborder">'
0616   html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
0617
0618   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0619     #print cat_name,present_subdirs,total_weight,average_success_rate
0620     html+='<div class="span-3 prepend-0 colborder">'
0621     html+='<h3>%s</h3>'%cat_name
0622     html+='<div><span class="alt">Avg. Success rate:</span></div>'
0623     html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
0624     html+='</div>'
0625     html+='<div class="span-9">'
0626
0627     html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
0628     html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
0629     html+='<ul>'
0630     for subdirname in sorted(present_subdirs.keys()):
0631       this_dir_dict=present_subdirs[subdirname]
0632       nsucc=this_dir_dict["nsucc"]
0633       weight=this_dir_dict["weight"]
0634       html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
0635     html+='</ul>'
0636     html+='</div>'
0637
0638     html+='<div class="span-6 last">'
0639     html+=build_gauge_js(average_success_rate,add_rate=True)
0640     html+='</div>'
0641
0642     html+='<hr>'
0643   return html+'<br><a href="#top">Top...</a> </div><hr>'
0644
0645  #-------------------------------------------------------------------------------
0646
0647 def make_twiki_table(dir_dict,aggregation_rules):
0648
0649   # decide the release
0650   meta= list(dir_dict.items())[0][1].meta
0651   releases=sorted([meta.release1,meta.release2])
0652   latest_release=releases[1].split("-")[0]
0653
0654
0655   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0656
0657   # Now Let's build the HTML
0658
0659   html= '<div class="span-20 colborder">'
0660   html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
0661   html+='<div>| Release | Comparison |'
0662   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0663     html+=cat_name
0664     html+=" | "
0665   html+='</div>'
0666
0667   html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release
0668
0669   # Now add all the line with small gauges
0670
0671   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0672     #print cat_name,present_subdirs,total_weight,average_success_rate
0673     html+=build_gauge_js(average_success_rate,w=40)
0674     html+=" | "
0675
0676   html+='</div> <a href="#top">Top...</a>'
0677   html+='<hr>'
0678   return html+'</div>'
0679
0680 #-------------------------------------------------------------------------------
0681
0682 def get_pie_tooltip(directory):
0683   tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
0684   return tooltip
0685
0686 #-------------------------------------------------------------------------------
0687
0688 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
0689
0690   aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
0691
0692   script="""
0693     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
0694     <script type="text/javascript">
0695       google.load("visualization", "1", {packages:["corechart"]});
0696       google.setOnLoadCallback(drawChart);
0697       function drawChart() {
0698         var data = new google.visualization.DataTable();
0699         data.addColumn('string', 'DQM Directory');
0700         data.addColumn('number', 'Success Rate');
0701         """
0702   script+="data.addRows(%i);\n"%len(aggr_pairs_info)
0703   counter=0
0704   for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
0705     #print subsystname,present_directories
0706     script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
0707     script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
0708     counter+=1
0709   script+="""
0710         var chart = new google.visualization.BarChart(document.getElementById('%s'));
0711         chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
0712                           vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
0713                          });
0714       }
0715     </script>
0716     """%(name,35*counter,title)
0717   return script
0718
0719
0720 #-------------------------------------------------------------------------------
0721
0722 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
0723   """Create a table, with as rows the directories and as columns the samples.
0724   Each box in the table will contain a pie chart linking to the directory.
0725   """
0726   #aggregation_rules={}
0727   #aggregation_rules_twiki={}
0728
0729   chdir(indir)
0730   if os.path.isabs(indir):
0731       title = basename(indir)
0732   else:
0733       title=indir
0734   title=title.strip(".")
0735   title=title.strip("/")
0736
0737
0738   # Get the list of pickles
0739   sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
0740
0741   # Load directories, build a list of all first level subdirs
0742   dir_unpicklers=[]
0743   n_unpicklers=0
0744   for sample_pkl in sample_pkls:
0745     dir_unpickler=unpickler(sample_pkl)
0746     dir_unpickler.start()
0747     n_unpicklers+=1
0748     dir_unpicklers.append(dir_unpickler)
0749     if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
0750       n_unpicklers=0
0751       for dir_unpickler in dir_unpicklers:
0752         dir_unpickler.join()
0753
0754   dir_dict={}
0755
0756   # create a fake global directory
0757   global_dir=Directory("global","")
0758   for dir_unpickler in dir_unpicklers:
0759     dir_unpickler.join()
0760     directory=dir_unpickler.directory
0761     #directory.prune("Run summary")
0762     #directory.calcStats()
0763     global_dir.meta=directory.meta
0764     dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
0765     global_dir.subdirs.append(directory)
0766
0767   global_dir.calcStats()
0768
0769   directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
0770   categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
0771
0772   page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
0773   rel1=""
0774   rel2=""
0775   try:
0776     rel1,rel2=title.split("VS")
0777   except:
0778     rel1=global_dir.meta.release1.split("-")[0]
0779     rel2=global_dir.meta.release2.split("-")[0]
0780     global_dir.meta.release1=rel1
0781     global_dir.meta.release2=rel2
0782
0783   # union of all subdirs names
0784   all_subdirs=[]
0785   for directory in dir_dict.values():
0786     for subdir_name in directory.get_subdirs_names():
0787       all_subdirs.append(subdir_name)
0788   all_subdirs=sorted(list(set(all_subdirs)))
0789
0790   #print " $$ all_subdirs: %s" %(all_subdirs)
0791
0792   # Get The title
0793   page_html+= '<div class="span-20">'+\
0794               '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
0795               '</div>'+\
0796               '<div class="span-1">'+\
0797               '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
0798               '</div>'+\
0799               '<hr>'
0800   page_html+='<div class="span-24"><p></p></div>\n'*3
0801
0802   # Get The summary section
0803   page_html+= get_summary_section(global_dir,False)
0804
0805   # Make the anchor sections
0806   page_html+= '<div class="span-24">'
0807   page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
0808               '<ul>'+\
0809               '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
0810               '<li><a href="#categories">Categories</a></li>'+\
0811               '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
0812           '<li><a href="#summary_table">Summary Table</a></li>'+\
0813               '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
0814               '<li><a href="#twiki_table">Twiki Table</a></li>'+\
0815               '</ul>'+\
0816               '</div><hr>'
0817
0818
0819   # Make the CategoriesBar chart
0820   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
0821   page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
0822
0823   # Make the gauges per categories
0824   page_html+=make_categories_summary(dir_dict,aggregation_rules)
0825
0826   # Make the Directories chart
0827   page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
0828
0829   # Barbarian vertical space. Suggestions are welcome
0830   for i in range(2):
0831     page_html+='<div class="span-24"><p></p></div>\n'
0832
0833
0834   # Prepare the table
0835   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'
0836
0837   div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
0838   page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
0839   page_html+="""
0840         <table>
0841         <thead>
0842           <tr>
0843           <th> <p class = "vertical_head center_head"></p> </th>
0844   """
0845
0846   # First row with samples
0847   page_html+="""
0848           <th> <p class = "vertical_head center_head">Summary</p></th>"""
0849
0850   sorted_samples=sorted(dir_dict.keys())
0851   for sample in sorted_samples:
0852     if "_" in sample and "Data" not in sample:
0853       sample_nick="_".join(sample.split("X_")[0].split("_")[:-1])
0854       # Cleaning for MC: just the fragment
0855     elif "Data" in sample and "RelVal" in sample:
0856       sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
0857       # Cleaning for Data: PD + Era + Run
0858     else:
0859       sample_nick = sample
0860
0861     page_html+="""
0862           <th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
0863   page_html+="</tr> \n </thead> \n </tbody> \n"
0864
0865
0866  # FIRST ROW
0867  # Now the summaries  at the beginning of the table
0868   page_html+="<tr>"
0869   page_html+='<td class="sticky-col">'
0870
0871   page_html+='<b>Summary</b></td>'
0872   page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
0873   for sample in sorted_samples:
0874     col=dir_dict[sample]
0875     # check if the directory was a top one or not
0876     summary_page_name="RelMonSummary.html"
0877     if col.name!="":
0878       summary_page_name=hash_name(col.name, hashing_flag)+".html"
0879     title = get_pie_tooltip(col)
0880     chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
0881     page_html+='<td>'
0882     page_html+='%s </a></td>' %(chart)
0883   page_html+="</tr>"
0884
0885   # Now the content
0886   for subdir_name in all_subdirs:
0887
0888     page_html+='          <tr>\n'
0889     page_html+='          <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name
0890
0891     row_summary=Directory("row_summary","")
0892     sample_counter=0
0893     n_samples=len(sorted_samples)
0894
0895     for sample in sorted_samples:
0896       subdirs_dict=directory.get_subdirs_dict()
0897       directory=dir_dict[sample]
0898       dir_is_there=subdir_name in subdirs_dict
0899       if dir_is_there:
0900         row_summary.subdirs.append(subdirs_dict[subdir_name])
0901
0902     # one first row for the summary!
0903     row_summary.calcStats()
0904     title = get_pie_tooltip(col)
0905     chart = row_summary.get_piechart_js(50)
0906     page_html+='<td><div>'#<div class="span-1">'
0907     page_html+= chart + '</div></td>'
0908
0909     for sample in sorted_samples:
0910       sample_counter+=1
0911
0912       directory=dir_dict[sample]
0913       subdirs_dict=directory.get_subdirs_dict()
0914
0915       # Check if the directory is the top one
0916       if directory.name!="":
0917         # We did not run on the topdir
0918         #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
0919         #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
0920         if hashing_flag:
0921           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
0922         else:
0923           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
0924       else:
0925         #print "   ## summary_page: %s"%(directory.name+subdir_name)
0926         #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
0927         summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
0928       dir_is_there=subdir_name in subdirs_dict
0929
0930       img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
0931       page_html+='<td><div class="span-1">'
0932
0933       if dir_is_there:
0934         #row_summary.subdirs.append(subdirs_dict[subdir_name])
0935         chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
0936         page_html+='%s'%chart
0937       else:
0938         page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
0939       page_html+='</div></td>'
0940
0941     page_html+="          </tr>\n"
0942
0943
0944
0945   page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'
0946
0947   page_html+=get_rank_section(global_dir)
0948
0949   #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
0950   # ^ commenting out for the moment, not really useful nor used
0951
0952   page_html+=get_page_footer()
0953   return page_html
0954
0955
0956 #-----------UPDATES------
0957 def hash_name(file_name, flag):
0958     if flag: #if hashing flag is ON then return
0959         if (3,0,0) <= sys.version_info:
0960             return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
0961         return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
0962     else:
0963         return file_name #return standart name