RelMon/python/directories2html.py

0001 ################################################################################
0002 # RelMon: a tool for automatic Release Comparison
0003 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0004 #
0005
0006 #
0007 #
0008 # Danilo Piparo CERN - danilo.piparo@cern.ch
0009 #
0010 ################################################################################
0011
0012 from builtins import range
0013 from os import chdir,getcwd,listdir,makedirs
0014 from os.path import basename,join,exists
0015 import html
0016
0017 import sys
0018 theargv=sys.argv
0019 sys.argv=[]
0020 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
0021 sys.argv=theargv
0022
0023 import os
0024 import hashlib
0025 import random
0026 from matplotlib.colors import to_hex
0027
0028 if "RELMON_SA" in os.environ:
0029   from .dirstructure import Comparison,Directory
0030   from .definitions import *
0031   from .utils import unpickler
0032 else:
0033   from Utilities.RelMon.dirstructure import Comparison,Directory
0034   from Utilities.RelMon.definitions import *
0035   from Utilities.RelMon.utils import unpickler
0036
0037 #-------------------------------------------------------------------------------
0038
0039 def encode_obj_url(url):
0040   for old,new in url_encode_dict.items():
0041     url=url.replace(old,new)
0042   return url
0043
0044 def plot_size(h=250,w=200):
0045   return "w=%s;h=%s" %(h,w)
0046
0047 def build_obj_addr(run,sample,version,plot_path,tier):
0048   slash="/"
0049   obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
0050   obj_url+=plot_path
0051   while obj_url.endswith(slash):
0052     obj_url=obj_url[:-1]
0053   while slash*2 in obj_url:
0054     obj_url=obj_url.replace(slash*2,slash)
0055   return obj_url
0056
0057 def build_obj(run,sample,version,plot_path,tier):
0058   obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
0059   return encode_obj_url(obj_url)
0060
0061 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
0062   fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
0063   fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
0064   fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
0065   if len(draw_opts)>0:
0066     fairy_url+="drawopts=%s;" %draw_opts
0067   fairy_url+= plot_size(h,w)
0068   fairy_url += ";ref=ratiooverlay"
0069   return fairy_url
0070
0071 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
0072   fairy_url = "%s/%s/plotfairy/" %(server,base_url)
0073   fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
0074   fairy_url+= "?%s"%plot_size(h,w)
0075   if len(draw_opts)>0:
0076     fairy_url+="drawopts=%s;" %draw_opts
0077   return fairy_url
0078
0079 #-------------------------------------------------------------------------------
0080 style_location="/cms-service-reldqm"
0081 def get_page_header(directory=None, standalone=False, additional_header=""):
0082   style_location="/cms-service-reldqm"
0083   if standalone:
0084     style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2
0085   javascripts=''
0086   style=''
0087   tablestyle=''
0088   thead_h = 400
0089   wrapper_h = 1500
0090   if directory!=None and len(directory.comparisons)>0:
0091     meta=directory.meta
0092     style='img.fail {border:1px solid #ff0000;}\n'+\
0093           'img.succes {border:1px solid #00ff00;}\n'+\
0094           'img.null {border:1px solid #ffff00;}\n'+\
0095           'img.skiped {border:1px solid #7a7a7a;}\n'+\
0096           'a.black_link:link {color: #333333}\n'+\
0097           'a.black_link:hover {color: #737373}\n'+\
0098           'a.black_link:visited {color: #333333}\n'+\
0099           'a.black_link:active {color: #333333}\n'
0100     ## fixed first row and first column table
0101
0102     wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)
0103
0104   tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
0105   tablestyle += 'table {  position: relative; border-collapse: separate; border-spacing: 0;} \n'
0106   tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
0107   tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
0108   tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
0109   tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
0110   tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1  ; width: 200px; left:0}\n'
0111   tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
0112   tablestyle += '.vertical_head {top: 60%; -webkit-transform:  translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
0113   javascripts=""
0114
0115
0116
0117   html='<html>'+\
0118        '<head>'+\
0119        '<title>RelMon Summary</title>'+\
0120        '<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' + \
0121        '<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%style_location+\
0122        '<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%style_location+\
0123        '<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%style_location+\
0124        '<style type="text/css">'+\
0125        '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
0126        '%s'%style+\
0127        '%s'%tablestyle +\
0128        '</style>'+\
0129        '%s'%javascripts+\
0130        '%s'%additional_header+\
0131        '</head>'+\
0132        '<body>'+\
0133        '<div class="container">'
0134
0135   return html
0136
0137 #-------------------------------------------------------------------------------
0138
0139 def get_page_footer():
0140     return '</div></body></html>'
0141
0142 #-------------------------------------------------------------------------------
0143
0144 def get_title_section(directory, hashing_flag, standalone, depth=2):
0145
0146   mother_name=basename(directory.mother_dir)
0147   mother_file_name=""
0148   if depth==1:
0149     mother_file_name="../RelMonSummary.html"
0150     if mother_name!="":
0151       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0152   elif depth==2:
0153     mother_file_name="RelMonSummary.html"
0154     #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
0155     if mother_name!="":
0156       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0157   else:
0158       if hashing_flag:
0159           files = directory.mother_dir.split("/")
0160           if len(files) != 1:
0161               dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
0162           else:
0163               dir_name = files[-1]
0164           dir_name = directory.mother_dir
0165           mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
0166       else:
0167           mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
0168           mother_file_name=mother_file_name.strip("_")
0169
0170   link_to_mother='<a href="%s">..</a>' %mother_file_name
0171   html= '<div class="span-20">'+\
0172         '<h1>%s</h1>'%directory.name+\
0173         '</div>'+\
0174         '<div class="span-1">'+\
0175         '<h1>%s</h1>'%link_to_mother+\
0176         '</div>'+\
0177         '<div class="span-3 last">'+\
0178         '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
0179         '</div>'+\
0180         '<hr>'
0181   if len(mother_name)>0:
0182     html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
0183           '<hr>'
0184   return html
0185
0186 #-------------------------------------------------------------------------------
0187 def get_dir_stats(directory):
0188   html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
0189   html+='<ul>'
0190   if directory.n_successes>0:
0191     html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
0192   if directory.n_nulls>0:
0193     html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
0194   if directory.n_fails>0:
0195     html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
0196   if directory.n_skiped>0:
0197     html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
0198   if directory.n_missing_objs>0:
0199     html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
0200   html+='</ul>'
0201   return html
0202
0203 #-------------------------------------------------------------------------------
0204
0205 def get_subdirs_section(directory, hashing_flag):
0206   if len(directory.subdirs)==0:
0207     return ""
0208   html= '<div class="span-20 colborder">'
0209   html+='<h2 class="alt">Sub-Directories</h2>'
0210   # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
0211   # so reverse :)
0212   sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
0213   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
0214   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
0215   for subdir in sorted_subdirs:
0216     name=subdir.name
0217     if hashing_flag:
0218         link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
0219     else:
0220         link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
0221         link=link.strip("_")
0222     html+='<div class="span-4 prepend-2 colborder">'
0223     html+='<h3>%s</h3>'%name
0224     html+='</div>'
0225
0226     html+='<div class="span-7">'
0227     html+=get_dir_stats(subdir)
0228     html+='</div>'
0229
0230     html+='<div class="span-6 last">'
0231     html+= subdir.get_piechart_js(120,link)
0232     html+='</div>'
0233
0234     html+='<hr>'
0235   return html+'</div>'
0236
0237
0238 #-------------------------------------------------------------------------------
0239
0240 def get_summary_section(directory,matrix_page=True):
0241
0242   # Hack find the first comparison and fill test and threshold
0243   # shall this be put in meta?
0244   test_name=""
0245   test_threshold=0
0246   for comparison in directory.comparisons:
0247       test_name=comparison.test_name
0248       test_threshold=comparison.test_thr
0249       break
0250   if len(test_name)==0:
0251     for subdir in directory.subdirs:
0252       for comparison in subdir.comparisons:
0253         test_name=comparison.test_name
0254         test_threshold=comparison.test_thr
0255         break
0256       if len(test_name)!=0:break
0257       if len(test_name)==0:
0258         for subsubdir in subdir.subdirs:
0259           for comparison in subsubdir.comparisons:
0260             test_name=comparison.test_name
0261             test_threshold=comparison.test_thr
0262             break
0263           if len(test_name)!=0:break
0264         if len(test_name)==0:
0265           for subsubsubdir in subsubdir.subdirs:
0266             for comparison in subsubsubdir.comparisons:
0267               test_name=comparison.test_name
0268               test_threshold=comparison.test_thr
0269               break
0270             if len(test_name)!=0:break
0271
0272
0273   meta=directory.meta
0274
0275   html= '<div class="span-6">'+\
0276         '<h3>Summary</h3>'
0277   html+=get_dir_stats(directory)
0278   html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
0279   html+='</div>'
0280
0281   html+='<div class="span-7 colborder">'+\
0282         '%s'%(directory.get_piechart_js(150)) +\
0283         '</div>'+\
0284         '<div class="span-9 last">'
0285   if matrix_page:
0286     html+='<h3>Sample:</h3>'+\
0287           '<p class="caps">%s</p>'%meta.sample1+\
0288           '<h3>Run1 and Run2:</h3>'+\
0289           '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
0290   html+='<h3>Releases:</h3>'+\
0291         '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
0292         '<h3>Statistical Test (Pvalue threshold):</h3>'+\
0293         '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
0294         '</div>'+\
0295         '<hr>'
0296   return html
0297
0298 #-------------------------------------------------------------------------------
0299
0300 def get_comparisons(category,directory):
0301   """Prepare the comparisons between histograms and organise them in the page.
0302   Moreover create separate pages with the overlay and the single plots.
0303   """
0304   counter=1
0305   tot_counter=1
0306
0307   # get the right ones
0308   comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
0309   n_comparisons=len(comparisons)
0310
0311   is_reverse=True
0312   if category == FAIL:
0313     is_reverse=False
0314   comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
0315
0316
0317   dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
0318   html_comparisons=""
0319   for comparison in comparisons:
0320     plot_name=comparison.img_name
0321     if "http://" not in plot_name:
0322       plot_name= basename(comparison.img_name)
0323     class_type="colborder"
0324     if counter==3 or tot_counter==n_comparisons:
0325       class_type=" colborder last"
0326     comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
0327
0328
0329     if directory.do_pngs:
0330       png_link=comparison.img_name
0331       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0332                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0333                       '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
0334     else:
0335       big_fairy=fairy_url(directory.meta.run1,
0336                         directory.meta.run2,
0337                         directory.meta.sample1,
0338                         directory.meta.sample2,
0339                         directory.meta.release1,
0340                         directory.meta.release2,
0341                         comp_abs_path,
0342                         directory.meta.tier1,
0343                         directory.meta.tier2,
0344                         "",600,600)
0345       small_fairy=fairy_url(directory.meta.run1,
0346                         directory.meta.run2,
0347                         directory.meta.sample1,
0348                         directory.meta.sample2,
0349                         directory.meta.release1,
0350                         directory.meta.release2,
0351                         comp_abs_path,
0352                         directory.meta.tier1,
0353                         directory.meta.tier2)
0354
0355       single_fairy1=fairy_url_single(directory.meta.run1,
0356                                    directory.meta.sample1,
0357                                    directory.meta.release1,
0358                                    comp_abs_path,
0359                                    directory.meta.tier1,
0360                                    "",500,500)
0361       single_fairy2=fairy_url_single(directory.meta.run2,
0362                                    directory.meta.sample2,
0363                                    directory.meta.release2,
0364                                    comp_abs_path,
0365                                    directory.meta.tier2,
0366                                    "",500,500)
0367
0368       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0369                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0370                       '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
0371                       '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
0372                       '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
0373
0374     if counter==3:
0375       html_comparisons+="<hr>"
0376       counter=0
0377     counter+=1
0378     tot_counter+=1
0379
0380   if len(html_comparisons)!=0:
0381     html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
0382     html+=html_comparisons
0383     html+='<hr>'
0384     return html
0385   return ""
0386
0387 #-------------------------------------------------------------------------------
0388
0389 def get_rank_section(directory):
0390 # do Rank histo png
0391     imgname="RankSummary.png"
0392     gStyle.SetPadTickY(0)
0393     c=TCanvas("ranks","ranks",500,400)
0394     #gStyle.SetOptStat(0)
0395     c.cd()
0396
0397     h=directory.rank_histo
0398     rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
0399     rank_histof.SetLineWidth(2)
0400     for i in range(0,h.GetNbinsX()+1):
0401       rank_histof.SetBinContent(i,h.GetBinContent(i))
0402     h.SetTitle("Ranks Summary;Rank;Frequency")
0403     h.Draw("Hist")
0404     c.Update()
0405     rank_histof.ComputeIntegral()
0406     integral = rank_histof.GetIntegral()
0407     rank_histof.SetContent(integral)
0408
0409     rightmax = 1.1*rank_histof.GetMaximum()
0410     scale = gPad.GetUymax()/rightmax
0411     rank_histof.SetLineColor(kRed)
0412     rank_histof.Scale(scale)
0413     rank_histof.Draw("same")
0414
0415     #draw an axis on the right side
0416     axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
0417     axis.SetTitle("Cumulative")
0418     axis.SetTitleColor(kRed)
0419     axis.SetLineColor(kRed)
0420     axis.SetLabelColor(kRed)
0421     axis.Draw()
0422
0423     rank_histof.Draw("Same");
0424
0425
0426     c.Print(imgname)
0427
0428     page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
0429     page_html+='<div class="span-19"><img src="%s"></div>' %imgname
0430     page_html+='</div> <a href="#top">Top...</a><hr>'
0431
0432     return page_html
0433
0434 #-------------------------------------------------------------------------------
0435
0436 #-------------------------------------------------------------------------------
0437 def get_missing_objs_section(directory):
0438     """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
0439     """
0440     page_html = "Unpaired in %s</br>"%(directory.filename1)
0441     for elem in directory.different_histograms['file1']:
0442         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
0443     page_html +="</br>"
0444     page_html += "Unpaired in %s</br>"%(directory.filename2)
0445     for elem in directory.different_histograms['file2']:
0446         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
0447     return page_html
0448 #-------------------------------------------------------------------------------
0449
0450 def directory2html(directory, hashing, standalone, depth=0):
0451   """Converts a directory tree into html pages, very nice ones.
0452   """
0453   #print "d2html: depth", str(depth)," dir ",directory.name
0454   depth+=1
0455   #old_cwd=getcwd()
0456   #if not exists(directory.name) and len(directory.name)>0:
0457     #makedirs(directory.name)
0458
0459   #if len(directory.name)>0:
0460     #chdir(directory.name)
0461
0462   for subdir in directory.subdirs:
0463     directory2html(subdir,hashing,standalone, depth)
0464
0465   page_html=get_page_header(directory, standalone)+\
0466             get_title_section(directory,hashing, standalone, depth)+\
0467             get_summary_section(directory)+\
0468             get_subdirs_section(directory, hashing)
0469
0470   for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
0471                      (directory.n_comp_nulls >0,NULL ),
0472
0473                      (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
0474                      (directory.n_comp_skiped >0,SKIPED)):
0475     if do_cat:
0476       page_html+=get_comparisons(cat,directory)
0477
0478   if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
0479     page_html += get_missing_objs_section(directory)
0480
0481   # Distribution of ranks
0482
0483   if depth==1:
0484     page_html+=get_rank_section(directory)
0485
0486
0487   page_html+=get_page_footer()
0488
0489   page_name=directory.name
0490
0491   if len(page_name)==0:
0492     page_name="RelMonSummary"
0493   if hashing:
0494       if page_name != "RelMonSummary":
0495           #print " ## oFile path: %s"%(join(directory.full_path))
0496           #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
0497           ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
0498       else:
0499           ofilename = "RelMonSummary.html"
0500   else:
0501       ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
0502       ofilename=ofilename.strip("_")
0503   #print "Writing on %s" %ofilename
0504   ofile=open(ofilename,"w")
0505   ofile.write(page_html)
0506   ofile.close()
0507
0508   #chdir(old_cwd)
0509
0510 def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):
0511
0512   color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
0513   font_size = int(w/9)
0514   gauge_max = 1. - rate
0515
0516   name = random.getrandbits(64) # just a random has for the canvas
0517   html = ""
0518   html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)
0519
0520   # "gauge" chart
0521   html += '<script> new Chart("%s",'%(name)
0522   html += '{ type: "doughnut",'
0523
0524   # data
0525   html += 'data: {'
0526   html += 'labels: ["Success", "Failure"],'
0527   html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
0528   html += 'data: [%f,%f]}] },'%(rate,gauge_max)
0529
0530   #options
0531   html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' ## in radiants
0532   html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
0533   html += '}); </script>'
0534   if add_rate:
0535     html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
0536     html += '<span style="color: %s; font-family: courier; font-size:  %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)
0537
0538   return html
0539 #-------------------------------------------------------------------------------
0540
0541 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
0542   # Let's make a summary for an overview in categories act on the global dir
0543   aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
0544
0545   list_of_names=[]
0546   if the_aggr_pairs==[]:
0547     for samplename,sampledir in dir_dict.items():
0548       for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
0549         if not subsysdirname in list_of_names:
0550           list_of_names.append(subsysdirname)
0551           the_aggr_pairs.append((subsysdirname,[subsysdirname]))
0552
0553   #print the_aggr_pairs
0554   for cat_name, subdir_list in the_aggr_pairs:
0555     total_successes=0.
0556     total_directory_successes=0
0557     total_weight=0.
0558     present_subdirs={}
0559     total_ndirs=0
0560     # Loop on samples
0561     for dirname, sampledir in dir_dict.items():
0562       # Loop on directories
0563       for subdirname,subdir in sampledir.get_subdirs_dict().items():
0564         if subdirname in subdir_list:
0565           nsucc=subdir.n_successes
0566           total_successes+=nsucc
0567           weight=subdir.weight
0568           total_weight+=weight
0569           total_ndirs+=1
0570
0571           total_directory_successes+= float(nsucc)/weight
0572           if subdirname in present_subdirs:
0573             this_dir_dict=present_subdirs[subdirname]
0574             this_dir_dict["nsucc"]+=nsucc
0575             this_dir_dict["weight"]+=weight
0576           else:
0577             present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
0578         # Make it usable also for subdirectories
0579         for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
0580           for pathname in [name for name in subdir_list if "/" in name]:
0581             selected_subdirname,selected_subsubdirname = pathname.split("/")
0582             if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
0583               #print "Studying directory ",subsubdirname," in directory ",subdirname
0584               nsucc=subsubdir.n_successes
0585               total_successes+=nsucc
0586               weight=subsubdir.weight
0587               total_weight+=weight
0588               total_ndirs+=1
0589               total_directory_successes+= float(nsucc)/weight
0590
0591               if subsubdirname in present_subdirs:
0592                 this_dir_dict=present_subdirs[subsubdirname]
0593                 this_dir_dict["nsucc"]+=nsucc
0594                 this_dir_dict["weight"]+=weight
0595               else:
0596                 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
0597
0598     if total_ndirs == 0:
0599       print("No directory of the category %s is present in the samples: skipping." %cat_name)
0600       continue
0601
0602     average_success_rate=total_directory_successes/(total_ndirs)
0603     aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
0604
0605   return aggr_pairs_info
0606
0607 #-------------------------------------------------------------------------------
0608
0609 def make_categories_summary(dir_dict,aggregation_rules):
0610
0611   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0612
0613   #print aggr_pairs_info
0614
0615   # Now Let's build the HTML
0616
0617   html= '<div class="span-20 colborder">'
0618   html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
0619
0620   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0621     #print cat_name,present_subdirs,total_weight,average_success_rate
0622     html+='<div class="span-3 prepend-0 colborder">'
0623     html+='<h3>%s</h3>'%cat_name
0624     html+='<div><span class="alt">Avg. Success rate:</span></div>'
0625     html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
0626     html+='</div>'
0627     html+='<div class="span-9">'
0628
0629     html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
0630     html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
0631     html+='<ul>'
0632     for subdirname in sorted(present_subdirs.keys()):
0633       this_dir_dict=present_subdirs[subdirname]
0634       nsucc=this_dir_dict["nsucc"]
0635       weight=this_dir_dict["weight"]
0636       html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
0637     html+='</ul>'
0638     html+='</div>'
0639
0640     html+='<div class="span-6 last">'
0641     html+=build_gauge_js(average_success_rate,add_rate=True)
0642     html+='</div>'
0643
0644     html+='<hr>'
0645   return html+'<br><a href="#top">Top...</a> </div><hr>'
0646
0647  #-------------------------------------------------------------------------------
0648
0649 def make_twiki_table(dir_dict,aggregation_rules):
0650
0651   # decide the release
0652   meta= list(dir_dict.items())[0][1].meta
0653   releases=sorted([meta.release1,meta.release2])
0654   latest_release=releases[1].split("-")[0]
0655
0656
0657   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0658
0659   # Now Let's build the HTML
0660
0661   html= '<div class="span-20 colborder">'
0662   html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
0663   html+='<div>| Release | Comparison |'
0664   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0665     html+=cat_name
0666     html+=" | "
0667   html+='</div>'
0668
0669   html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release
0670
0671   # Now add all the line with small gauges
0672
0673   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0674     #print cat_name,present_subdirs,total_weight,average_success_rate
0675     html+=build_gauge_js(average_success_rate,w=40)
0676     html+=" | "
0677
0678   html+='</div> <a href="#top">Top...</a>'
0679   html+='<hr>'
0680   return html+'</div>'
0681
0682 #-------------------------------------------------------------------------------
0683
0684 def get_pie_tooltip(directory):
0685   tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
0686   return tooltip
0687
0688 #-------------------------------------------------------------------------------
0689
0690 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
0691
0692   aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
0693
0694   script="""
0695     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
0696     <script type="text/javascript">
0697       google.load("visualization", "1", {packages:["corechart"]});
0698       google.setOnLoadCallback(drawChart);
0699       function drawChart() {
0700         var data = new google.visualization.DataTable();
0701         data.addColumn('string', 'DQM Directory');
0702         data.addColumn('number', 'Success Rate');
0703         """
0704   script+="data.addRows(%i);\n"%len(aggr_pairs_info)
0705   counter=0
0706   for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
0707     #print subsystname,present_directories
0708     script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
0709     script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
0710     counter+=1
0711   script+="""
0712         var chart = new google.visualization.BarChart(document.getElementById('%s'));
0713         chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
0714                           vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
0715                          });
0716       }
0717     </script>
0718     """%(name,35*counter,title)
0719   return script
0720
0721
0722 #-------------------------------------------------------------------------------
0723
0724 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
0725   """Create a table, with as rows the directories and as columns the samples.
0726   Each box in the table will contain a pie chart linking to the directory.
0727   """
0728   #aggregation_rules={}
0729   #aggregation_rules_twiki={}
0730
0731   chdir(indir)
0732   if os.path.isabs(indir):
0733       title = basename(indir)
0734   else:
0735       title=indir
0736   title=title.strip(".")
0737   title=title.strip("/")
0738
0739
0740   # Get the list of pickles
0741   sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
0742
0743   # Load directories, build a list of all first level subdirs
0744   dir_unpicklers=[]
0745   n_unpicklers=0
0746   for sample_pkl in sample_pkls:
0747     dir_unpickler=unpickler(sample_pkl)
0748     dir_unpickler.start()
0749     n_unpicklers+=1
0750     dir_unpicklers.append(dir_unpickler)
0751     if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
0752       n_unpicklers=0
0753       for dir_unpickler in dir_unpicklers:
0754         dir_unpickler.join()
0755
0756   dir_dict={}
0757
0758   # create a fake global directory
0759   global_dir=Directory("global","")
0760   for dir_unpickler in dir_unpicklers:
0761     dir_unpickler.join()
0762     directory=dir_unpickler.directory
0763     #directory.prune("Run summary")
0764     #directory.calcStats()
0765     global_dir.meta=directory.meta
0766     dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
0767     global_dir.subdirs.append(directory)
0768
0769   global_dir.calcStats()
0770
0771   directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
0772   categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
0773
0774   page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
0775   rel1=""
0776   rel2=""
0777   try:
0778     rel1,rel2=title.split("VS")
0779   except:
0780     rel1=global_dir.meta.release1.split("-")[0]
0781     rel2=global_dir.meta.release2.split("-")[0]
0782     global_dir.meta.release1=rel1
0783     global_dir.meta.release2=rel2
0784
0785   # union of all subdirs names
0786   all_subdirs=[]
0787   for directory in dir_dict.values():
0788     for subdir_name in directory.get_subdirs_names():
0789       all_subdirs.append(subdir_name)
0790   all_subdirs=sorted(list(set(all_subdirs)))
0791
0792   #print " $$ all_subdirs: %s" %(all_subdirs)
0793
0794   # Get The title
0795   page_html+= '<div class="span-20">'+\
0796               '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
0797               '</div>'+\
0798               '<div class="span-1">'+\
0799               '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
0800               '</div>'+\
0801               '<hr>'
0802   page_html+='<div class="span-24"><p></p></div>\n'*3
0803
0804   # Get The summary section
0805   page_html+= get_summary_section(global_dir,False)
0806
0807   # Make the anchor sections
0808   page_html+= '<div class="span-24">'
0809   page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
0810               '<ul>'+\
0811               '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
0812               '<li><a href="#categories">Categories</a></li>'+\
0813               '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
0814           '<li><a href="#summary_table">Summary Table</a></li>'+\
0815               '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
0816               '<li><a href="#twiki_table">Twiki Table</a></li>'+\
0817               '</ul>'+\
0818               '</div><hr>'
0819
0820
0821   # Make the CategoriesBar chart
0822   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
0823   page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
0824
0825   # Make the gauges per categories
0826   page_html+=make_categories_summary(dir_dict,aggregation_rules)
0827
0828   # Make the Directories chart
0829   page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
0830
0831   # Barbarian vertical space. Suggestions are welcome
0832   for i in range(2):
0833     page_html+='<div class="span-24"><p></p></div>\n'
0834
0835
0836   # Prepare the table
0837   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'
0838
0839   div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
0840   page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
0841   page_html+="""
0842         <table>
0843         <thead>
0844           <tr>
0845           <th> <p class = "vertical_head center_head"></p> </th>
0846   """
0847
0848   # First row with samples
0849   page_html+="""
0850           <th> <p class = "vertical_head center_head">Summary</p></th>"""
0851
0852   sorted_samples=sorted(dir_dict.keys())
0853   for sample in sorted_samples:
0854     if "_" in sample and "Data" not in sample:
0855       sample_nick="_".join(sample.split("X_")[0].split("_")[:-1])
0856       # Cleaning for MC: just the fragment
0857     elif "Data" in sample and "RelVal" in sample:
0858       sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
0859       # Cleaning for Data: PD + Era + Run
0860     else:
0861       sample_nick = sample
0862
0863     page_html+="""
0864           <th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
0865   page_html+="</tr> \n </thead> \n </tbody> \n"
0866
0867
0868  # FIRST ROW
0869  # Now the summaries  at the beginning of the table
0870   page_html+="<tr>"
0871   page_html+='<td class="sticky-col">'
0872
0873   page_html+='<b>Summary</b></td>'
0874   page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
0875   for sample in sorted_samples:
0876     col=dir_dict[sample]
0877     # check if the directory was a top one or not
0878     summary_page_name="RelMonSummary.html"
0879     if col.name!="":
0880       summary_page_name=hash_name(col.name, hashing_flag)+".html"
0881     title = get_pie_tooltip(col)
0882     chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
0883     page_html+='<td>'
0884     page_html+='%s </a></td>' %(chart)
0885   page_html+="</tr>"
0886
0887   # Now the content
0888   for subdir_name in all_subdirs:
0889
0890     page_html+='          <tr>\n'
0891     page_html+='          <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name
0892
0893     row_summary=Directory("row_summary","")
0894     sample_counter=0
0895     n_samples=len(sorted_samples)
0896
0897     for sample in sorted_samples:
0898       subdirs_dict=directory.get_subdirs_dict()
0899       directory=dir_dict[sample]
0900       dir_is_there=subdir_name in subdirs_dict
0901       if dir_is_there:
0902         row_summary.subdirs.append(subdirs_dict[subdir_name])
0903
0904     # one first row for the summary!
0905     row_summary.calcStats()
0906     title = get_pie_tooltip(col)
0907     chart = row_summary.get_piechart_js(50)
0908     page_html+='<td><div>'#<div class="span-1">'
0909     page_html+= chart + '</div></td>'
0910
0911     for sample in sorted_samples:
0912       sample_counter+=1
0913
0914       directory=dir_dict[sample]
0915       subdirs_dict=directory.get_subdirs_dict()
0916
0917       # Check if the directory is the top one
0918       if directory.name!="":
0919         # We did not run on the topdir
0920         #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
0921         #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
0922         if hashing_flag:
0923           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
0924         else:
0925           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
0926       else:
0927         #print "   ## summary_page: %s"%(directory.name+subdir_name)
0928         #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
0929         summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
0930       dir_is_there=subdir_name in subdirs_dict
0931
0932       img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
0933       page_html+='<td><div class="span-1">'
0934
0935       if dir_is_there:
0936         #row_summary.subdirs.append(subdirs_dict[subdir_name])
0937         chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
0938         page_html+='%s'%chart
0939       else:
0940         page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
0941       page_html+='</div></td>'
0942
0943     page_html+="          </tr>\n"
0944
0945
0946
0947   page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'
0948
0949   page_html+=get_rank_section(global_dir)
0950
0951   #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
0952   # ^ commenting out for the moment, not really useful nor used
0953
0954   page_html+=get_page_footer()
0955   return page_html
0956
0957
0958 #-----------UPDATES------
0959 def hash_name(file_name, flag):
0960     if flag: #if hashing flag is ON then return
0961         if (3,0,0) <= sys.version_info:
0962             return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
0963         return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
0964     else:
0965         return file_name #return standart name