RelMon/python/directories2html.py

0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 ################################################################################
0004 # RelMon: a tool for automatic Release Comparison
0005 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0006 #
0007
0008 #
0009 #
0010 # Danilo Piparo CERN - danilo.piparo@cern.ch
0011 #
0012 ################################################################################
0013
0014 from builtins import range
0015 from os import chdir,getcwd,listdir,makedirs
0016 from os.path import basename,join,exists
0017 import html
0018
0019 import sys
0020 theargv=sys.argv
0021 sys.argv=[]
0022 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
0023 sys.argv=theargv
0024
0025 import os
0026 if "RELMON_SA" in os.environ:
0027   from .dirstructure import Comparison,Directory
0028   from .definitions import *
0029   from .utils import unpickler
0030 else:
0031   from Utilities.RelMon.dirstructure import Comparison,Directory
0032   from Utilities.RelMon.definitions import *
0033   from Utilities.RelMon.utils import unpickler
0034
0035   import hashlib
0036 #-------------------------------------------------------------------------------
0037
0038 def encode_obj_url(url):
0039   for old,new in url_encode_dict.items():
0040     url=url.replace(old,new)
0041   return url
0042
0043 def plot_size(h=250,w=200):
0044   return "w=%s;h=%s" %(h,w)
0045
0046 def build_obj_addr(run,sample,version,plot_path,tier):
0047   slash="/"
0048   obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
0049   obj_url+=plot_path
0050   while obj_url.endswith(slash):
0051     obj_url=obj_url[:-1]
0052   while slash*2 in obj_url:
0053     obj_url=obj_url.replace(slash*2,slash)
0054   return obj_url
0055
0056 def build_obj(run,sample,version,plot_path,tier):
0057   obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
0058   return encode_obj_url(obj_url)
0059
0060 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
0061   fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
0062   fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
0063   fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
0064   if len(draw_opts)>0:
0065     fairy_url+="drawopts=%s;" %draw_opts
0066   fairy_url+= plot_size(h,w)
0067   fairy_url += ";ref=ratiooverlay"
0068   return fairy_url
0069
0070 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
0071   fairy_url = "%s/%s/plotfairy/" %(server,base_url)
0072   fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
0073   fairy_url+= "?%s"%plot_size(h,w)
0074   if len(draw_opts)>0:
0075     fairy_url+="drawopts=%s;" %draw_opts
0076   return fairy_url
0077
0078 #-------------------------------------------------------------------------------
0079 style_location="/cms-service-reldqm"
0080 def get_page_header(directory=None, standalone=False, additional_header=""):
0081   style_location="/cms-service-reldqm"
0082   if standalone:
0083     style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
0084   javascripts=''
0085   style=''
0086   if directory!=None and len(directory.comparisons)>0:
0087     meta=directory.meta
0088     style='img.fail {border:1px solid #ff0000;}\n'+\
0089           'img.succes {border:1px solid #00ff00;}\n'+\
0090           'img.null {border:1px solid #ffff00;}\n'+\
0091           'img.skiped {border:1px solid #7a7a7a;}\n'+\
0092           'a.black_link:link {color: #333333}\n'+\
0093           'a.black_link:hover {color: #737373}\n'+\
0094           'a.black_link:visited {color: #333333}\n'+\
0095           'a.black_link:active {color: #333333}\n'
0096   javascripts=""
0097
0098
0099   html='<html>'+\
0100        '<head>'+\
0101        '<title>RelMon Summary</title>'+\
0102        '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
0103        '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
0104        '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
0105        '<style type="text/css">'+\
0106        '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
0107        '%s'%style+\
0108        '</style>'+\
0109        '%s'%javascripts+\
0110        '%s'%additional_header+\
0111        '</head>'+\
0112        '<body>'+\
0113        '<div class="container">'
0114
0115   return html
0116
0117 #-------------------------------------------------------------------------------
0118
0119 def get_page_footer():
0120     return '</div></body></html>'
0121
0122 #-------------------------------------------------------------------------------
0123
0124 def get_title_section(directory, hashing_flag, standalone, depth=2):
0125   if standalone:
0126       cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
0127   else:
0128       cms_logo_url = "cms-service-reldqm/style/CMS.gif"
0129   mother_name=basename(directory.mother_dir)
0130   mother_file_name=""
0131   if depth==1:
0132     mother_file_name="../RelMonSummary.html"
0133     if mother_name!="":
0134       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0135   elif depth==2:
0136     mother_file_name="RelMonSummary.html"
0137     #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
0138     if mother_name!="":
0139       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0140   else:
0141       if hashing_flag:
0142           files = directory.mother_dir.split("/")
0143           if len(files) != 1:
0144               dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
0145           else:
0146               dir_name = files[-1]
0147           dir_name = directory.mother_dir
0148           mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
0149       else:
0150           mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
0151           mother_file_name=mother_file_name.strip("_")
0152
0153   link_to_mother='<a href="%s">..</a>' %mother_file_name
0154   html= '<div class="span-20">'+\
0155         '<h1>%s</h1>'%directory.name+\
0156         '</div>'+\
0157         '<div class="span-1">'+\
0158         '<h1>%s</h1>'%link_to_mother+\
0159         '</div>'+\
0160         '<div class="span-3 last">'+\
0161         '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
0162         '</div>'+\
0163         '<hr>'
0164   if len(mother_name)>0:
0165     html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
0166           '<hr>'
0167   return html
0168
0169 #-------------------------------------------------------------------------------
0170 def get_dir_stats(directory):
0171   html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
0172   html+='<ul>'
0173   if directory.n_successes>0:
0174     html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
0175   if directory.n_nulls>0:
0176     html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
0177   if directory.n_fails>0:
0178     html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
0179   if directory.n_skiped>0:
0180     html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
0181   if directory.n_missing_objs>0:
0182     html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
0183   html+='</ul>'
0184   return html
0185
0186 #-------------------------------------------------------------------------------
0187
0188 def get_subdirs_section(directory, hashing_flag):
0189   if len(directory.subdirs)==0:
0190     return ""
0191   html= '<div class="span-20 colborder">'
0192   html+='<h2 class="alt">Sub-Directories</h2>'
0193   # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
0194   # so reverse :)
0195   sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
0196   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
0197   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
0198   for subdir in sorted_subdirs:
0199     name=subdir.name
0200     if hashing_flag:
0201         link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
0202     else:
0203         link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
0204         link=link.strip("_")
0205     html+='<div class="span-4 prepend-2 colborder">'
0206     html+='<h3>%s</h3>'%name
0207     html+='</div>'
0208
0209     html+='<div class="span-7">'
0210     html+=get_dir_stats(subdir)
0211     html+='</div>'
0212
0213     html+='<div class="span-6 last">'
0214     html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
0215     html+='</div>'
0216
0217     html+='<hr>'
0218   return html+'</div>'
0219
0220
0221 #-------------------------------------------------------------------------------
0222
0223 def get_summary_section(directory,matrix_page=True):
0224
0225   # Hack find the first comparison and fill test and threshold
0226   # shall this be put in meta?
0227   test_name=""
0228   test_threshold=0
0229   for comparison in directory.comparisons:
0230       test_name=comparison.test_name
0231       test_threshold=comparison.test_thr
0232       break
0233   if len(test_name)==0:
0234     for subdir in directory.subdirs:
0235       for comparison in subdir.comparisons:
0236         test_name=comparison.test_name
0237         test_threshold=comparison.test_thr
0238         break
0239       if len(test_name)!=0:break
0240       if len(test_name)==0:
0241         for subsubdir in subdir.subdirs:
0242           for comparison in subsubdir.comparisons:
0243             test_name=comparison.test_name
0244             test_threshold=comparison.test_thr
0245             break
0246           if len(test_name)!=0:break
0247         if len(test_name)==0:
0248           for subsubsubdir in subsubdir.subdirs:
0249             for comparison in subsubsubdir.comparisons:
0250               test_name=comparison.test_name
0251               test_threshold=comparison.test_thr
0252               break
0253             if len(test_name)!=0:break
0254
0255
0256   meta=directory.meta
0257
0258   html= '<div class="span-6">'+\
0259         '<h3>Summary</h3>'
0260   html+=get_dir_stats(directory)
0261   html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
0262   html+='</div>'
0263
0264   html+='<div class="span-7 colborder">'+\
0265         '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
0266         '</div>'+\
0267         '<div class="span-9 last">'
0268   if matrix_page:
0269     html+='<h3>Sample:</h3>'+\
0270           '<p class="caps">%s</p>'%meta.sample1+\
0271           '<h3>Run1 and Run2:</h3>'+\
0272           '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
0273   html+='<h3>Releases:</h3>'+\
0274         '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
0275         '<h3>Statistical Test (Pvalue threshold):</h3>'+\
0276         '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
0277         '</div>'+\
0278         '<hr>'
0279   return html
0280
0281 #-------------------------------------------------------------------------------
0282
0283 def get_comparisons(category,directory):
0284   """Prepare the comparisons between histograms and organise them in the page.
0285   Moreover create separate pages with the overlay and the single plots.
0286   """
0287   counter=1
0288   tot_counter=1
0289
0290   # get the right ones
0291   comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
0292   n_comparisons=len(comparisons)
0293
0294   is_reverse=True
0295   if category == FAIL:
0296     is_reverse=False
0297   comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
0298
0299
0300   dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
0301   html_comparisons=""
0302   for comparison in comparisons:
0303     plot_name=comparison.img_name
0304     if "http://" not in plot_name:
0305       plot_name= basename(comparison.img_name)
0306     class_type="colborder"
0307     if counter==3 or tot_counter==n_comparisons:
0308       class_type=" colborder last"
0309     comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
0310
0311
0312     if directory.do_pngs:
0313       png_link=comparison.img_name
0314       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0315                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0316                       '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
0317     else:
0318       big_fairy=fairy_url(directory.meta.run1,
0319                         directory.meta.run2,
0320                         directory.meta.sample1,
0321                         directory.meta.sample2,
0322                         directory.meta.release1,
0323                         directory.meta.release2,
0324                         comp_abs_path,
0325                         directory.meta.tier1,
0326                         directory.meta.tier2,
0327                         "",600,600)
0328       small_fairy=fairy_url(directory.meta.run1,
0329                         directory.meta.run2,
0330                         directory.meta.sample1,
0331                         directory.meta.sample2,
0332                         directory.meta.release1,
0333                         directory.meta.release2,
0334                         comp_abs_path,
0335                         directory.meta.tier1,
0336                         directory.meta.tier2)
0337
0338       single_fairy1=fairy_url_single(directory.meta.run1,
0339                                    directory.meta.sample1,
0340                                    directory.meta.release1,
0341                                    comp_abs_path,
0342                                    directory.meta.tier1,
0343                                    "",500,500)
0344       single_fairy2=fairy_url_single(directory.meta.run2,
0345                                    directory.meta.sample2,
0346                                    directory.meta.release2,
0347                                    comp_abs_path,
0348                                    directory.meta.tier2,
0349                                    "",500,500)
0350
0351       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0352                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0353                       '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
0354                       '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
0355                       '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
0356
0357     if counter==3:
0358       html_comparisons+="<hr>"
0359       counter=0
0360     counter+=1
0361     tot_counter+=1
0362
0363   if len(html_comparisons)!=0:
0364     html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
0365     html+=html_comparisons
0366     html+='<hr>'
0367     return html
0368   return ""
0369
0370 #-------------------------------------------------------------------------------
0371
0372 def get_rank_section(directory):
0373 # do Rank histo png
0374     imgname="RankSummary.png"
0375     gStyle.SetPadTickY(0)
0376     c=TCanvas("ranks","ranks",500,400)
0377     #gStyle.SetOptStat(0)
0378     c.cd()
0379
0380     h=directory.rank_histo
0381     rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
0382     rank_histof.SetLineWidth(2)
0383     for i in range(0,h.GetNbinsX()+1):
0384       rank_histof.SetBinContent(i,h.GetBinContent(i))
0385     h.SetTitle("Ranks Summary;Rank;Frequency")
0386     h.Draw("Hist")
0387     c.Update()
0388     rank_histof.ComputeIntegral()
0389     integral = rank_histof.GetIntegral()
0390     rank_histof.SetContent(integral)
0391
0392     rightmax = 1.1*rank_histof.GetMaximum()
0393     scale = gPad.GetUymax()/rightmax
0394     rank_histof.SetLineColor(kRed)
0395     rank_histof.Scale(scale)
0396     rank_histof.Draw("same")
0397
0398     #draw an axis on the right side
0399     axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
0400     axis.SetTitle("Cumulative")
0401     axis.SetTitleColor(kRed)
0402     axis.SetLineColor(kRed)
0403     axis.SetLabelColor(kRed)
0404     axis.Draw()
0405
0406     rank_histof.Draw("Same");
0407
0408
0409     c.Print(imgname)
0410
0411     page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
0412     page_html+='<div class="span-19"><img src="%s"></div>' %imgname
0413     page_html+='</div> <a href="#top">Top...</a><hr>'
0414
0415     return page_html
0416
0417 #-------------------------------------------------------------------------------
0418
0419 #-------------------------------------------------------------------------------
0420 def get_missing_objs_section(directory):
0421     """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
0422     """
0423     page_html = "Unpaired in %s</br>"%(directory.filename1)
0424     for elem in directory.different_histograms['file1']:
0425         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
0426     page_html +="</br>"
0427     page_html += "Unpaired in %s</br>"%(directory.filename2)
0428     for elem in directory.different_histograms['file2']:
0429         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
0430     return page_html
0431 #-------------------------------------------------------------------------------
0432
0433 def directory2html(directory, hashing, standalone, depth=0):
0434   """Converts a directory tree into html pages, very nice ones.
0435   """
0436   #print "d2html: depth", str(depth)," dir ",directory.name
0437   depth+=1
0438   #old_cwd=getcwd()
0439   #if not exists(directory.name) and len(directory.name)>0:
0440     #makedirs(directory.name)
0441
0442   #if len(directory.name)>0:
0443     #chdir(directory.name)
0444
0445   for subdir in directory.subdirs:
0446     directory2html(subdir,hashing,standalone, depth)
0447
0448   page_html=get_page_header(directory, standalone)+\
0449             get_title_section(directory,hashing, standalone, depth)+\
0450             get_summary_section(directory)+\
0451             get_subdirs_section(directory, hashing)
0452
0453   for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
0454                      (directory.n_comp_nulls >0,NULL ),
0455
0456                      (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
0457                      (directory.n_comp_skiped >0,SKIPED)):
0458     if do_cat:
0459       page_html+=get_comparisons(cat,directory)
0460
0461   if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
0462     page_html += get_missing_objs_section(directory)
0463
0464   # Distribution of ranks
0465
0466   if depth==1:
0467     page_html+=get_rank_section(directory)
0468
0469
0470   page_html+=get_page_footer()
0471
0472   page_name=directory.name
0473
0474   if len(page_name)==0:
0475     page_name="RelMonSummary"
0476   if hashing:
0477       if page_name != "RelMonSummary":
0478           #print " ## oFile path: %s"%(join(directory.full_path))
0479           #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
0480           ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
0481       else:
0482           ofilename = "RelMonSummary.html"
0483   else:
0484       ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
0485       ofilename=ofilename.strip("_")
0486   #print "Writing on %s" %ofilename
0487   ofile=open(ofilename,"w")
0488   ofile.write(page_html)
0489   ofile.close()
0490
0491   #chdir(old_cwd)
0492
0493 #-------------------------------------------------------------------------------
0494
0495 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
0496   total_success_rate_scaled=(total_success_rate-minrate)
0497   total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
0498   if total_success_rate_scaled_repr<0:
0499     total_success_rate_scaled_repr=0
0500   size_s="200x100"
0501   if small:
0502     size_s="40x30"
0503   #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
0504   gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
0505   gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
0506   if not small:
0507     gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
0508     gauge_link+="&chma=10,10,10,0"
0509   img_tag= '<img src="%s">'%gauge_link
0510   if escaped:
0511     img_tag=html.escape(img_tag)
0512   return img_tag
0513
0514 #-------------------------------------------------------------------------------
0515
0516 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
0517   # Let's make a summary for an overview in categories act on the global dir
0518   aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
0519
0520   list_of_names=[]
0521   if the_aggr_pairs==[]:
0522     for samplename,sampledir in dir_dict.items():
0523       for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
0524         if not subsysdirname in list_of_names:
0525           list_of_names.append(subsysdirname)
0526           the_aggr_pairs.append((subsysdirname,[subsysdirname]))
0527
0528   #print the_aggr_pairs
0529   for cat_name, subdir_list in the_aggr_pairs:
0530     total_successes=0.
0531     total_directory_successes=0
0532     total_weight=0.
0533     present_subdirs={}
0534     total_ndirs=0
0535     # Loop on samples
0536     for dirname, sampledir in dir_dict.items():
0537       # Loop on directories
0538       for subdirname,subdir in sampledir.get_subdirs_dict().items():
0539         if subdirname in subdir_list:
0540           nsucc=subdir.n_successes
0541           total_successes+=nsucc
0542           weight=subdir.weight
0543           total_weight+=weight
0544           total_ndirs+=1
0545
0546           total_directory_successes+= float(nsucc)/weight
0547           if subdirname in present_subdirs:
0548             this_dir_dict=present_subdirs[subdirname]
0549             this_dir_dict["nsucc"]+=nsucc
0550             this_dir_dict["weight"]+=weight
0551           else:
0552             present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
0553         # Make it usable also for subdirectories
0554         for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
0555           for pathname in [name for name in subdir_list if "/" in name]:
0556             selected_subdirname,selected_subsubdirname = pathname.split("/")
0557             if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
0558               #print "Studying directory ",subsubdirname," in directory ",subdirname
0559               nsucc=subsubdir.n_successes
0560               total_successes+=nsucc
0561               weight=subsubdir.weight
0562               total_weight+=weight
0563               total_ndirs+=1
0564               total_directory_successes+= float(nsucc)/weight
0565
0566               if subsubdirname in present_subdirs:
0567                 this_dir_dict=present_subdirs[subsubdirname]
0568                 this_dir_dict["nsucc"]+=nsucc
0569                 this_dir_dict["weight"]+=weight
0570               else:
0571                 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
0572
0573     if total_ndirs == 0:
0574       print("No directory of the category %s is present in the samples: skipping." %cat_name)
0575       continue
0576
0577     average_success_rate=total_directory_successes/(total_ndirs)
0578     aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
0579
0580   return aggr_pairs_info
0581
0582 #-------------------------------------------------------------------------------
0583
0584 def make_categories_summary(dir_dict,aggregation_rules):
0585
0586   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0587
0588   #print aggr_pairs_info
0589
0590   # Now Let's build the HTML
0591
0592   html= '<div class="span-20 colborder">'
0593   html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
0594
0595   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0596     #print cat_name,present_subdirs,total_weight,average_success_rate
0597     html+='<div class="span-3 prepend-0 colborder">'
0598     html+='<h3>%s</h3>'%cat_name
0599     html+='<div><span class="alt">Avg. Success rate:</span></div>'
0600     html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
0601     html+='</div>'
0602     html+='<div class="span-9">'
0603
0604     html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
0605     html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
0606     html+='<ul>'
0607     for subdirname in sorted(present_subdirs.keys()):
0608       this_dir_dict=present_subdirs[subdirname]
0609       nsucc=this_dir_dict["nsucc"]
0610       weight=this_dir_dict["weight"]
0611       html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
0612     html+='</ul>'
0613     html+='</div>'
0614
0615     html+='<div class="span-6 last">'
0616     html+=build_gauge(average_success_rate)
0617     html+='</div>'
0618
0619     html+='<hr>'
0620   return html+'<br><a href="#top">Top...</a> </div><hr>'
0621
0622  #-------------------------------------------------------------------------------
0623
0624 def make_twiki_table(dir_dict,aggregation_rules):
0625
0626   # decide the release
0627   meta= list(dir_dict.items())[0][1].meta
0628   releases=sorted([meta.release1,meta.release2])
0629   latest_release=releases[1].split("-")[0]
0630
0631
0632   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0633
0634   # Now Let's build the HTML
0635
0636   html= '<div class="span-20 colborder">'
0637   html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
0638   html+='<div>| Release | Comparison |'
0639   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0640     html+=cat_name
0641     html+=" | "
0642   html+='</div>'
0643
0644   html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release
0645
0646   # Now add all the line with small gauges
0647
0648   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0649     #print cat_name,present_subdirs,total_weight,average_success_rate
0650     html+=build_gauge(average_success_rate,small=True,escaped=True)
0651     html+=" | "
0652
0653   html+='</div> <a href="#top">Top...</a>'
0654   html+='<hr>'
0655   return html+'</div>'
0656
0657 #-------------------------------------------------------------------------------
0658
0659 def get_pie_tooltip(directory):
0660   tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
0661   return tooltip
0662
0663 #-------------------------------------------------------------------------------
0664
0665 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
0666
0667   aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
0668
0669   script="""
0670     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
0671     <script type="text/javascript">
0672       google.load("visualization", "1", {packages:["corechart"]});
0673       google.setOnLoadCallback(drawChart);
0674       function drawChart() {
0675         var data = new google.visualization.DataTable();
0676         data.addColumn('string', 'DQM Directory');
0677         data.addColumn('number', 'Success Rate');
0678         """
0679   script+="data.addRows(%i);\n"%len(aggr_pairs_info)
0680   counter=0
0681   for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
0682     #print subsystname,present_directories
0683     script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
0684     script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
0685     counter+=1
0686   script+="""
0687         var chart = new google.visualization.BarChart(document.getElementById('%s'));
0688         chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
0689                           vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
0690                          });
0691       }
0692     </script>
0693     """%(name,40*counter,title)
0694   return script
0695
0696
0697 #-------------------------------------------------------------------------------
0698
0699 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
0700   """Create a table, with as rows the directories and as columns the samples.
0701   Each box in the table will contain a pie chart linking to the directory.
0702   """
0703   #aggregation_rules={}
0704   #aggregation_rules_twiki={}
0705
0706   chdir(indir)
0707   if os.path.isabs(indir):
0708       title = basename(indir)
0709   else:
0710       title=indir
0711   title=title.strip(".")
0712   title=title.strip("/")
0713
0714
0715   # Get the list of pickles
0716   sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
0717
0718   # Load directories, build a list of all first level subdirs
0719   dir_unpicklers=[]
0720   n_unpicklers=0
0721   for sample_pkl in sample_pkls:
0722     dir_unpickler=unpickler(sample_pkl)
0723     dir_unpickler.start()
0724     n_unpicklers+=1
0725     dir_unpicklers.append(dir_unpickler)
0726     if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
0727       n_unpicklers=0
0728       for dir_unpickler in dir_unpicklers:
0729         dir_unpickler.join()
0730
0731   dir_dict={}
0732
0733   # create a fake global directory
0734   global_dir=Directory("global","")
0735   for dir_unpickler in dir_unpicklers:
0736     dir_unpickler.join()
0737     directory=dir_unpickler.directory
0738     #directory.prune("Run summary")
0739     #directory.calcStats()
0740     global_dir.meta=directory.meta
0741     dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
0742     global_dir.subdirs.append(directory)
0743
0744   global_dir.calcStats()
0745
0746   directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
0747   categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
0748
0749   page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
0750   rel1=""
0751   rel2=""
0752   try:
0753     rel1,rel2=title.split("VS")
0754   except:
0755     rel1=global_dir.meta.release1.split("-")[0]
0756     rel2=global_dir.meta.release2.split("-")[0]
0757     global_dir.meta.release1=rel1
0758     global_dir.meta.release2=rel2
0759
0760   # union of all subdirs names
0761   all_subdirs=[]
0762   for directory in dir_dict.values():
0763     for subdir_name in directory.get_subdirs_names():
0764       all_subdirs.append(subdir_name)
0765   all_subdirs=sorted(list(set(all_subdirs)))
0766
0767   #print " $$ all_subdirs: %s" %(all_subdirs)
0768
0769   # Get The title
0770   page_html+= '<div class="span-20">'+\
0771               '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
0772               '</div>'+\
0773               '<div class="span-1">'+\
0774               '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
0775               '</div>'+\
0776               '<hr>'
0777   page_html+='<div class="span-24"><p></p></div>\n'*3
0778
0779   # Get The summary section
0780   page_html+= get_summary_section(global_dir,False)
0781
0782   # Make the anchor sections
0783   page_html+= '<div class="span-24">'
0784   page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
0785               '<ul>'+\
0786               '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
0787               '<li><a href="#categories">Categories</a></li>'+\
0788               '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
0789           '<li><a href="#summary_table">Summary Table</a></li>'+\
0790               '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
0791               '<li><a href="#twiki_table">Twiki Table</a></li>'+\
0792               '</ul>'+\
0793               '</div><hr>'
0794
0795
0796   # Make the CategoriesBar chart
0797   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
0798   page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
0799
0800   # Make the gauges per categories
0801   page_html+=make_categories_summary(dir_dict,aggregation_rules)
0802
0803   # Make the Directories chart
0804   page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
0805   page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
0806
0807   # Barbarian vertical space. Suggestions are welcome
0808   for i in range(2):
0809     page_html+='<div class="span-24"><p></p></div>\n'
0810
0811
0812   # Prepare the table
0813   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
0814
0815   for i in range(5):
0816     page_html+='<div class="span-24"><p></p></div>\n'
0817
0818   page_html+="""
0819         <table border="1" >
0820           <tr>
0821           <td> </td>
0822   """
0823
0824   # First row with samples
0825   page_html+="""
0826           <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
0827
0828   sorted_samples=sorted(dir_dict.keys())
0829   for sample in sorted_samples:
0830     sample_nick=sample
0831     ## For runs: put only the number after the _
0832     #if "_" in sample:
0833       #run_number=sample.split("_")[-1]
0834       #if (not run_number.isalpha()) and len(run_number)>=6:
0835     #sample_nick=run_number
0836
0837
0838     page_html+="""
0839           <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
0840   page_html+="          </tr>\n"
0841
0842
0843  # FIRST ROW
0844  # Now the summaries  at the beginning of the table
0845   page_html+="<tr>"
0846   page_html+='<td  style="background-color:white;"><div class="span-1">'
0847
0848   page_html+='<b>Summary</b></div></td>'
0849   page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
0850   for sample in sorted_samples:
0851     col=dir_dict[sample]
0852     # check if the directory was a top one or not
0853     summary_page_name="RelMonSummary.html"
0854     if col.name!="":
0855       summary_page_name=hash_name(col.name, hashing_flag)+".html"
0856     img_link=col.get_summary_chart_ajax(55,55)
0857     page_html+='<td  style="background-color:white;"><div class="span-1">'
0858     page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
0859   page_html+="</tr>"
0860
0861   # Now the content
0862   for subdir_name in all_subdirs:
0863
0864     page_html+='          <tr>\n'
0865     page_html+='          <td style="background-color:white;">%s</td>\n' %subdir_name
0866
0867     row_summary=Directory("row_summary","")
0868     sample_counter=0
0869     n_samples=len(sorted_samples)
0870
0871     for sample in sorted_samples:
0872       subdirs_dict=directory.get_subdirs_dict()
0873       directory=dir_dict[sample]
0874       dir_is_there=subdir_name in subdirs_dict
0875       if dir_is_there:
0876         row_summary.subdirs.append(subdirs_dict[subdir_name])
0877
0878     # one first row for the summary!
0879     row_summary.calcStats()
0880     img_link=row_summary.get_summary_chart_ajax(55,55)
0881     page_html+='<td  style="background-color:white;"><div class="span-1">'
0882     page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
0883
0884     for sample in sorted_samples:
0885       sample_counter+=1
0886
0887       directory=dir_dict[sample]
0888       subdirs_dict=directory.get_subdirs_dict()
0889
0890       # Check if the directory is the top one
0891       if directory.name!="":
0892         # We did not run on the topdir
0893         #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
0894         #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
0895         if hashing_flag:
0896           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
0897         else:
0898           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
0899       else:
0900         #print "   ## summary_page: %s"%(directory.name+subdir_name)
0901         #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
0902         summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
0903       dir_is_there=subdir_name in subdirs_dict
0904
0905       img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
0906       img_tooltip="N/A"
0907       if dir_is_there:
0908         #row_summary.subdirs.append(subdirs_dict[subdir_name])
0909         img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
0910         img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
0911
0912       page_html+='<td  style="background-color:white;"><div class="span-1">'
0913       if dir_is_there:
0914         page_html+='<a href="%s">'%(summary_page)
0915       page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
0916       if dir_is_there:
0917         page_html+='</a>'
0918       page_html+='</div></td>'
0919
0920     page_html+="          </tr>\n"
0921
0922
0923
0924   page_html+='</table> <a href="#top">Top...</a><hr>'
0925
0926   page_html+=get_rank_section(global_dir)
0927
0928   page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
0929
0930   page_html+=get_page_footer()
0931   return page_html
0932
0933
0934 #-----------UPDATES------
0935 def hash_name(file_name, flag):
0936     #print "     HashFILE name: "+file_name
0937     if flag: #if hashing flag is ON then return
0938         return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
0939     else:
0940         return file_name #return standart name