Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2023-11-29 01:33:17

0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 ################################################################################
0004 # RelMon: a tool for automatic Release Comparison                              
0005 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0006 #
0007 
0008 #
0009 #                                                                              
0010 # Danilo Piparo CERN - danilo.piparo@cern.ch                                   
0011 #                                                                              
0012 ################################################################################
0013 
0014 from builtins import range
0015 from os import chdir,getcwd,listdir,makedirs
0016 from os.path import basename,join,exists
0017 import html
0018 
0019 import sys
0020 theargv=sys.argv
0021 sys.argv=[]
0022 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
0023 sys.argv=theargv
0024 
0025 import os
0026 import hashlib
0027 
0028 if "RELMON_SA" in os.environ:
0029   from .dirstructure import Comparison,Directory
0030   from .definitions import *
0031   from .utils import unpickler
0032 else:
0033   from Utilities.RelMon.dirstructure import Comparison,Directory
0034   from Utilities.RelMon.definitions import *
0035   from Utilities.RelMon.utils import unpickler
0036   
0037 #-------------------------------------------------------------------------------
0038 
0039 def encode_obj_url(url):
0040   for old,new in url_encode_dict.items():
0041     url=url.replace(old,new)
0042   return url
0043 
0044 def plot_size(h=250,w=200):
0045   return "w=%s;h=%s" %(h,w)
0046 
0047 def build_obj_addr(run,sample,version,plot_path,tier):
0048   slash="/"
0049   obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
0050   obj_url+=plot_path
0051   while obj_url.endswith(slash):
0052     obj_url=obj_url[:-1]  
0053   while slash*2 in obj_url:
0054     obj_url=obj_url.replace(slash*2,slash)
0055   return obj_url
0056   
0057 def build_obj(run,sample,version,plot_path,tier):
0058   obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
0059   return encode_obj_url(obj_url)
0060 
0061 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
0062   fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
0063   fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
0064   fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
0065   if len(draw_opts)>0:
0066     fairy_url+="drawopts=%s;" %draw_opts
0067   fairy_url+= plot_size(h,w)
0068   fairy_url += ";ref=ratiooverlay"
0069   return fairy_url
0070 
0071 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
0072   fairy_url = "%s/%s/plotfairy/" %(server,base_url)
0073   fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
0074   fairy_url+= "?%s"%plot_size(h,w)  
0075   if len(draw_opts)>0:
0076     fairy_url+="drawopts=%s;" %draw_opts  
0077   return fairy_url
0078  
0079 #-------------------------------------------------------------------------------
0080 style_location="/cms-service-reldqm"
0081 def get_page_header(directory=None, standalone=False, additional_header=""):
0082   style_location="/cms-service-reldqm"
0083   if standalone:
0084     style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
0085   javascripts=''
0086   style=''
0087   if directory!=None and len(directory.comparisons)>0:
0088     meta=directory.meta
0089     style='img.fail {border:1px solid #ff0000;}\n'+\
0090           'img.succes {border:1px solid #00ff00;}\n'+\
0091           'img.null {border:1px solid #ffff00;}\n'+\
0092           'img.skiped {border:1px solid #7a7a7a;}\n'+\
0093           'a.black_link:link {color: #333333}\n'+\
0094           'a.black_link:hover {color: #737373}\n'+\
0095           'a.black_link:visited {color: #333333}\n'+\
0096           'a.black_link:active {color: #333333}\n'
0097   javascripts=""
0098   
0099   
0100   html='<html>'+\
0101        '<head>'+\
0102        '<title>RelMon Summary</title>'+\
0103        '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
0104        '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
0105        '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
0106        '<style type="text/css">'+\
0107        '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
0108        '%s'%style+\
0109        '</style>'+\
0110        '%s'%javascripts+\
0111        '%s'%additional_header+\
0112        '</head>'+\
0113        '<body>'+\
0114        '<div class="container">'
0115   
0116   return html
0117   
0118 #-------------------------------------------------------------------------------
0119 
0120 def get_page_footer():
0121     return '</div></body></html>'
0122 
0123 #-------------------------------------------------------------------------------
0124 
0125 def get_title_section(directory, hashing_flag, standalone, depth=2):
0126   if standalone:
0127       cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
0128   else:
0129       cms_logo_url = "cms-service-reldqm/style/CMS.gif"
0130   mother_name=basename(directory.mother_dir)
0131   mother_file_name=""
0132   if depth==1:
0133     mother_file_name="../RelMonSummary.html"
0134     if mother_name!="":
0135       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0136   elif depth==2:
0137     mother_file_name="RelMonSummary.html"
0138     #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
0139     if mother_name!="":
0140       mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0141   else:
0142       if hashing_flag:
0143           files = directory.mother_dir.split("/")
0144           if len(files) != 1:
0145               dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
0146           else:
0147               dir_name = files[-1]
0148           dir_name = directory.mother_dir
0149           mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
0150       else:
0151           mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
0152           mother_file_name=mother_file_name.strip("_")
0153       
0154   link_to_mother='<a href="%s">..</a>' %mother_file_name
0155   html= '<div class="span-20">'+\
0156         '<h1>%s</h1>'%directory.name+\
0157         '</div>'+\
0158         '<div class="span-1">'+\
0159         '<h1>%s</h1>'%link_to_mother+\
0160         '</div>'+\
0161         '<div class="span-3 last">'+\
0162         '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
0163         '</div>'+\
0164         '<hr>' 
0165   if len(mother_name)>0:
0166     html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
0167           '<hr>' 
0168   return html
0169  
0170 #-------------------------------------------------------------------------------
0171 def get_dir_stats(directory):
0172   html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
0173   html+='<ul>'
0174   if directory.n_successes>0:
0175     html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
0176   if directory.n_nulls>0:
0177     html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
0178   if directory.n_fails>0:
0179     html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
0180   if directory.n_skiped>0:
0181     html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
0182   if directory.n_missing_objs>0:
0183     html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
0184   html+='</ul>'
0185   return html
0186 
0187 #-------------------------------------------------------------------------------
0188 
0189 def get_subdirs_section(directory, hashing_flag): 
0190   if len(directory.subdirs)==0:
0191     return ""
0192   html= '<div class="span-20 colborder">'
0193   html+='<h2 class="alt">Sub-Directories</h2>'
0194   # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
0195   # so reverse :)
0196   sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
0197   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
0198   sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
0199   for subdir in sorted_subdirs:
0200     name=subdir.name
0201     if hashing_flag:
0202         link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
0203     else:
0204         link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
0205         link=link.strip("_")
0206     html+='<div class="span-4 prepend-2 colborder">'
0207     html+='<h3>%s</h3>'%name
0208     html+='</div>'
0209     
0210     html+='<div class="span-7">'
0211     html+=get_dir_stats(subdir)
0212     html+='</div>'
0213     
0214     html+='<div class="span-6 last">'
0215     html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
0216     html+='</div>'
0217     
0218     html+='<hr>'
0219   return html+'</div>'
0220 
0221  
0222 #-------------------------------------------------------------------------------
0223 
0224 def get_summary_section(directory,matrix_page=True):
0225   
0226   # Hack find the first comparison and fill test and threshold
0227   # shall this be put in meta?
0228   test_name=""
0229   test_threshold=0
0230   for comparison in directory.comparisons:
0231       test_name=comparison.test_name
0232       test_threshold=comparison.test_thr
0233       break
0234   if len(test_name)==0:
0235     for subdir in directory.subdirs:  
0236       for comparison in subdir.comparisons:
0237         test_name=comparison.test_name
0238         test_threshold=comparison.test_thr
0239         break
0240       if len(test_name)!=0:break      
0241       if len(test_name)==0:  
0242         for subsubdir in subdir.subdirs:          
0243           for comparison in subsubdir.comparisons:
0244             test_name=comparison.test_name
0245             test_threshold=comparison.test_thr
0246             break
0247           if len(test_name)!=0:break
0248         if len(test_name)==0:      
0249           for subsubsubdir in subsubdir.subdirs:
0250             for comparison in subsubsubdir.comparisons:
0251               test_name=comparison.test_name
0252               test_threshold=comparison.test_thr
0253               break
0254             if len(test_name)!=0:break
0255 
0256 
0257   meta=directory.meta
0258   
0259   html= '<div class="span-6">'+\
0260         '<h3>Summary</h3>'
0261   html+=get_dir_stats(directory)
0262   html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
0263   html+='</div>'
0264         
0265   html+='<div class="span-7 colborder">'+\
0266         '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
0267         '</div>'+\
0268         '<div class="span-9 last">'
0269   if matrix_page:
0270     html+='<h3>Sample:</h3>'+\
0271           '<p class="caps">%s</p>'%meta.sample1+\
0272           '<h3>Run1 and Run2:</h3>'+\
0273           '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
0274   html+='<h3>Releases:</h3>'+\
0275         '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
0276         '<h3>Statistical Test (Pvalue threshold):</h3>'+\
0277         '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
0278         '</div>'+\
0279         '<hr>'
0280   return html
0281 
0282 #-------------------------------------------------------------------------------
0283 
0284 def get_comparisons(category,directory):
0285   """Prepare the comparisons between histograms and organise them in the page.
0286   Moreover create separate pages with the overlay and the single plots.
0287   """
0288   counter=1
0289   tot_counter=1
0290   
0291   # get the right ones
0292   comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]] 
0293   n_comparisons=len(comparisons)    
0294 
0295   is_reverse=True
0296   if category == FAIL:
0297     is_reverse=False
0298   comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
0299 
0300   
0301   dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
0302   html_comparisons=""
0303   for comparison in comparisons:
0304     plot_name=comparison.img_name
0305     if "http://" not in plot_name:
0306       plot_name= basename(comparison.img_name)
0307     class_type="colborder"    
0308     if counter==3 or tot_counter==n_comparisons:
0309       class_type=" colborder last"
0310     comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
0311 
0312 
0313     if directory.do_pngs:
0314       png_link=comparison.img_name
0315       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0316                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0317                       '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
0318     else:
0319       big_fairy=fairy_url(directory.meta.run1,
0320                         directory.meta.run2,
0321                         directory.meta.sample1,
0322                         directory.meta.sample2,
0323                         directory.meta.release1,
0324                         directory.meta.release2,
0325                         comp_abs_path,
0326                         directory.meta.tier1,
0327                         directory.meta.tier2,
0328                         "",600,600)
0329       small_fairy=fairy_url(directory.meta.run1,
0330                         directory.meta.run2,
0331                         directory.meta.sample1,
0332                         directory.meta.sample2,
0333                         directory.meta.release1,
0334                         directory.meta.release2,
0335                         comp_abs_path,
0336                         directory.meta.tier1,
0337                         directory.meta.tier2)    
0338 
0339       single_fairy1=fairy_url_single(directory.meta.run1,
0340                                    directory.meta.sample1,
0341                                    directory.meta.release1,
0342                                    comp_abs_path,
0343                                    directory.meta.tier1,
0344                                    "",500,500)
0345       single_fairy2=fairy_url_single(directory.meta.run2,
0346                                    directory.meta.sample2,
0347                                    directory.meta.release2,
0348                                    comp_abs_path,
0349                                    directory.meta.tier2,
0350                                    "",500,500)
0351 
0352       html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0353                       '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0354                       '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
0355                       '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
0356                       '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
0357 
0358     if counter==3:                    
0359       html_comparisons+="<hr>"
0360       counter=0
0361     counter+=1
0362     tot_counter+=1
0363 
0364   if len(html_comparisons)!=0:
0365     html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
0366     html+=html_comparisons
0367     html+='<hr>'
0368     return html
0369   return ""
0370 
0371 #-------------------------------------------------------------------------------
0372 
0373 def get_rank_section(directory):
0374 # do Rank histo png
0375     imgname="RankSummary.png"
0376     gStyle.SetPadTickY(0)
0377     c=TCanvas("ranks","ranks",500,400)
0378     #gStyle.SetOptStat(0)
0379     c.cd()
0380 
0381     h=directory.rank_histo
0382     rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
0383     rank_histof.SetLineWidth(2)
0384     for i in range(0,h.GetNbinsX()+1):
0385       rank_histof.SetBinContent(i,h.GetBinContent(i))
0386     h.SetTitle("Ranks Summary;Rank;Frequency")
0387     h.Draw("Hist")
0388     c.Update()
0389     rank_histof.ComputeIntegral()
0390     integral = rank_histof.GetIntegral()
0391     rank_histof.SetContent(integral)
0392 
0393     rightmax = 1.1*rank_histof.GetMaximum()
0394     scale = gPad.GetUymax()/rightmax
0395     rank_histof.SetLineColor(kRed)
0396     rank_histof.Scale(scale)
0397     rank_histof.Draw("same")
0398 
0399     #draw an axis on the right side
0400     axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
0401     axis.SetTitle("Cumulative")
0402     axis.SetTitleColor(kRed)
0403     axis.SetLineColor(kRed)
0404     axis.SetLabelColor(kRed)
0405     axis.Draw()
0406 
0407     rank_histof.Draw("Same");
0408     
0409 
0410     c.Print(imgname)
0411 
0412     page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
0413     page_html+='<div class="span-19"><img src="%s"></div>' %imgname
0414     page_html+='</div> <a href="#top">Top...</a><hr>'
0415 
0416     return page_html
0417     
0418 #-------------------------------------------------------------------------------
0419 
0420 #-------------------------------------------------------------------------------
0421 def get_missing_objs_section(directory):
0422     """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
0423     """
0424     page_html = "Unpaired in %s</br>"%(directory.filename1)
0425     for elem in directory.different_histograms['file1']:
0426         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
0427     page_html +="</br>"
0428     page_html += "Unpaired in %s</br>"%(directory.filename2)
0429     for elem in directory.different_histograms['file2']:
0430         page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
0431     return page_html
0432 #-------------------------------------------------------------------------------
0433 
0434 def directory2html(directory, hashing, standalone, depth=0):
0435   """Converts a directory tree into html pages, very nice ones.
0436   """
0437   #print "d2html: depth", str(depth)," dir ",directory.name
0438   depth+=1
0439   #old_cwd=getcwd()
0440   #if not exists(directory.name) and len(directory.name)>0:
0441     #makedirs(directory.name)
0442   
0443   #if len(directory.name)>0:
0444     #chdir(directory.name)
0445   
0446   for subdir in directory.subdirs:
0447     directory2html(subdir,hashing,standalone, depth)
0448   
0449   page_html=get_page_header(directory, standalone)+\
0450             get_title_section(directory,hashing, standalone, depth)+\
0451             get_summary_section(directory)+\
0452             get_subdirs_section(directory, hashing)
0453 
0454   for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
0455                      (directory.n_comp_nulls >0,NULL ),
0456                      
0457                      (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
0458                      (directory.n_comp_skiped >0,SKIPED)):
0459     if do_cat:
0460       page_html+=get_comparisons(cat,directory)
0461     
0462   if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
0463     page_html += get_missing_objs_section(directory)
0464 
0465   # Distribution of ranks
0466 
0467   if depth==1:
0468     page_html+=get_rank_section(directory)
0469 
0470 
0471   page_html+=get_page_footer()
0472 
0473   page_name=directory.name
0474 
0475   if len(page_name)==0:
0476     page_name="RelMonSummary"
0477   if hashing:
0478       if page_name != "RelMonSummary":
0479           #print " ## oFile path: %s"%(join(directory.full_path))
0480           #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
0481           ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
0482       else:
0483           ofilename = "RelMonSummary.html"
0484   else:
0485       ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
0486       ofilename=ofilename.strip("_")
0487   #print "Writing on %s" %ofilename
0488   ofile=open(ofilename,"w")
0489   ofile.write(page_html)
0490   ofile.close()
0491 
0492   #chdir(old_cwd)
0493 
0494 #-------------------------------------------------------------------------------
0495 
0496 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
0497   total_success_rate_scaled=(total_success_rate-minrate)
0498   total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
0499   if total_success_rate_scaled_repr<0:
0500     total_success_rate_scaled_repr=0
0501   size_s="200x100"
0502   if small:
0503     size_s="40x30"
0504   #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
0505   gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
0506   gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
0507   if not small:
0508     gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
0509     gauge_link+="&chma=10,10,10,0"
0510   img_tag= '<img src="%s">'%gauge_link
0511   if escaped:
0512     img_tag=html.escape(img_tag)    
0513   return img_tag
0514 
0515 #-------------------------------------------------------------------------------
0516 
0517 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
0518   # Let's make a summary for an overview in categories act on the global dir
0519   aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
0520 
0521   list_of_names=[]
0522   if the_aggr_pairs==[]:
0523     for samplename,sampledir in dir_dict.items():
0524       for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
0525         if not subsysdirname in list_of_names:
0526           list_of_names.append(subsysdirname)
0527           the_aggr_pairs.append((subsysdirname,[subsysdirname]))  
0528           
0529   #print the_aggr_pairs
0530   for cat_name, subdir_list in the_aggr_pairs:
0531     total_successes=0.
0532     total_directory_successes=0
0533     total_weight=0.    
0534     present_subdirs={}
0535     total_ndirs=0
0536     # Loop on samples
0537     for dirname, sampledir in dir_dict.items():
0538       # Loop on directories
0539       for subdirname,subdir in sampledir.get_subdirs_dict().items():        
0540         if subdirname in subdir_list:          
0541           nsucc=subdir.n_successes
0542           total_successes+=nsucc
0543           weight=subdir.weight
0544           total_weight+=weight
0545           total_ndirs+=1
0546           
0547           total_directory_successes+= float(nsucc)/weight
0548           if subdirname in present_subdirs:
0549             this_dir_dict=present_subdirs[subdirname]
0550             this_dir_dict["nsucc"]+=nsucc
0551             this_dir_dict["weight"]+=weight
0552           else:
0553             present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
0554         # Make it usable also for subdirectories
0555         for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():          
0556           for pathname in [name for name in subdir_list if "/" in name]:           
0557             selected_subdirname,selected_subsubdirname = pathname.split("/")
0558             if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
0559               #print "Studying directory ",subsubdirname," in directory ",subdirname
0560               nsucc=subsubdir.n_successes
0561               total_successes+=nsucc
0562               weight=subsubdir.weight
0563               total_weight+=weight
0564               total_ndirs+=1              
0565               total_directory_successes+= float(nsucc)/weight
0566               
0567               if subsubdirname in present_subdirs:
0568                 this_dir_dict=present_subdirs[subsubdirname]
0569                 this_dir_dict["nsucc"]+=nsucc
0570                 this_dir_dict["weight"]+=weight
0571               else:
0572                 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}      
0573 
0574     if total_ndirs == 0:
0575       print("No directory of the category %s is present in the samples: skipping." %cat_name)
0576       continue
0577     
0578     average_success_rate=total_directory_successes/(total_ndirs)
0579     aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
0580     
0581   return aggr_pairs_info
0582 
0583 #-------------------------------------------------------------------------------
0584 
0585 def make_categories_summary(dir_dict,aggregation_rules):
0586     
0587   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0588   
0589   #print aggr_pairs_info
0590   
0591   # Now Let's build the HTML
0592   
0593   html= '<div class="span-20 colborder">'
0594   html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
0595 
0596   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0597     #print cat_name,present_subdirs,total_weight,average_success_rate
0598     html+='<div class="span-3 prepend-0 colborder">'
0599     html+='<h3>%s</h3>'%cat_name
0600     html+='<div><span class="alt">Avg. Success rate:</span></div>'
0601     html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
0602     html+='</div>'    
0603     html+='<div class="span-9">'
0604 
0605     html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
0606     html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
0607     html+='<ul>'    
0608     for subdirname in sorted(present_subdirs.keys()):
0609       this_dir_dict=present_subdirs[subdirname]
0610       nsucc=this_dir_dict["nsucc"]
0611       weight=this_dir_dict["weight"]
0612       html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
0613     html+='</ul>'    
0614     html+='</div>'
0615     
0616     html+='<div class="span-6 last">'
0617     html+=build_gauge(average_success_rate)
0618     html+='</div>'
0619     
0620     html+='<hr>'
0621   return html+'<br><a href="#top">Top...</a> </div><hr>'  
0622     
0623  #-------------------------------------------------------------------------------
0624 
0625 def make_twiki_table(dir_dict,aggregation_rules):
0626   
0627   # decide the release
0628   meta= list(dir_dict.items())[0][1].meta
0629   releases=sorted([meta.release1,meta.release2])
0630   latest_release=releases[1].split("-")[0]
0631   
0632   
0633   aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0634   
0635   # Now Let's build the HTML
0636   
0637   html= '<div class="span-20 colborder">'
0638   html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
0639   html+='<div>| Release | Comparison |'
0640   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0641     html+=cat_name
0642     html+=" | "
0643   html+='</div>'
0644   
0645   html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release
0646 
0647   # Now add all the line with small gauges
0648 
0649   for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0650     #print cat_name,present_subdirs,total_weight,average_success_rate
0651     html+=build_gauge(average_success_rate,small=True,escaped=True)
0652     html+=" | "    
0653   
0654   html+='</div> <a href="#top">Top...</a>'
0655   html+='<hr>'
0656   return html+'</div>'
0657   
0658 #-------------------------------------------------------------------------------
0659 
0660 def get_pie_tooltip(directory):
0661   tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
0662   return tooltip
0663 
0664 #-------------------------------------------------------------------------------
0665 
0666 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):  
0667   
0668   aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)       
0669 
0670   script="""
0671     <script type="text/javascript" src="https://www.google.com/jsapi"></script>
0672     <script type="text/javascript">
0673       google.load("visualization", "1", {packages:["corechart"]});
0674       google.setOnLoadCallback(drawChart);
0675       function drawChart() {
0676         var data = new google.visualization.DataTable();
0677         data.addColumn('string', 'DQM Directory');
0678         data.addColumn('number', 'Success Rate');
0679         """
0680   script+="data.addRows(%i);\n"%len(aggr_pairs_info)
0681   counter=0
0682   for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
0683     #print subsystname,present_directories
0684     script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
0685     script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
0686     counter+=1
0687   script+="""
0688         var chart = new google.visualization.BarChart(document.getElementById('%s'));
0689         chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
0690                           vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
0691                          });
0692       }
0693     </script>
0694     """%(name,40*counter,title)
0695   return script
0696 
0697 
0698 #-------------------------------------------------------------------------------
0699 
0700 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
0701   """Create a table, with as rows the directories and as columns the samples.
0702   Each box in the table will contain a pie chart linking to the directory.
0703   """  
0704   #aggregation_rules={}
0705   #aggregation_rules_twiki={}
0706 
0707   chdir(indir)
0708   if os.path.isabs(indir):
0709       title = basename(indir)
0710   else:
0711       title=indir
0712   title=title.strip(".")
0713   title=title.strip("/")
0714   
0715   
0716   # Get the list of pickles
0717   sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
0718   
0719   # Load directories, build a list of all first level subdirs  
0720   dir_unpicklers=[]
0721   n_unpicklers=0
0722   for sample_pkl in sample_pkls:
0723     dir_unpickler=unpickler(sample_pkl)
0724     dir_unpickler.start()
0725     n_unpicklers+=1
0726     dir_unpicklers.append(dir_unpickler)
0727     if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
0728       n_unpicklers=0
0729       for dir_unpickler in dir_unpicklers:
0730         dir_unpickler.join()
0731   
0732   dir_dict={}
0733   
0734   # create a fake global directory
0735   global_dir=Directory("global","")  
0736   for dir_unpickler in dir_unpicklers:
0737     dir_unpickler.join()
0738     directory=dir_unpickler.directory
0739     #directory.prune("Run summary")    
0740     #directory.calcStats()
0741     global_dir.meta=directory.meta
0742     dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
0743     global_dir.subdirs.append(directory)
0744   
0745   global_dir.calcStats()
0746   
0747   directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
0748   categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
0749   
0750   page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
0751   rel1=""
0752   rel2=""
0753   try:
0754     rel1,rel2=title.split("VS")
0755   except:
0756     rel1=global_dir.meta.release1.split("-")[0]
0757     rel2=global_dir.meta.release2.split("-")[0] 
0758     global_dir.meta.release1=rel1
0759     global_dir.meta.release2=rel2
0760     
0761   # union of all subdirs names
0762   all_subdirs=[]
0763   for directory in dir_dict.values():
0764     for subdir_name in directory.get_subdirs_names():
0765       all_subdirs.append(subdir_name)
0766   all_subdirs=sorted(list(set(all_subdirs)))
0767   
0768   #print " $$ all_subdirs: %s" %(all_subdirs)
0769   
0770   # Get The title
0771   page_html+= '<div class="span-20">'+\
0772               '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
0773               '</div>'+\
0774               '<div class="span-1">'+\
0775               '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
0776               '</div>'+\
0777               '<hr>'
0778   page_html+='<div class="span-24"><p></p></div>\n'*3
0779   
0780   # Get The summary section
0781   page_html+= get_summary_section(global_dir,False)  
0782 
0783   # Make the anchor sections
0784   page_html+= '<div class="span-24">'
0785   page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
0786               '<ul>'+\
0787               '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
0788               '<li><a href="#categories">Categories</a></li>'+\
0789               '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
0790           '<li><a href="#summary_table">Summary Table</a></li>'+\
0791               '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
0792               '<li><a href="#twiki_table">Twiki Table</a></li>'+\
0793               '</ul>'+\
0794               '</div><hr>'
0795 
0796 
0797   # Make the CategoriesBar chart
0798   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
0799   page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
0800 
0801   # Make the gauges per categories
0802   page_html+=make_categories_summary(dir_dict,aggregation_rules)
0803 
0804   # Make the Directories chart
0805   page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
0806   page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
0807   
0808   # Barbarian vertical space. Suggestions are welcome
0809   for i in range(2):
0810     page_html+='<div class="span-24"><p></p></div>\n'
0811 
0812  
0813   # Prepare the table
0814   page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
0815 
0816   for i in range(5):
0817     page_html+='<div class="span-24"><p></p></div>\n'
0818     
0819   page_html+="""
0820         <table border="1" >
0821           <tr>
0822           <td> </td>          
0823   """
0824   
0825   # First row with samples
0826   page_html+="""
0827           <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
0828 
0829   sorted_samples=sorted(dir_dict.keys())
0830   for sample in sorted_samples:
0831     sample_nick=sample
0832     ## For runs: put only the number after the _
0833     #if "_" in sample:
0834       #run_number=sample.split("_")[-1]      
0835       #if (not run_number.isalpha()) and len(run_number)>=6:
0836     #sample_nick=run_number
0837       
0838       
0839     page_html+="""
0840           <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
0841   page_html+="          </tr>\n"
0842 
0843 
0844  # FIRST ROW
0845  # Now the summaries  at the beginning of the table
0846   page_html+="<tr>"
0847   page_html+='<td  style="background-color:white;"><div class="span-1">'
0848   
0849   page_html+='<b>Summary</b></div></td>'
0850   page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
0851   for sample in sorted_samples:
0852     col=dir_dict[sample]
0853     # check if the directory was a top one or not
0854     summary_page_name="RelMonSummary.html"
0855     if col.name!="":
0856       summary_page_name=hash_name(col.name, hashing_flag)+".html"
0857     img_link=col.get_summary_chart_ajax(55,55)
0858     page_html+='<td  style="background-color:white;"><div class="span-1">'
0859     page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
0860   page_html+="</tr>"
0861 
0862   # Now the content
0863   for subdir_name in all_subdirs:  
0864 
0865     page_html+='          <tr>\n'
0866     page_html+='          <td style="background-color:white;">%s</td>\n' %subdir_name  
0867 
0868     row_summary=Directory("row_summary","")
0869     sample_counter=0
0870     n_samples=len(sorted_samples)    
0871 
0872     for sample in sorted_samples:
0873       subdirs_dict=directory.get_subdirs_dict()
0874       directory=dir_dict[sample]
0875       dir_is_there=subdir_name in subdirs_dict
0876       if dir_is_there:
0877         row_summary.subdirs.append(subdirs_dict[subdir_name])
0878 
0879     # one first row for the summary!
0880     row_summary.calcStats()
0881     img_link=row_summary.get_summary_chart_ajax(55,55)
0882     page_html+='<td  style="background-color:white;"><div class="span-1">'
0883     page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
0884 
0885     for sample in sorted_samples:
0886       sample_counter+=1      
0887 
0888       directory=dir_dict[sample]
0889       subdirs_dict=directory.get_subdirs_dict()
0890 
0891       # Check if the directory is the top one
0892       if directory.name!="":
0893         # We did not run on the topdir
0894         #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
0895         #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
0896         if hashing_flag:
0897           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
0898         else:
0899           summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
0900       else:
0901         #print "   ## summary_page: %s"%(directory.name+subdir_name)
0902         #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
0903         summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
0904       dir_is_there=subdir_name in subdirs_dict
0905 
0906       img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
0907       img_tooltip="N/A"
0908       if dir_is_there:
0909         #row_summary.subdirs.append(subdirs_dict[subdir_name])
0910         img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
0911         img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
0912 
0913       page_html+='<td  style="background-color:white;"><div class="span-1">'
0914       if dir_is_there:
0915         page_html+='<a href="%s">'%(summary_page)
0916       page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
0917       if dir_is_there:
0918         page_html+='</a>'
0919       page_html+='</div></td>' 
0920 
0921     page_html+="          </tr>\n"        
0922 
0923 
0924 
0925   page_html+='</table> <a href="#top">Top...</a><hr>'
0926 
0927   page_html+=get_rank_section(global_dir)
0928 
0929   page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
0930 
0931   page_html+=get_page_footer()
0932   return page_html  
0933 
0934 
0935 #-----------UPDATES------
0936 def hash_name(file_name, flag):
0937     if flag: #if hashing flag is ON then return
0938         if (3,0,0) <= sys.version_info:
0939             return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
0940         return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
0941     else:
0942         return file_name #return standart name