File indexing completed on 2021-11-23 04:29:22
0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013
0014 from builtins import range
0015 from os import chdir,getcwd,listdir,makedirs
0016 from os.path import basename,join,exists
0017 import html
0018
0019 import sys
0020 theargv=sys.argv
0021 sys.argv=[]
0022 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
0023 sys.argv=theargv
0024
0025 import os
0026 if "RELMON_SA" in os.environ:
0027 from .dirstructure import Comparison,Directory
0028 from .definitions import *
0029 from .utils import unpickler
0030 else:
0031 from Utilities.RelMon.dirstructure import Comparison,Directory
0032 from Utilities.RelMon.definitions import *
0033 from Utilities.RelMon.utils import unpickler
0034
0035 import hashlib
0036
0037
0038 def encode_obj_url(url):
0039 for old,new in url_encode_dict.items():
0040 url=url.replace(old,new)
0041 return url
0042
0043 def plot_size(h=250,w=200):
0044 return "w=%s;h=%s" %(h,w)
0045
0046 def build_obj_addr(run,sample,version,plot_path,tier):
0047 slash="/"
0048 obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
0049 obj_url+=plot_path
0050 while obj_url.endswith(slash):
0051 obj_url=obj_url[:-1]
0052 while slash*2 in obj_url:
0053 obj_url=obj_url.replace(slash*2,slash)
0054 return obj_url
0055
0056 def build_obj(run,sample,version,plot_path,tier):
0057 obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
0058 return encode_obj_url(obj_url)
0059
0060 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
0061 fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
0062 fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
0063 fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
0064 if len(draw_opts)>0:
0065 fairy_url+="drawopts=%s;" %draw_opts
0066 fairy_url+= plot_size(h,w)
0067 fairy_url += ";ref=ratiooverlay"
0068 return fairy_url
0069
0070 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
0071 fairy_url = "%s/%s/plotfairy/" %(server,base_url)
0072 fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
0073 fairy_url+= "?%s"%plot_size(h,w)
0074 if len(draw_opts)>0:
0075 fairy_url+="drawopts=%s;" %draw_opts
0076 return fairy_url
0077
0078
0079 style_location="/cms-service-reldqm"
0080 def get_page_header(directory=None, standalone=False, additional_header=""):
0081 style_location="/cms-service-reldqm"
0082 if standalone:
0083 style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
0084 javascripts=''
0085 style=''
0086 if directory!=None and len(directory.comparisons)>0:
0087 meta=directory.meta
0088 style='img.fail {border:1px solid #ff0000;}\n'+\
0089 'img.succes {border:1px solid #00ff00;}\n'+\
0090 'img.null {border:1px solid #ffff00;}\n'+\
0091 'img.skiped {border:1px solid #7a7a7a;}\n'+\
0092 'a.black_link:link {color: #333333}\n'+\
0093 'a.black_link:hover {color: #737373}\n'+\
0094 'a.black_link:visited {color: #333333}\n'+\
0095 'a.black_link:active {color: #333333}\n'
0096 javascripts=""
0097
0098
0099 html='<html>'+\
0100 '<head>'+\
0101 '<title>RelMon Summary</title>'+\
0102 '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
0103 '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
0104 '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
0105 '<style type="text/css">'+\
0106 '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
0107 '%s'%style+\
0108 '</style>'+\
0109 '%s'%javascripts+\
0110 '%s'%additional_header+\
0111 '</head>'+\
0112 '<body>'+\
0113 '<div class="container">'
0114
0115 return html
0116
0117
0118
0119 def get_page_footer():
0120 return '</div></body></html>'
0121
0122
0123
0124 def get_title_section(directory, hashing_flag, standalone, depth=2):
0125 if standalone:
0126 cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
0127 else:
0128 cms_logo_url = "cms-service-reldqm/style/CMS.gif"
0129 mother_name=basename(directory.mother_dir)
0130 mother_file_name=""
0131 if depth==1:
0132 mother_file_name="../RelMonSummary.html"
0133 if mother_name!="":
0134 mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0135 elif depth==2:
0136 mother_file_name="RelMonSummary.html"
0137
0138 if mother_name!="":
0139 mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
0140 else:
0141 if hashing_flag:
0142 files = directory.mother_dir.split("/")
0143 if len(files) != 1:
0144 dir_name = files[-2]+files[-1]
0145 else:
0146 dir_name = files[-1]
0147 dir_name = directory.mother_dir
0148 mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
0149 else:
0150 mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
0151 mother_file_name=mother_file_name.strip("_")
0152
0153 link_to_mother='<a href="%s">..</a>' %mother_file_name
0154 html= '<div class="span-20">'+\
0155 '<h1>%s</h1>'%directory.name+\
0156 '</div>'+\
0157 '<div class="span-1">'+\
0158 '<h1>%s</h1>'%link_to_mother+\
0159 '</div>'+\
0160 '<div class="span-3 last">'+\
0161 '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
0162 '</div>'+\
0163 '<hr>'
0164 if len(mother_name)>0:
0165 html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
0166 '<hr>'
0167 return html
0168
0169
0170 def get_dir_stats(directory):
0171 html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
0172 html+='<ul>'
0173 if directory.n_successes>0:
0174 html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
0175 if directory.n_nulls>0:
0176 html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
0177 if directory.n_fails>0:
0178 html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
0179 if directory.n_skiped>0:
0180 html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
0181 if directory.n_missing_objs>0:
0182 html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
0183 html+='</ul>'
0184 return html
0185
0186
0187
0188 def get_subdirs_section(directory, hashing_flag):
0189 if len(directory.subdirs)==0:
0190 return ""
0191 html= '<div class="span-20 colborder">'
0192 html+='<h2 class="alt">Sub-Directories</h2>'
0193
0194
0195 sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
0196 sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
0197 sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
0198 for subdir in sorted_subdirs:
0199 name=subdir.name
0200 if hashing_flag:
0201 link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag))
0202 else:
0203 link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
0204 link=link.strip("_")
0205 html+='<div class="span-4 prepend-2 colborder">'
0206 html+='<h3>%s</h3>'%name
0207 html+='</div>'
0208
0209 html+='<div class="span-7">'
0210 html+=get_dir_stats(subdir)
0211 html+='</div>'
0212
0213 html+='<div class="span-6 last">'
0214 html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
0215 html+='</div>'
0216
0217 html+='<hr>'
0218 return html+'</div>'
0219
0220
0221
0222
0223 def get_summary_section(directory,matrix_page=True):
0224
0225
0226
0227 test_name=""
0228 test_threshold=0
0229 for comparison in directory.comparisons:
0230 test_name=comparison.test_name
0231 test_threshold=comparison.test_thr
0232 break
0233 if len(test_name)==0:
0234 for subdir in directory.subdirs:
0235 for comparison in subdir.comparisons:
0236 test_name=comparison.test_name
0237 test_threshold=comparison.test_thr
0238 break
0239 if len(test_name)!=0:break
0240 if len(test_name)==0:
0241 for subsubdir in subdir.subdirs:
0242 for comparison in subsubdir.comparisons:
0243 test_name=comparison.test_name
0244 test_threshold=comparison.test_thr
0245 break
0246 if len(test_name)!=0:break
0247 if len(test_name)==0:
0248 for subsubsubdir in subsubdir.subdirs:
0249 for comparison in subsubsubdir.comparisons:
0250 test_name=comparison.test_name
0251 test_threshold=comparison.test_thr
0252 break
0253 if len(test_name)!=0:break
0254
0255
0256 meta=directory.meta
0257
0258 html= '<div class="span-6">'+\
0259 '<h3>Summary</h3>'
0260 html+=get_dir_stats(directory)
0261 html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
0262 html+='</div>'
0263
0264 html+='<div class="span-7 colborder">'+\
0265 '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
0266 '</div>'+\
0267 '<div class="span-9 last">'
0268 if matrix_page:
0269 html+='<h3>Sample:</h3>'+\
0270 '<p class="caps">%s</p>'%meta.sample1+\
0271 '<h3>Run1 and Run2:</h3>'+\
0272 '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
0273 html+='<h3>Releases:</h3>'+\
0274 '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
0275 '<h3>Statistical Test (Pvalue threshold):</h3>'+\
0276 '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
0277 '</div>'+\
0278 '<hr>'
0279 return html
0280
0281
0282
0283 def get_comparisons(category,directory):
0284 """Prepare the comparisons between histograms and organise them in the page.
0285 Moreover create separate pages with the overlay and the single plots.
0286 """
0287 counter=1
0288 tot_counter=1
0289
0290
0291 comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
0292 n_comparisons=len(comparisons)
0293
0294 is_reverse=True
0295 if category == FAIL:
0296 is_reverse=False
0297 comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
0298
0299
0300 dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
0301 html_comparisons=""
0302 for comparison in comparisons:
0303 plot_name=comparison.img_name
0304 if "http://" not in plot_name:
0305 plot_name= basename(comparison.img_name)
0306 class_type="colborder"
0307 if counter==3 or tot_counter==n_comparisons:
0308 class_type=" colborder last"
0309 comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
0310
0311
0312 if directory.do_pngs:
0313 png_link=comparison.img_name
0314 html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0315 '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0316 '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
0317 else:
0318 big_fairy=fairy_url(directory.meta.run1,
0319 directory.meta.run2,
0320 directory.meta.sample1,
0321 directory.meta.sample2,
0322 directory.meta.release1,
0323 directory.meta.release2,
0324 comp_abs_path,
0325 directory.meta.tier1,
0326 directory.meta.tier2,
0327 "",600,600)
0328 small_fairy=fairy_url(directory.meta.run1,
0329 directory.meta.run2,
0330 directory.meta.sample1,
0331 directory.meta.sample2,
0332 directory.meta.release1,
0333 directory.meta.release2,
0334 comp_abs_path,
0335 directory.meta.tier1,
0336 directory.meta.tier2)
0337
0338 single_fairy1=fairy_url_single(directory.meta.run1,
0339 directory.meta.sample1,
0340 directory.meta.release1,
0341 comp_abs_path,
0342 directory.meta.tier1,
0343 "",500,500)
0344 single_fairy2=fairy_url_single(directory.meta.run2,
0345 directory.meta.sample2,
0346 directory.meta.release2,
0347 comp_abs_path,
0348 directory.meta.tier2,
0349 "",500,500)
0350
0351 html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
0352 '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
0353 '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
0354 '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
0355 '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
0356
0357 if counter==3:
0358 html_comparisons+="<hr>"
0359 counter=0
0360 counter+=1
0361 tot_counter+=1
0362
0363 if len(html_comparisons)!=0:
0364 html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
0365 html+=html_comparisons
0366 html+='<hr>'
0367 return html
0368 return ""
0369
0370
0371
0372 def get_rank_section(directory):
0373
0374 imgname="RankSummary.png"
0375 gStyle.SetPadTickY(0)
0376 c=TCanvas("ranks","ranks",500,400)
0377
0378 c.cd()
0379
0380 h=directory.rank_histo
0381 rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
0382 rank_histof.SetLineWidth(2)
0383 for i in range(0,h.GetNbinsX()+1):
0384 rank_histof.SetBinContent(i,h.GetBinContent(i))
0385 h.SetTitle("Ranks Summary;Rank;Frequency")
0386 h.Draw("Hist")
0387 c.Update()
0388 rank_histof.ComputeIntegral()
0389 integral = rank_histof.GetIntegral()
0390 rank_histof.SetContent(integral)
0391
0392 rightmax = 1.1*rank_histof.GetMaximum()
0393 scale = gPad.GetUymax()/rightmax
0394 rank_histof.SetLineColor(kRed)
0395 rank_histof.Scale(scale)
0396 rank_histof.Draw("same")
0397
0398
0399 axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
0400 axis.SetTitle("Cumulative")
0401 axis.SetTitleColor(kRed)
0402 axis.SetLineColor(kRed)
0403 axis.SetLabelColor(kRed)
0404 axis.Draw()
0405
0406 rank_histof.Draw("Same");
0407
0408
0409 c.Print(imgname)
0410
0411 page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
0412 page_html+='<div class="span-19"><img src="%s"></div>' %imgname
0413 page_html+='</div> <a href="#top">Top...</a><hr>'
0414
0415 return page_html
0416
0417
0418
0419
0420 def get_missing_objs_section(directory):
0421 """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
0422 """
0423 page_html = "Unpaired in %s</br>"%(directory.filename1)
0424 for elem in directory.different_histograms['file1']:
0425 page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
0426 page_html +="</br>"
0427 page_html += "Unpaired in %s</br>"%(directory.filename2)
0428 for elem in directory.different_histograms['file2']:
0429 page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
0430 return page_html
0431
0432
0433 def directory2html(directory, hashing, standalone, depth=0):
0434 """Converts a directory tree into html pages, very nice ones.
0435 """
0436
0437 depth+=1
0438
0439
0440
0441
0442
0443
0444
0445 for subdir in directory.subdirs:
0446 directory2html(subdir,hashing,standalone, depth)
0447
0448 page_html=get_page_header(directory, standalone)+\
0449 get_title_section(directory,hashing, standalone, depth)+\
0450 get_summary_section(directory)+\
0451 get_subdirs_section(directory, hashing)
0452
0453 for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
0454 (directory.n_comp_nulls >0,NULL ),
0455
0456 (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
0457 (directory.n_comp_skiped >0,SKIPED)):
0458 if do_cat:
0459 page_html+=get_comparisons(cat,directory)
0460
0461 if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
0462 page_html += get_missing_objs_section(directory)
0463
0464
0465
0466 if depth==1:
0467 page_html+=get_rank_section(directory)
0468
0469
0470 page_html+=get_page_footer()
0471
0472 page_name=directory.name
0473
0474 if len(page_name)==0:
0475 page_name="RelMonSummary"
0476 if hashing:
0477 if page_name != "RelMonSummary":
0478
0479
0480 ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing))
0481 else:
0482 ofilename = "RelMonSummary.html"
0483 else:
0484 ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
0485 ofilename=ofilename.strip("_")
0486
0487 ofile=open(ofilename,"w")
0488 ofile.write(page_html)
0489 ofile.close()
0490
0491
0492
0493
0494
0495 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
0496 total_success_rate_scaled=(total_success_rate-minrate)
0497 total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
0498 if total_success_rate_scaled_repr<0:
0499 total_success_rate_scaled_repr=0
0500 size_s="200x100"
0501 if small:
0502 size_s="40x30"
0503
0504 gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
0505 gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
0506 if not small:
0507 gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
0508 gauge_link+="&chma=10,10,10,0"
0509 img_tag= '<img src="%s">'%gauge_link
0510 if escaped:
0511 img_tag=html.escape(img_tag)
0512 return img_tag
0513
0514
0515
0516 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
0517
0518 aggr_pairs_info=[]
0519
0520 list_of_names=[]
0521 if the_aggr_pairs==[]:
0522 for samplename,sampledir in dir_dict.items():
0523 for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
0524 if not subsysdirname in list_of_names:
0525 list_of_names.append(subsysdirname)
0526 the_aggr_pairs.append((subsysdirname,[subsysdirname]))
0527
0528
0529 for cat_name, subdir_list in the_aggr_pairs:
0530 total_successes=0.
0531 total_directory_successes=0
0532 total_weight=0.
0533 present_subdirs={}
0534 total_ndirs=0
0535
0536 for dirname, sampledir in dir_dict.items():
0537
0538 for subdirname,subdir in sampledir.get_subdirs_dict().items():
0539 if subdirname in subdir_list:
0540 nsucc=subdir.n_successes
0541 total_successes+=nsucc
0542 weight=subdir.weight
0543 total_weight+=weight
0544 total_ndirs+=1
0545
0546 total_directory_successes+= float(nsucc)/weight
0547 if subdirname in present_subdirs:
0548 this_dir_dict=present_subdirs[subdirname]
0549 this_dir_dict["nsucc"]+=nsucc
0550 this_dir_dict["weight"]+=weight
0551 else:
0552 present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
0553
0554 for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
0555 for pathname in [name for name in subdir_list if "/" in name]:
0556 selected_subdirname,selected_subsubdirname = pathname.split("/")
0557 if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
0558
0559 nsucc=subsubdir.n_successes
0560 total_successes+=nsucc
0561 weight=subsubdir.weight
0562 total_weight+=weight
0563 total_ndirs+=1
0564 total_directory_successes+= float(nsucc)/weight
0565
0566 if subsubdirname in present_subdirs:
0567 this_dir_dict=present_subdirs[subsubdirname]
0568 this_dir_dict["nsucc"]+=nsucc
0569 this_dir_dict["weight"]+=weight
0570 else:
0571 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
0572
0573 if total_ndirs == 0:
0574 print("No directory of the category %s is present in the samples: skipping." %cat_name)
0575 continue
0576
0577 average_success_rate=total_directory_successes/(total_ndirs)
0578 aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
0579
0580 return aggr_pairs_info
0581
0582
0583
0584 def make_categories_summary(dir_dict,aggregation_rules):
0585
0586 aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0587
0588
0589
0590
0591
0592 html= '<div class="span-20 colborder">'
0593 html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
0594
0595 for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0596
0597 html+='<div class="span-3 prepend-0 colborder">'
0598 html+='<h3>%s</h3>'%cat_name
0599 html+='<div><span class="alt">Avg. Success rate:</span></div>'
0600 html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
0601 html+='</div>'
0602 html+='<div class="span-9">'
0603
0604 html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
0605 html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
0606 html+='<ul>'
0607 for subdirname in sorted(present_subdirs.keys()):
0608 this_dir_dict=present_subdirs[subdirname]
0609 nsucc=this_dir_dict["nsucc"]
0610 weight=this_dir_dict["weight"]
0611 html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
0612 html+='</ul>'
0613 html+='</div>'
0614
0615 html+='<div class="span-6 last">'
0616 html+=build_gauge(average_success_rate)
0617 html+='</div>'
0618
0619 html+='<hr>'
0620 return html+'<br><a href="#top">Top...</a> </div><hr>'
0621
0622
0623
0624 def make_twiki_table(dir_dict,aggregation_rules):
0625
0626
0627 meta= list(dir_dict.items())[0][1].meta
0628 releases=sorted([meta.release1,meta.release2])
0629 latest_release=releases[1].split("-")[0]
0630
0631
0632 aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
0633
0634
0635
0636 html= '<div class="span-20 colborder">'
0637 html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
0638 html+='<div>| Release | Comparison |'
0639 for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0640 html+=cat_name
0641 html+=" | "
0642 html+='</div>'
0643
0644 html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
0645
0646
0647
0648 for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
0649
0650 html+=build_gauge(average_success_rate,small=True,escaped=True)
0651 html+=" | "
0652
0653 html+='</div> <a href="#top">Top...</a>'
0654 html+='<hr>'
0655 return html+'</div>'
0656
0657
0658
0659 def get_pie_tooltip(directory):
0660 tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
0661 return tooltip
0662
0663
0664
0665 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
0666
0667 aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
0668
0669 script="""
0670 <script type="text/javascript" src="https://www.google.com/jsapi"></script>
0671 <script type="text/javascript">
0672 google.load("visualization", "1", {packages:["corechart"]});
0673 google.setOnLoadCallback(drawChart);
0674 function drawChart() {
0675 var data = new google.visualization.DataTable();
0676 data.addColumn('string', 'DQM Directory');
0677 data.addColumn('number', 'Success Rate');
0678 """
0679 script+="data.addRows(%i);\n"%len(aggr_pairs_info)
0680 counter=0
0681 for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
0682
0683 script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
0684 script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
0685 counter+=1
0686 script+="""
0687 var chart = new google.visualization.BarChart(document.getElementById('%s'));
0688 chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
0689 vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
0690 });
0691 }
0692 </script>
0693 """%(name,40*counter,title)
0694 return script
0695
0696
0697
0698
0699 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
0700 """Create a table, with as rows the directories and as columns the samples.
0701 Each box in the table will contain a pie chart linking to the directory.
0702 """
0703
0704
0705
0706 chdir(indir)
0707 if os.path.isabs(indir):
0708 title = basename(indir)
0709 else:
0710 title=indir
0711 title=title.strip(".")
0712 title=title.strip("/")
0713
0714
0715
0716 sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
0717
0718
0719 dir_unpicklers=[]
0720 n_unpicklers=0
0721 for sample_pkl in sample_pkls:
0722 dir_unpickler=unpickler(sample_pkl)
0723 dir_unpickler.start()
0724 n_unpicklers+=1
0725 dir_unpicklers.append(dir_unpickler)
0726 if n_unpicklers>=1:
0727 n_unpicklers=0
0728 for dir_unpickler in dir_unpicklers:
0729 dir_unpickler.join()
0730
0731 dir_dict={}
0732
0733
0734 global_dir=Directory("global","")
0735 for dir_unpickler in dir_unpicklers:
0736 dir_unpickler.join()
0737 directory=dir_unpickler.directory
0738
0739
0740 global_dir.meta=directory.meta
0741 dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
0742 global_dir.subdirs.append(directory)
0743
0744 global_dir.calcStats()
0745
0746 directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
0747 categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
0748
0749 page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
0750 rel1=""
0751 rel2=""
0752 try:
0753 rel1,rel2=title.split("VS")
0754 except:
0755 rel1=global_dir.meta.release1.split("-")[0]
0756 rel2=global_dir.meta.release2.split("-")[0]
0757 global_dir.meta.release1=rel1
0758 global_dir.meta.release2=rel2
0759
0760
0761 all_subdirs=[]
0762 for directory in dir_dict.values():
0763 for subdir_name in directory.get_subdirs_names():
0764 all_subdirs.append(subdir_name)
0765 all_subdirs=sorted(list(set(all_subdirs)))
0766
0767
0768
0769
0770 page_html+= '<div class="span-20">'+\
0771 '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
0772 '</div>'+\
0773 '<div class="span-1">'+\
0774 '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
0775 '</div>'+\
0776 '<hr>'
0777 page_html+='<div class="span-24"><p></p></div>\n'*3
0778
0779
0780 page_html+= get_summary_section(global_dir,False)
0781
0782
0783 page_html+= '<div class="span-24">'
0784 page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
0785 '<ul>'+\
0786 '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
0787 '<li><a href="#categories">Categories</a></li>'+\
0788 '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
0789 '<li><a href="#summary_table">Summary Table</a></li>'+\
0790 '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
0791 '<li><a href="#twiki_table">Twiki Table</a></li>'+\
0792 '</ul>'+\
0793 '</div><hr>'
0794
0795
0796
0797 page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
0798 page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
0799
0800
0801 page_html+=make_categories_summary(dir_dict,aggregation_rules)
0802
0803
0804 page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
0805 page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
0806
0807
0808 for i in range(2):
0809 page_html+='<div class="span-24"><p></p></div>\n'
0810
0811
0812
0813 page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
0814
0815 for i in range(5):
0816 page_html+='<div class="span-24"><p></p></div>\n'
0817
0818 page_html+="""
0819 <table border="1" >
0820 <tr>
0821 <td> </td>
0822 """
0823
0824
0825 page_html+="""
0826 <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
0827
0828 sorted_samples=sorted(dir_dict.keys())
0829 for sample in sorted_samples:
0830 sample_nick=sample
0831
0832
0833
0834
0835
0836
0837
0838 page_html+="""
0839 <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
0840 page_html+=" </tr>\n"
0841
0842
0843
0844
0845 page_html+="<tr>"
0846 page_html+='<td style="background-color:white;"><div class="span-1">'
0847
0848 page_html+='<b>Summary</b></div></td>'
0849 page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
0850 for sample in sorted_samples:
0851 col=dir_dict[sample]
0852
0853 summary_page_name="RelMonSummary.html"
0854 if col.name!="":
0855 summary_page_name=hash_name(col.name, hashing_flag)+".html"
0856 img_link=col.get_summary_chart_ajax(55,55)
0857 page_html+='<td style="background-color:white;"><div class="span-1">'
0858 page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
0859 page_html+="</tr>"
0860
0861
0862 for subdir_name in all_subdirs:
0863
0864 page_html+=' <tr>\n'
0865 page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
0866
0867 row_summary=Directory("row_summary","")
0868 sample_counter=0
0869 n_samples=len(sorted_samples)
0870
0871 for sample in sorted_samples:
0872 subdirs_dict=directory.get_subdirs_dict()
0873 directory=dir_dict[sample]
0874 dir_is_there=subdir_name in subdirs_dict
0875 if dir_is_there:
0876 row_summary.subdirs.append(subdirs_dict[subdir_name])
0877
0878
0879 row_summary.calcStats()
0880 img_link=row_summary.get_summary_chart_ajax(55,55)
0881 page_html+='<td style="background-color:white;"><div class="span-1">'
0882 page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
0883
0884 for sample in sorted_samples:
0885 sample_counter+=1
0886
0887 directory=dir_dict[sample]
0888 subdirs_dict=directory.get_subdirs_dict()
0889
0890
0891 if directory.name!="":
0892
0893
0894
0895 if hashing_flag:
0896 summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
0897 else:
0898 summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
0899 else:
0900
0901
0902 summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
0903 dir_is_there=subdir_name in subdirs_dict
0904
0905 img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
0906 img_tooltip="N/A"
0907 if dir_is_there:
0908
0909 img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
0910 img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
0911
0912 page_html+='<td style="background-color:white;"><div class="span-1">'
0913 if dir_is_there:
0914 page_html+='<a href="%s">'%(summary_page)
0915 page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
0916 if dir_is_there:
0917 page_html+='</a>'
0918 page_html+='</div></td>'
0919
0920 page_html+=" </tr>\n"
0921
0922
0923
0924 page_html+='</table> <a href="#top">Top...</a><hr>'
0925
0926 page_html+=get_rank_section(global_dir)
0927
0928 page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
0929
0930 page_html+=get_page_footer()
0931 return page_html
0932
0933
0934
0935 def hash_name(file_name, flag):
0936
0937 if flag:
0938 return hashlib.md5(file_name).hexdigest()[:10]
0939 else:
0940 return file_name