RelMon/python/dirstructure.py

0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 ################################################################################
0004 # RelMon: a tool for automatic Release Comparison
0005 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0006 #
0007 #
0008 #
0009 # Danilo Piparo CERN - danilo.piparo@cern.ch
0010 #
0011 ################################################################################
0012
0013 from builtins import range
0014 from array import array
0015 from copy import deepcopy
0016 from os import chdir,getcwd,listdir,makedirs,rmdir
0017 from os.path import exists,join
0018 import random
0019
0020 import sys
0021 argv=sys.argv
0022 import ROOT
0023 sys.argv=argv
0024
0025 from .definitions import *
0026 from .utils import setTDRStyle
0027
0028
0029 # Something nice and familiar
0030 setTDRStyle()
0031
0032 # Do not display the canvases
0033 ROOT.gROOT.SetBatch(ROOT.kTRUE)
0034
0035
0036 #-------------------------------------------------------------------------------
0037 _log_level=5
0038 def logger(msg_level,message):
0039   if msg_level>=_log_level:
0040     print("[%s] %s" %(asctime(),message))
0041
0042 #-------------------------------------------------------------------------------
0043
0044 class Weighted(object):
0045   def __init__(self,name,weight=1):
0046     self.name=name
0047     self.weight=weight
0048
0049
0050 #-------------------------------------------------------------------------------
0051 class CompInfo(object):
0052   def __init__(self,sample1="",sample2="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
0053     self.sample1=sample1
0054     self.sample2=sample2
0055     self.release1=release1
0056     self.release2=release2
0057     self.run1=run1
0058     self.run2=run2
0059     self.tier1=tier1
0060     self.tier2=tier2
0061
0062 #-------------------------------------------------------------------------------
0063 class Directory(Weighted):
0064   def __init__(self,name,mother_dir="",meta=CompInfo(),draw_success=False,do_pngs=False):
0065     self.mother_dir=mother_dir
0066     self.meta=meta
0067     self.subdirs=[]
0068     self.comparisons=[]
0069     self.n_fails=0
0070     self.n_successes=0
0071     self.n_nulls=0
0072     self.n_skiped = 0
0073     self.n_comp_skiped = 0
0074     self.n_comp_fails=0
0075     self.n_comp_successes=0
0076     self.n_comp_nulls=0
0077     self.weight=0
0078     self.stats_calculated=False
0079     Weighted.__init__(self,name)
0080     self.draw_success=draw_success
0081     self.do_pngs=do_pngs
0082     self.rank_histo=ROOT.TH1I("rh%s"%name,"",50,-0.01,1.001)
0083     self.rank_histo.SetDirectory(0)
0084     self.different_histograms = {}
0085     self.different_histograms['file1']= {}
0086     self.different_histograms['file2']= {}
0087     self.filename1 = ""
0088     self.filename2 = ""
0089     self.n_missing_objs = 0
0090     self.full_path = ""
0091
0092   def is_empty(self):
0093     if len(self.subdirs)==0 and len(self.comparisons)==0:
0094       return True
0095     return False
0096
0097   def calcStats(self,make_pie=True):
0098     '''Walk all subdirs and calculate weight,fails and successes.
0099     Moreove propagate the sample and releases names.
0100     '''
0101     if self.stats_calculated:
0102       return 0
0103
0104     self.n_fails=0
0105     self.n_successes=0
0106     self.n_nulls=0
0107     self.n_comp_fails=0
0108     self.n_comp_successes=0
0109     self.n_comp_nulls=0
0110     self.weight=0
0111
0112     self.n_skiped = 0
0113     self.n_comp_skiped = 0
0114     self.n_missing_objs = len(self.different_histograms['file1'])+len(self.different_histograms['file2'])
0115     if self.n_missing_objs != 0:
0116       print("    [*] Missing in %s: %s" %(self.filename1, self.different_histograms['file1']))
0117       print("    [*] Missing in %s: %s" %(self.filename2, self.different_histograms['file2']))
0118     # clean from empty dirs
0119     self.subdirs = [subdir for subdir in self.subdirs if not subdir.is_empty()]
0120
0121     for comp in self.comparisons:
0122       if comp.status == SKIPED: #in case its in black list & skiped
0123           self.n_skiped += 1
0124           self.n_comp_skiped += 1
0125           self.weight+=1
0126       else: #else original code -> to check for Fails and Successes
0127           self.rank_histo.Fill(comp.rank)
0128           self.weight+=1
0129           if comp.status == FAIL:
0130               self.n_fails+=1
0131               self.n_comp_fails+=1
0132           elif comp.status == SUCCESS:
0133               self.n_successes+=1
0134               self.n_comp_successes+=1
0135           else:
0136               self.n_nulls+=1
0137               self.n_comp_nulls+=1
0138
0139     for subdir in self.subdirs:
0140       subdir.mother_dir=join(self.mother_dir,self.name)
0141       subdir.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
0142       subdir.calcStats(make_pie)
0143       subdir.meta=self.meta
0144       self.weight+=subdir.weight
0145       self.n_fails+=subdir.n_fails
0146       self.n_successes+=subdir.n_successes
0147       self.n_nulls+=subdir.n_nulls
0148
0149       self.n_skiped+=subdir.n_skiped
0150       self.n_missing_objs += subdir.n_missing_objs
0151
0152       self.rank_histo.Add(subdir.rank_histo)
0153
0154     self.stats_calculated=True
0155     self.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
0156     #if make_pie:
0157       #self.__create_pie_image()
0158
0159   def get_subdirs_dict(self):
0160     subdirdict={}
0161     for subdir in self.subdirs:
0162       subdirdict[subdir.name]=subdir
0163     return subdirdict
0164
0165   def get_subdirs_names(self):
0166     subdirnames=[]
0167     for subdir in self.subdirs:
0168       subdirnames.append(subdir.name)
0169     return subdirnames
0170
0171   def get_piechart_js(self,w=400,link=None):
0172
0173     """
0174     Build the HTML snippet to render a piechart with chart.js
0175     """
0176     if self.get_success_rate()>=99.9: # if the success rate is very high let's make the page lighter
0177       img_link = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/5ee98db210c0898fd34b4deac3653fa2bdff269b/report_website/lime_circle.png"
0178       html ='<img src="%s" height=%d width=%d>' %(img_link,w,w)
0179       if link is not None:
0180         html = '<a href="%s"> %s </a>' %(link,html)
0181       return html
0182
0183     name = random.getrandbits(64) # just a random has for the canvas
0184     html = ""
0185     html += '<canvas id="%s" height=%d width=%d></canvas>'%(name,w,w)
0186     # piechart
0187     html += '<script> new Chart("%s",'%(name)
0188     html += '{ type: "pie",'
0189
0190     # data
0191     html += 'data: {'
0192     html += 'labels: ["Success", "Null" , "Failure", "Skipped"],'
0193     html += 'datasets: [{ backgroundColor: ["lime","yellow","red","grey"],'
0194     html += 'data: [%.2f,%.2f,%.2f,%.2f]}] },'%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())
0195
0196     #display options
0197     html += 'options: { '
0198
0199     if link is not None:
0200       html += 'onClick : function(event) { window.open("%s", "_blank");},'%(link)
0201
0202
0203     html +='legend: { display: false }, responsive : false, hover: {mode: null}, tooltips: {enabled: false}'
0204     #tooltips: {enabled: false}, hover: {mode: null},'
0205
0206     html += '}}); </script>'
0207
0208     return html
0209
0210   def print_report(self,indent="",verbose=False):
0211     if len(indent)==0:
0212       self.calcStats(make_pie=False)
0213     # print small failure report
0214     if verbose:
0215       fail_comps=[comp for comp in self.comparisons if comp.status==FAIL]
0216       fail_comps=sorted(fail_comps,key=lambda comp:comp.name )
0217       if len(fail_comps)>0:
0218         print(indent+"* %s/%s:" %(self.mother_dir,self.name))
0219         for comp in fail_comps:
0220           print(indent+" - %s: %s Test Failed (pval = %s) " %(comp.name,comp.test_name,comp.rank))
0221       for subdir in self.subdirs:
0222         subdir.print_report(indent+"  ",verbose)
0223
0224     if len(indent)==0:
0225       print("\n%s - summary of %s tests:" %(self.name,self.weight))
0226       print(" o Failiures: %.2f%% (%s/%s)" %(self.get_fail_rate(),self.n_fails,self.weight))
0227       print(" o Nulls: %.2f%% (%s/%s) " %(self.get_null_rate(),self.n_nulls,self.weight))
0228       print(" o Successes: %.2f%% (%s/%s) " %(self.get_success_rate(),self.n_successes,self.weight))
0229       print(" o Skipped: %.2f%% (%s/%s) " %(self.get_skiped_rate(),self.n_skiped,self.weight))
0230       print(" o Missing objects: %s" %(self.n_missing_objs))
0231
0232   def get_skiped_rate(self):
0233     if self.weight == 0: return 0
0234     return 100.*self.n_skiped/self.weight
0235   def get_fail_rate(self):
0236     if self.weight == 0:return 0
0237     return 100.*self.n_fails/self.weight
0238
0239   def get_success_rate(self):
0240     if self.weight == 0:return 1
0241     return 100.*self.n_successes/self.weight
0242
0243   def get_null_rate(self):
0244     if self.weight == 0:return 0
0245     return 100.*self.n_nulls/self.weight
0246
0247   def __get_full_path(self):
0248     #print "Mother is %s" %self.mother_dir
0249     if len(self.mother_dir)==0:
0250       return self.name
0251     return join(self.mother_dir,self.name)
0252
0253   def __create_on_disk(self):
0254     if not exists(self.mother_dir) and len(self.mother_dir)!=0:
0255       makedirs(self.mother_dir)
0256     full_path=self.__get_full_path()
0257     if not exists(full_path) and len(full_path)>0:
0258       makedirs(full_path)
0259
0260   def get_summary_chart_name(self):
0261     return join(self.__get_full_path(),"summary_chart.png")
0262
0263   def __create_pie_image(self):
0264     self.__create_on_disk()
0265     vals=[]
0266     colors=[]
0267     for n,col in zip((self.n_fails,self.n_nulls,self.n_successes,self.n_skiped),(ROOT.kRed,ROOT.kYellow,ROOT.kGreen,ROOT.kBlue)):
0268       if n!=0:
0269         vals.append(n)
0270         colors.append(col)
0271     valsa=array('f',vals)
0272     colorsa=array('i',colors)
0273     can = ROOT.TCanvas("cpie","TPie test",100,100);
0274     try:
0275       pie = TPie("ThePie",self.name,len(vals),valsa,colorsa);
0276       label_n=0
0277       if self.n_fails!=0:
0278         pie.SetEntryLabel(label_n, "Fail: %.1f(%i)" %(self.get_fail_rate(),self.n_fails) );
0279         label_n+=1
0280       if self.n_nulls!=0:
0281         pie.SetEntryLabel(label_n, "Null: %.1f(%i)" %(self.get_null_rate(),self.n_nulls) );
0282         label_n+=1
0283       if self.n_successes!=0:
0284         pie.SetEntryLabel(label_n, "Success: %.1f(%i)" %(self.get_success_rate(),self.n_successes) );
0285       if self.n_skiped!=0:
0286         pie.SetEntryLabel(label_n, "Skipped: %.1f(%i)" %(self.get_skiped_rate(),self.n_skiped));
0287       pie.SetY(.52);
0288       pie.SetAngularOffset(0.);
0289       pie.SetLabelsOffset(-.3);
0290       #pie.SetLabelFormat("#splitline{%val (%perc)}{%txt}");
0291       pie.Draw("3d  nol");
0292       can.Print(self.get_summary_chart_name());
0293     except:
0294       print("self.name = %s" %self.name)
0295       print("len(vals) = %s (vals=%s)" %(len(vals),vals))
0296       print("valsa = %s" %valsa)
0297       print("colorsa = %s" %colorsa)
0298
0299   def prune(self,expandable_dir):
0300     """Eliminate from the tree the directory the expandable ones.
0301     """
0302     #print "pruning %s" %self.name
0303     exp_index=-1
0304     counter=0
0305     for subdir in self.subdirs:
0306       # Eliminate any trace of the expandable path in the mother directories
0307       # for depths higher than 1
0308       subdir.mother_dir=subdir.mother_dir.replace("/"+expandable_dir,"")
0309       if subdir.name==expandable_dir:
0310         exp_index=counter
0311       counter+=1
0312
0313     # Did we find an expandable?
0314     if exp_index>=0:
0315       exp_dir=self.subdirs[exp_index]
0316       for subsubdir in exp_dir.subdirs:
0317         #print "*******",subsubdir.mother_dir,
0318         subsubdir.mother_dir=subsubdir.mother_dir.replace("/"+expandable_dir,"")
0319         while "//" in subsubdir.mother_dir:
0320           print(subsubdir.mother_dir)
0321           subsubdir.mother_dir=subsubdir.mother_dir.replace("//","/")
0322         #print "*******",subsubdir.mother_dir
0323         self.subdirs.append(subsubdir)
0324
0325         for comp in exp_dir.comparisons:
0326           comp.mother_dir=comp.mother_dir.replace("/"+expandable_dir,"")
0327           while "//" in comp.mother_dir:
0328               comp.mother_dir
0329               comp.mother_dir=comp.mother_dir.replace("/")
0330           if not comp in self.comparisons:  #in case not to  append same comparisons few times
0331               self.comparisons.append(comp)  # add a comparison
0332               self.n_comp_fails = exp_dir.n_comp_fails  #copy to-be removed directory
0333               self.n_comp_nulls = exp_dir.n_comp_nulls  # numbers to parent directory
0334               self.n_comp_successes = exp_dir.n_comp_successes
0335               self.n_comp_skiped = exp_dir.n_comp_skiped
0336
0337       del self.subdirs[exp_index]
0338       self.prune(expandable_dir)
0339
0340     for subdir in self.subdirs:
0341       subdir.prune(expandable_dir)
0342
0343   def __repr__(self):
0344     if self.is_empty():
0345       return "%s seems to be empty. Please check!" %self.name
0346     content="%s , Rates: Success %.2f%%(%s) - Fail %.2f%%(%s) - Null %.2f%%(%s)\n" %(self.name,self.get_success_rate(),self.n_successes,self.get_fail_rate(),self.n_fails,self.get_null_rate(),self.n_nulls)
0347     for subdir in self.subdirs:
0348       content+=" %s\n" % subdir
0349     for comp in self.comparisons:
0350       content+=" %s\n" % comp
0351     return content
0352
0353 #-------------------------------------------------------------------------------
0354 from multiprocessing import Process
0355 def print_multi_threaded(canvas,img_name):
0356     canvas.Print(img_name)
0357
0358 tcanvas_print_processes=[]
0359 #-------------------------------------------------------------------------------
0360
0361 class Comparison(Weighted):
0362   canvas_xsize=500
0363   canvas_ysize=400
0364   def __init__(self,name,mother_dir,h1,h2,stat_test,draw_success=False,do_pngs=False, skip=False):
0365     self.name=name
0366     self.png_name="placeholder.png"
0367     self.mother_dir=mother_dir
0368     self.img_name=""
0369     #self.draw_success=draw_success
0370     Weighted.__init__(self,name)
0371
0372     stat_test.set_operands(h1,h2)
0373     if skip:
0374         self.status = SKIPED
0375         self.test_name=stat_test.name
0376         self.test_name=stat_test.name
0377         self.test_thr=stat_test.threshold
0378         self.rank = 0
0379     else:
0380         self.status=stat_test.get_status()
0381         self.rank=stat_test.get_rank()
0382         self.test_name=stat_test.name
0383         self.test_thr=stat_test.threshold
0384         self.do_pngs=do_pngs
0385         self.draw_success=draw_success or not do_pngs
0386         if ((self.status==FAIL or self.status==NULL or self.status == SKIPED or self.draw_success) and self.do_pngs):
0387             self.__make_image(h1,h2)
0388         #self.__make_image(h1,h2)
0389
0390   def __make_img_dir(self):
0391     if not exists(self.mother_dir):
0392       makedirs(self.mother_dir)
0393
0394   def __get_img_name(self):
0395     #self.__make_img_dir()
0396     #print "MOTHER: ",self.mother_dir
0397     self.img_name="%s/%s.png"%(self.mother_dir,self.name)
0398     self.img_name=self.img_name.replace("Run summary","")
0399     self.img_name=self.img_name.replace("/","_")
0400     self.img_name=self.img_name.strip("_")
0401     #print "IMAGE NAME: %s " %self.img_name
0402     return self.img_name
0403
0404   def tcanvas_slow(self,canvas):
0405     #print "About to print %s" %self.img_name
0406     #print_multi_threaded(canvas,self.img_name)
0407     #print "-->Printed"
0408
0409     p = Process(target=print_multi_threaded, args=(canvas,self.img_name))
0410     p.start()
0411     tcanvas_print_processes.append(p)
0412     n_proc=len(tcanvas_print_processes)
0413     if n_proc>3:
0414       p_to_remove=[]
0415       for iprocess in range(0,n_proc):
0416         p=tcanvas_print_processes[iprocess]
0417         p.join()
0418         p_to_remove.append(iprocess)
0419
0420       adjustment=0
0421       for iprocess in p_to_remove:
0422         tcanvas_print_processes.pop(iprocess-adjustment)
0423         adjustment+=1
0424
0425   def __make_image(self,obj1,obj2):
0426     self.img_name=self.__get_img_name()
0427     if self.rank==-1:
0428       return 0
0429
0430     canvas=ROOT.TCanvas(self.name,self.name,Comparison.canvas_xsize,Comparison.canvas_ysize)
0431     objs=(obj1,obj2)
0432
0433     # Add some specifics for the graphs
0434     obj1.SetTitle(self.name)
0435
0436     if obj1.GetNbinsY()!=0 and not "2" in obj1.ClassName() :
0437       obj1 .SetLineWidth(2)
0438       obj2 .SetLineWidth(2)
0439
0440       obj1.SetMarkerStyle(8)
0441       obj1.SetMarkerSize(.8)
0442
0443       obj2.SetMarkerStyle(8)
0444       obj2.SetMarkerSize(.8)
0445
0446       obj1.SetMarkerColor(ROOT.kBlue)
0447       obj1.SetLineColor(ROOT.kBlue)
0448
0449       obj2.SetMarkerColor(ROOT.kRed)
0450       obj2.SetLineColor(ROOT.kRed)
0451
0452       obj1.Draw("EP")
0453       #Statsbox
0454       obj2.Draw("HistSames")
0455       #ROOT.gPad.Update()
0456       #if 'stats' in map(lambda o: o.GetName(),list(ROOT.gPad.GetListOfPrimitives())):
0457         #st = gPad.GetPrimitive("stats")
0458         #st.SetY1NDC(0.575)
0459         #st.SetY2NDC(0.735)
0460         #st.SetLineColor(ROOT.kRed)
0461         #st.SetTextColor(ROOT.kRed)
0462         #print st
0463     else:
0464       obj1.Draw("Colz")
0465       ROOT.gPad.Update()
0466       #if 'stats' in map(lambda o: o.GetName(),list(ROOT.gPad.GetListOfPrimitives())):
0467         #st = ROOT.gPad.GetPrimitive("stats")
0468         #st.SetY1NDC(0.575)
0469         #st.SetY2NDC(0.735)
0470         #st.SetLineColor(ROOT.kRed)
0471         #st.SetTextColor(ROOT.kRed)
0472         #print st
0473       obj2.Draw("ColSame")
0474
0475     # Put together the TLatex for the stat test if possible
0476     color=ROOT.kGreen+2 # which is green, as everybody knows
0477     if self.status==FAIL:
0478       print("This comparison failed %f" %self.rank)
0479       color=ROOT.kRed
0480     elif self.status==NULL:
0481       color=ROOT.kYellow
0482     elif self.status==SKIPED:
0483       color=ROOT.kBlue #check if kBlue exists ;)
0484
0485     lat_text="#scale[.7]{#color[%s]{%s: %2.2f}}" %(color,self.test_name,self.rank)
0486     lat=ROOT.TLatex(.1,.91,lat_text)
0487     lat.SetNDC()
0488     lat.Draw()
0489
0490     # Put also the stats together!
0491     n1=obj1.GetEntries()
0492     if n1> 100000:
0493       n1="%e"%n1
0494     else:
0495       n1="%s"%n1
0496     n2=obj2.GetEntries()
0497     if n2> 100000:
0498       n2="%e"%n2
0499     else:
0500       n2="%s"%n2
0501
0502     lat_text1="#scale[.7]{#color[%s]{Entries: %s}}" %(obj1.GetLineColor(),n1)
0503     lat1=ROOT.TLatex(.3,.91,lat_text1)
0504     lat1.SetNDC()
0505     lat1.Draw()
0506
0507
0508     lat_text2="#scale[.7]{#color[%s]{Entries: %s}}" %(obj2.GetLineColor(),n2)
0509     lat2=ROOT.TLatex(.6,.91,lat_text2)
0510     lat2.SetNDC()
0511     lat2.Draw()
0512
0513
0514     self.tcanvas_slow(canvas)
0515
0516   def __repr__(self):
0517     return "%s , (%s=%s). IMG=%s. status=%s" %(self.name,self.test_name,self.rank,self.img_name,self.status)
0518
0519 #-------------------------------------------------------------------------------