RelMon/python/dirstructure.py

0001 ################################################################################
0002 # RelMon: a tool for automatic Release Comparison
0003 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0004 #
0005 #
0006 #
0007 # Danilo Piparo CERN - danilo.piparo@cern.ch
0008 #
0009 ################################################################################
0010
0011 from builtins import range
0012 from array import array
0013 from copy import deepcopy
0014 from os import chdir,getcwd,listdir,makedirs,rmdir
0015 from os.path import exists,join
0016 import random
0017
0018 import sys
0019 argv=sys.argv
0020 import ROOT
0021 sys.argv=argv
0022
0023 from .definitions import *
0024 from .utils import setTDRStyle
0025
0026
0027 # Something nice and familiar
0028 setTDRStyle()
0029
0030 # Do not display the canvases
0031 ROOT.gROOT.SetBatch(ROOT.kTRUE)
0032
0033
0034 #-------------------------------------------------------------------------------
0035 _log_level=5
0036 def logger(msg_level,message):
0037   if msg_level>=_log_level:
0038     print("[%s] %s" %(asctime(),message))
0039
0040 #-------------------------------------------------------------------------------
0041
0042 class Weighted(object):
0043   def __init__(self,name,weight=1):
0044     self.name=name
0045     self.weight=weight
0046
0047
0048 #-------------------------------------------------------------------------------
0049 class CompInfo(object):
0050   def __init__(self,sample1="",sample2="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
0051     self.sample1=sample1
0052     self.sample2=sample2
0053     self.release1=release1
0054     self.release2=release2
0055     self.run1=run1
0056     self.run2=run2
0057     self.tier1=tier1
0058     self.tier2=tier2
0059
0060 #-------------------------------------------------------------------------------
0061 class Directory(Weighted):
0062   def __init__(self,name,mother_dir="",meta=CompInfo(),draw_success=False,do_pngs=False):
0063     self.mother_dir=mother_dir
0064     self.meta=meta
0065     self.subdirs=[]
0066     self.comparisons=[]
0067     self.n_fails=0
0068     self.n_successes=0
0069     self.n_nulls=0
0070     self.n_skiped = 0
0071     self.n_comp_skiped = 0
0072     self.n_comp_fails=0
0073     self.n_comp_successes=0
0074     self.n_comp_nulls=0
0075     self.weight=0
0076     self.stats_calculated=False
0077     Weighted.__init__(self,name)
0078     self.draw_success=draw_success
0079     self.do_pngs=do_pngs
0080     self.rank_histo=ROOT.TH1I("rh%s"%name,"",50,-0.01,1.001)
0081     self.rank_histo.SetDirectory(0)
0082     self.different_histograms = {}
0083     self.different_histograms['file1']= {}
0084     self.different_histograms['file2']= {}
0085     self.filename1 = ""
0086     self.filename2 = ""
0087     self.n_missing_objs = 0
0088     self.full_path = ""
0089
0090   def is_empty(self):
0091     if len(self.subdirs)==0 and len(self.comparisons)==0:
0092       return True
0093     return False
0094
0095   def calcStats(self,make_pie=True):
0096     '''Walk all subdirs and calculate weight,fails and successes.
0097     Moreove propagate the sample and releases names.
0098     '''
0099     if self.stats_calculated:
0100       return 0
0101
0102     self.n_fails=0
0103     self.n_successes=0
0104     self.n_nulls=0
0105     self.n_comp_fails=0
0106     self.n_comp_successes=0
0107     self.n_comp_nulls=0
0108     self.weight=0
0109
0110     self.n_skiped = 0
0111     self.n_comp_skiped = 0
0112     self.n_missing_objs = len(self.different_histograms['file1'])+len(self.different_histograms['file2'])
0113     if self.n_missing_objs != 0:
0114       print("    [*] Missing in %s: %s" %(self.filename1, self.different_histograms['file1']))
0115       print("    [*] Missing in %s: %s" %(self.filename2, self.different_histograms['file2']))
0116     # clean from empty dirs
0117     self.subdirs = [subdir for subdir in self.subdirs if not subdir.is_empty()]
0118
0119     for comp in self.comparisons:
0120       if comp.status == SKIPED: #in case its in black list & skiped
0121           self.n_skiped += 1
0122           self.n_comp_skiped += 1
0123           self.weight+=1
0124       else: #else original code -> to check for Fails and Successes
0125           self.rank_histo.Fill(comp.rank)
0126           self.weight+=1
0127           if comp.status == FAIL:
0128               self.n_fails+=1
0129               self.n_comp_fails+=1
0130           elif comp.status == SUCCESS:
0131               self.n_successes+=1
0132               self.n_comp_successes+=1
0133           else:
0134               self.n_nulls+=1
0135               self.n_comp_nulls+=1
0136
0137     for subdir in self.subdirs:
0138       subdir.mother_dir=join(self.mother_dir,self.name)
0139       subdir.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
0140       subdir.calcStats(make_pie)
0141       subdir.meta=self.meta
0142       self.weight+=subdir.weight
0143       self.n_fails+=subdir.n_fails
0144       self.n_successes+=subdir.n_successes
0145       self.n_nulls+=subdir.n_nulls
0146
0147       self.n_skiped+=subdir.n_skiped
0148       self.n_missing_objs += subdir.n_missing_objs
0149
0150       self.rank_histo.Add(subdir.rank_histo)
0151
0152     self.stats_calculated=True
0153     self.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
0154     #if make_pie:
0155       #self.__create_pie_image()
0156
0157   def get_subdirs_dict(self):
0158     subdirdict={}
0159     for subdir in self.subdirs:
0160       subdirdict[subdir.name]=subdir
0161     return subdirdict
0162
0163   def get_subdirs_names(self):
0164     subdirnames=[]
0165     for subdir in self.subdirs:
0166       subdirnames.append(subdir.name)
0167     return subdirnames
0168
0169   def get_piechart_js(self,w=400,link=None):
0170
0171     """
0172     Build the HTML snippet to render a piechart with chart.js
0173     """
0174     if self.get_success_rate()>=99.9: # if the success rate is very high let's make the page lighter
0175       img_link = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/5ee98db210c0898fd34b4deac3653fa2bdff269b/report_website/lime_circle.png"
0176       html ='<img src="%s" height=%d width=%d>' %(img_link,w,w)
0177       if link is not None:
0178         html = '<a href="%s"> %s </a>' %(link,html)
0179       return html
0180
0181     name = random.getrandbits(64) # just a random has for the canvas
0182     html = ""
0183     html += '<canvas id="%s" height=%d width=%d></canvas>'%(name,w,w)
0184     # piechart
0185     html += '<script> new Chart("%s",'%(name)
0186     html += '{ type: "pie",'
0187
0188     # data
0189     html += 'data: {'
0190     html += 'labels: ["Success", "Null" , "Failure", "Skipped"],'
0191     html += 'datasets: [{ backgroundColor: ["lime","yellow","red","grey"],'
0192     html += 'data: [%.2f,%.2f,%.2f,%.2f]}] },'%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())
0193
0194     #display options
0195     html += 'options: { '
0196
0197     if link is not None:
0198       html += 'onClick : function(event) { window.open("%s", "_blank");},'%(link)
0199
0200
0201     html +='legend: { display: false }, responsive : false, hover: {mode: null}, tooltips: {enabled: false}'
0202     #tooltips: {enabled: false}, hover: {mode: null},'
0203
0204     html += '}}); </script>'
0205
0206     return html
0207
0208   def print_report(self,indent="",verbose=False):
0209     if len(indent)==0:
0210       self.calcStats(make_pie=False)
0211     # print small failure report
0212     if verbose:
0213       fail_comps=[comp for comp in self.comparisons if comp.status==FAIL]
0214       fail_comps=sorted(fail_comps,key=lambda comp:comp.name )
0215       if len(fail_comps)>0:
0216         print(indent+"* %s/%s:" %(self.mother_dir,self.name))
0217         for comp in fail_comps:
0218           print(indent+" - %s: %s Test Failed (pval = %s) " %(comp.name,comp.test_name,comp.rank))
0219       for subdir in self.subdirs:
0220         subdir.print_report(indent+"  ",verbose)
0221
0222     if len(indent)==0:
0223       print("\n%s - summary of %s tests:" %(self.name,self.weight))
0224       print(" o Failiures: %.2f%% (%s/%s)" %(self.get_fail_rate(),self.n_fails,self.weight))
0225       print(" o Nulls: %.2f%% (%s/%s) " %(self.get_null_rate(),self.n_nulls,self.weight))
0226       print(" o Successes: %.2f%% (%s/%s) " %(self.get_success_rate(),self.n_successes,self.weight))
0227       print(" o Skipped: %.2f%% (%s/%s) " %(self.get_skiped_rate(),self.n_skiped,self.weight))
0228       print(" o Missing objects: %s" %(self.n_missing_objs))
0229
0230   def get_skiped_rate(self):
0231     if self.weight == 0: return 0
0232     return 100.*self.n_skiped/self.weight
0233   def get_fail_rate(self):
0234     if self.weight == 0:return 0
0235     return 100.*self.n_fails/self.weight
0236
0237   def get_success_rate(self):
0238     if self.weight == 0:return 1
0239     return 100.*self.n_successes/self.weight
0240
0241   def get_null_rate(self):
0242     if self.weight == 0:return 0
0243     return 100.*self.n_nulls/self.weight
0244
0245   def __get_full_path(self):
0246     #print "Mother is %s" %self.mother_dir
0247     if len(self.mother_dir)==0:
0248       return self.name
0249     return join(self.mother_dir,self.name)
0250
0251   def __create_on_disk(self):
0252     if not exists(self.mother_dir) and len(self.mother_dir)!=0:
0253       makedirs(self.mother_dir)
0254     full_path=self.__get_full_path()
0255     if not exists(full_path) and len(full_path)>0:
0256       makedirs(full_path)
0257
0258   def get_summary_chart_name(self):
0259     return join(self.__get_full_path(),"summary_chart.png")
0260
0261   def __create_pie_image(self):
0262     self.__create_on_disk()
0263     vals=[]
0264     colors=[]
0265     for n,col in zip((self.n_fails,self.n_nulls,self.n_successes,self.n_skiped),(ROOT.kRed,ROOT.kYellow,ROOT.kGreen,ROOT.kBlue)):
0266       if n!=0:
0267         vals.append(n)
0268         colors.append(col)
0269     valsa=array('f',vals)
0270     colorsa=array('i',colors)
0271     can = ROOT.TCanvas("cpie","TPie test",100,100);
0272     try:
0273       pie = TPie("ThePie",self.name,len(vals),valsa,colorsa);
0274       label_n=0
0275       if self.n_fails!=0:
0276         pie.SetEntryLabel(label_n, "Fail: %.1f(%i)" %(self.get_fail_rate(),self.n_fails) );
0277         label_n+=1
0278       if self.n_nulls!=0:
0279         pie.SetEntryLabel(label_n, "Null: %.1f(%i)" %(self.get_null_rate(),self.n_nulls) );
0280         label_n+=1
0281       if self.n_successes!=0:
0282         pie.SetEntryLabel(label_n, "Success: %.1f(%i)" %(self.get_success_rate(),self.n_successes) );
0283       if self.n_skiped!=0:
0284         pie.SetEntryLabel(label_n, "Skipped: %.1f(%i)" %(self.get_skiped_rate(),self.n_skiped));
0285       pie.SetY(.52);
0286       pie.SetAngularOffset(0.);
0287       pie.SetLabelsOffset(-.3);
0288       #pie.SetLabelFormat("#splitline{%val (%perc)}{%txt}");
0289       pie.Draw("3d  nol");
0290       can.Print(self.get_summary_chart_name());
0291     except:
0292       print("self.name = %s" %self.name)
0293       print("len(vals) = %s (vals=%s)" %(len(vals),vals))
0294       print("valsa = %s" %valsa)
0295       print("colorsa = %s" %colorsa)
0296
0297   def prune(self,expandable_dir):
0298     """Eliminate from the tree the directory the expandable ones.
0299     """
0300     #print "pruning %s" %self.name
0301     exp_index=-1
0302     counter=0
0303     for subdir in self.subdirs:
0304       # Eliminate any trace of the expandable path in the mother directories
0305       # for depths higher than 1
0306       subdir.mother_dir=subdir.mother_dir.replace("/"+expandable_dir,"")
0307       if subdir.name==expandable_dir:
0308         exp_index=counter
0309       counter+=1
0310
0311     # Did we find an expandable?
0312     if exp_index>=0:
0313       exp_dir=self.subdirs[exp_index]
0314       for subsubdir in exp_dir.subdirs:
0315         #print "*******",subsubdir.mother_dir,
0316         subsubdir.mother_dir=subsubdir.mother_dir.replace("/"+expandable_dir,"")
0317         while "//" in subsubdir.mother_dir:
0318           print(subsubdir.mother_dir)
0319           subsubdir.mother_dir=subsubdir.mother_dir.replace("//","/")
0320         #print "*******",subsubdir.mother_dir
0321         self.subdirs.append(subsubdir)
0322
0323         for comp in exp_dir.comparisons:
0324           comp.mother_dir=comp.mother_dir.replace("/"+expandable_dir,"")
0325           while "//" in comp.mother_dir:
0326               comp.mother_dir
0327               comp.mother_dir=comp.mother_dir.replace("/")
0328           if not comp in self.comparisons:  #in case not to  append same comparisons few times
0329               self.comparisons.append(comp)  # add a comparison
0330               self.n_comp_fails = exp_dir.n_comp_fails  #copy to-be removed directory
0331               self.n_comp_nulls = exp_dir.n_comp_nulls  # numbers to parent directory
0332               self.n_comp_successes = exp_dir.n_comp_successes
0333               self.n_comp_skiped = exp_dir.n_comp_skiped
0334
0335       del self.subdirs[exp_index]
0336       self.prune(expandable_dir)
0337
0338     for subdir in self.subdirs:
0339       subdir.prune(expandable_dir)
0340
0341   def __repr__(self):
0342     if self.is_empty():
0343       return "%s seems to be empty. Please check!" %self.name
0344     content="%s , Rates: Success %.2f%%(%s) - Fail %.2f%%(%s) - Null %.2f%%(%s)\n" %(self.name,self.get_success_rate(),self.n_successes,self.get_fail_rate(),self.n_fails,self.get_null_rate(),self.n_nulls)
0345     for subdir in self.subdirs:
0346       content+=" %s\n" % subdir
0347     for comp in self.comparisons:
0348       content+=" %s\n" % comp
0349     return content
0350
0351 #-------------------------------------------------------------------------------
0352 from multiprocessing import Process
0353 def print_multi_threaded(canvas,img_name):
0354     canvas.Print(img_name)
0355
0356 tcanvas_print_processes=[]
0357 #-------------------------------------------------------------------------------
0358
0359 class Comparison(Weighted):
0360   canvas_xsize=500
0361   canvas_ysize=400
0362   def __init__(self,name,mother_dir,h1,h2,stat_test,draw_success=False,do_pngs=False, skip=False):
0363     self.name=name
0364     self.png_name="placeholder.png"
0365     self.mother_dir=mother_dir
0366     self.img_name=""
0367     #self.draw_success=draw_success
0368     Weighted.__init__(self,name)
0369
0370     stat_test.set_operands(h1,h2)
0371     if skip:
0372         self.status = SKIPED
0373         self.test_name=stat_test.name
0374         self.test_name=stat_test.name
0375         self.test_thr=stat_test.threshold
0376         self.rank = 0
0377     else:
0378         self.status=stat_test.get_status()
0379         self.rank=stat_test.get_rank()
0380         self.test_name=stat_test.name
0381         self.test_thr=stat_test.threshold
0382         self.do_pngs=do_pngs
0383         self.draw_success=draw_success or not do_pngs
0384         if ((self.status==FAIL or self.status==NULL or self.status == SKIPED or self.draw_success) and self.do_pngs):
0385             self.__make_image(h1,h2)
0386         #self.__make_image(h1,h2)
0387
0388   def __make_img_dir(self):
0389     if not exists(self.mother_dir):
0390       makedirs(self.mother_dir)
0391
0392   def __get_img_name(self):
0393     #self.__make_img_dir()
0394     #print "MOTHER: ",self.mother_dir
0395     self.img_name="%s/%s.png"%(self.mother_dir,self.name)
0396     self.img_name=self.img_name.replace("Run summary","")
0397     self.img_name=self.img_name.replace("/","_")
0398     self.img_name=self.img_name.strip("_")
0399     #print "IMAGE NAME: %s " %self.img_name
0400     return self.img_name
0401
0402   def tcanvas_slow(self,canvas):
0403     #print "About to print %s" %self.img_name
0404     #print_multi_threaded(canvas,self.img_name)
0405     #print "-->Printed"
0406
0407     p = Process(target=print_multi_threaded, args=(canvas,self.img_name))
0408     p.start()
0409     tcanvas_print_processes.append(p)
0410     n_proc=len(tcanvas_print_processes)
0411     if n_proc>3:
0412       p_to_remove=[]
0413       for iprocess in range(0,n_proc):
0414         p=tcanvas_print_processes[iprocess]
0415         p.join()
0416         p_to_remove.append(iprocess)
0417
0418       adjustment=0
0419       for iprocess in p_to_remove:
0420         tcanvas_print_processes.pop(iprocess-adjustment)
0421         adjustment+=1
0422
0423   def __make_image(self,obj1,obj2):
0424     self.img_name=self.__get_img_name()
0425     if self.rank==-1:
0426       return 0
0427
0428     canvas=ROOT.TCanvas(self.name,self.name,Comparison.canvas_xsize,Comparison.canvas_ysize)
0429     objs=(obj1,obj2)
0430
0431     # Add some specifics for the graphs
0432     obj1.SetTitle(self.name)
0433
0434     if obj1.GetNbinsY()!=0 and not "2" in obj1.ClassName() :
0435       obj1 .SetLineWidth(2)
0436       obj2 .SetLineWidth(2)
0437
0438       obj1.SetMarkerStyle(8)
0439       obj1.SetMarkerSize(.8)
0440
0441       obj2.SetMarkerStyle(8)
0442       obj2.SetMarkerSize(.8)
0443
0444       obj1.SetMarkerColor(ROOT.kBlue)
0445       obj1.SetLineColor(ROOT.kBlue)
0446
0447       obj2.SetMarkerColor(ROOT.kRed)
0448       obj2.SetLineColor(ROOT.kRed)
0449
0450       obj1.Draw("EP")
0451       #Statsbox
0452       obj2.Draw("HistSames")
0453       #ROOT.gPad.Update()
0454       #if 'stats' in map(lambda o: o.GetName(),list(ROOT.gPad.GetListOfPrimitives())):
0455         #st = gPad.GetPrimitive("stats")
0456         #st.SetY1NDC(0.575)
0457         #st.SetY2NDC(0.735)
0458         #st.SetLineColor(ROOT.kRed)
0459         #st.SetTextColor(ROOT.kRed)
0460         #print st
0461     else:
0462       obj1.Draw("Colz")
0463       ROOT.gPad.Update()
0464       #if 'stats' in map(lambda o: o.GetName(),list(ROOT.gPad.GetListOfPrimitives())):
0465         #st = ROOT.gPad.GetPrimitive("stats")
0466         #st.SetY1NDC(0.575)
0467         #st.SetY2NDC(0.735)
0468         #st.SetLineColor(ROOT.kRed)
0469         #st.SetTextColor(ROOT.kRed)
0470         #print st
0471       obj2.Draw("ColSame")
0472
0473     # Put together the TLatex for the stat test if possible
0474     color=ROOT.kGreen+2 # which is green, as everybody knows
0475     if self.status==FAIL:
0476       print("This comparison failed %f" %self.rank)
0477       color=ROOT.kRed
0478     elif self.status==NULL:
0479       color=ROOT.kYellow
0480     elif self.status==SKIPED:
0481       color=ROOT.kBlue #check if kBlue exists ;)
0482
0483     lat_text="#scale[.7]{#color[%s]{%s: %2.2f}}" %(color,self.test_name,self.rank)
0484     lat=ROOT.TLatex(.1,.91,lat_text)
0485     lat.SetNDC()
0486     lat.Draw()
0487
0488     # Put also the stats together!
0489     n1=obj1.GetEntries()
0490     if n1> 100000:
0491       n1="%e"%n1
0492     else:
0493       n1="%s"%n1
0494     n2=obj2.GetEntries()
0495     if n2> 100000:
0496       n2="%e"%n2
0497     else:
0498       n2="%s"%n2
0499
0500     lat_text1="#scale[.7]{#color[%s]{Entries: %s}}" %(obj1.GetLineColor(),n1)
0501     lat1=ROOT.TLatex(.3,.91,lat_text1)
0502     lat1.SetNDC()
0503     lat1.Draw()
0504
0505
0506     lat_text2="#scale[.7]{#color[%s]{Entries: %s}}" %(obj2.GetLineColor(),n2)
0507     lat2=ROOT.TLatex(.6,.91,lat_text2)
0508     lat2.SetNDC()
0509     lat2.Draw()
0510
0511
0512     self.tcanvas_slow(canvas)
0513
0514   def __repr__(self):
0515     return "%s , (%s=%s). IMG=%s. status=%s" %(self.name,self.test_name,self.rank,self.img_name,self.status)
0516
0517 #-------------------------------------------------------------------------------