RelMon/python/dirstructure.py

0001 from __future__ import print_function
0002 from __future__ import absolute_import
0003 ################################################################################
0004 # RelMon: a tool for automatic Release Comparison
0005 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0006 #
0007 #
0008 #
0009 # Danilo Piparo CERN - danilo.piparo@cern.ch
0010 #
0011 ################################################################################
0012
0013 from builtins import range
0014 from array import array
0015 from copy import deepcopy
0016 from os import chdir,getcwd,listdir,makedirs,rmdir
0017 from os.path import exists,join
0018
0019 import sys
0020 argv=sys.argv
0021 import ROOT
0022 sys.argv=argv
0023
0024 from .definitions import *
0025 from .utils import setTDRStyle
0026
0027
0028 # Something nice and familiar
0029 setTDRStyle()
0030
0031 # Do not display the canvases
0032 ROOT.gROOT.SetBatch(ROOT.kTRUE)
0033
0034
0035 #-------------------------------------------------------------------------------
0036 _log_level=5
0037 def logger(msg_level,message):
0038   if msg_level>=_log_level:
0039     print("[%s] %s" %(asctime(),message))
0040
0041 #-------------------------------------------------------------------------------
0042
0043 class Weighted(object):
0044   def __init__(self,name,weight=1):
0045     self.name=name
0046     self.weight=weight
0047
0048
0049 #-------------------------------------------------------------------------------
0050 class CompInfo(object):
0051   def __init__(self,sample1="",sample2="",release1="",release2="",run1="",run2="",tier1=0,tier2=0):
0052     self.sample1=sample1
0053     self.sample2=sample2
0054     self.release1=release1
0055     self.release2=release2
0056     self.run1=run1
0057     self.run2=run2
0058     self.tier1=tier1
0059     self.tier2=tier2
0060
0061 #-------------------------------------------------------------------------------
0062 class Directory(Weighted):
0063   def __init__(self,name,mother_dir="",meta=CompInfo(),draw_success=False,do_pngs=False):
0064     self.mother_dir=mother_dir
0065     self.meta=meta
0066     self.subdirs=[]
0067     self.comparisons=[]
0068     self.n_fails=0
0069     self.n_successes=0
0070     self.n_nulls=0
0071     self.n_skiped = 0
0072     self.n_comp_skiped = 0
0073     self.n_comp_fails=0
0074     self.n_comp_successes=0
0075     self.n_comp_nulls=0
0076     self.weight=0
0077     self.stats_calculated=False
0078     Weighted.__init__(self,name)
0079     self.draw_success=draw_success
0080     self.do_pngs=do_pngs
0081     self.rank_histo=ROOT.TH1I("rh%s"%name,"",50,-0.01,1.001)
0082     self.rank_histo.SetDirectory(0)
0083     self.different_histograms = {}
0084     self.different_histograms['file1']= {}
0085     self.different_histograms['file2']= {}
0086     self.filename1 = ""
0087     self.filename2 = ""
0088     self.n_missing_objs = 0
0089     self.full_path = ""
0090
0091   def is_empty(self):
0092     if len(self.subdirs)==0 and len(self.comparisons)==0:
0093       return True
0094     return False
0095
0096   def calcStats(self,make_pie=True):
0097     '''Walk all subdirs and calculate weight,fails and successes.
0098     Moreove propagate the sample and releases names.
0099     '''
0100     if self.stats_calculated:
0101       return 0
0102
0103     self.n_fails=0
0104     self.n_successes=0
0105     self.n_nulls=0
0106     self.n_comp_fails=0
0107     self.n_comp_successes=0
0108     self.n_comp_nulls=0
0109     self.weight=0
0110
0111     self.n_skiped = 0
0112     self.n_comp_skiped = 0
0113     self.n_missing_objs = len(self.different_histograms['file1'])+len(self.different_histograms['file2'])
0114     if self.n_missing_objs != 0:
0115       print("    [*] Missing in %s: %s" %(self.filename1, self.different_histograms['file1']))
0116       print("    [*] Missing in %s: %s" %(self.filename2, self.different_histograms['file2']))
0117     # clean from empty dirs
0118     self.subdirs = [subdir for subdir in self.subdirs if not subdir.is_empty()]
0119
0120     for comp in self.comparisons:
0121       if comp.status == SKIPED: #in case its in black list & skiped
0122           self.n_skiped += 1
0123           self.n_comp_skiped += 1
0124           self.weight+=1
0125       else: #else original code -> to check for Fails and Successes
0126           self.rank_histo.Fill(comp.rank)
0127           self.weight+=1
0128           if comp.status == FAIL:
0129               self.n_fails+=1
0130               self.n_comp_fails+=1
0131           elif comp.status == SUCCESS:
0132               self.n_successes+=1
0133               self.n_comp_successes+=1
0134           else:
0135               self.n_nulls+=1
0136               self.n_comp_nulls+=1
0137
0138     for subdir in self.subdirs:
0139       subdir.mother_dir=join(self.mother_dir,self.name)
0140       subdir.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
0141       subdir.calcStats(make_pie)
0142       subdir.meta=self.meta
0143       self.weight+=subdir.weight
0144       self.n_fails+=subdir.n_fails
0145       self.n_successes+=subdir.n_successes
0146       self.n_nulls+=subdir.n_nulls
0147
0148       self.n_skiped+=subdir.n_skiped
0149       self.n_missing_objs += subdir.n_missing_objs
0150
0151       self.rank_histo.Add(subdir.rank_histo)
0152
0153     self.stats_calculated=True
0154     self.full_path = join(self.mother_dir,self.name).replace("/Run summary","")
0155     #if make_pie:
0156       #self.__create_pie_image()
0157
0158   def get_subdirs_dict(self):
0159     subdirdict={}
0160     for subdir in self.subdirs:
0161       subdirdict[subdir.name]=subdir
0162     return subdirdict
0163
0164   def get_subdirs_names(self):
0165     subdirnames=[]
0166     for subdir in self.subdirs:
0167       subdirnames.append(subdir.name)
0168     return subdirnames
0169
0170   def get_summary_chart_ajax(self,w=400,h=300):
0171     """Emit the ajax to build a pie chart using google apis...
0172     """
0173     url = "https://chart.googleapis.com/chart?"
0174     url+= "cht=p3" # Select the 3d chart
0175     #url+= "&chl=Success|Null|Fail" # give labels
0176     url+= "&chco=00FF00|FFFF00|FF0000|7A7A7A" # give colours to labels
0177     url+= "&chs=%sx%s" %(w,h)
0178     #url+= "&chtt=%s" %self.name
0179     url+= "&chd=t:%.2f,%.2f,%.2f,%.2f"%(self.get_success_rate(),self.get_null_rate(),self.get_fail_rate(),self.get_skiped_rate())
0180
0181     return url
0182
0183   def print_report(self,indent="",verbose=False):
0184     if len(indent)==0:
0185       self.calcStats(make_pie=False)
0186     # print small failure report
0187     if verbose:
0188       fail_comps=[comp for comp in self.comparisons if comp.status==FAIL]
0189       fail_comps=sorted(fail_comps,key=lambda comp:comp.name )
0190       if len(fail_comps)>0:
0191         print(indent+"* %s/%s:" %(self.mother_dir,self.name))
0192         for comp in fail_comps:
0193           print(indent+" - %s: %s Test Failed (pval = %s) " %(comp.name,comp.test_name,comp.rank))
0194       for subdir in self.subdirs:
0195         subdir.print_report(indent+"  ",verbose)
0196
0197     if len(indent)==0:
0198       print("\n%s - summary of %s tests:" %(self.name,self.weight))
0199       print(" o Failiures: %.2f%% (%s/%s)" %(self.get_fail_rate(),self.n_fails,self.weight))
0200       print(" o Nulls: %.2f%% (%s/%s) " %(self.get_null_rate(),self.n_nulls,self.weight))
0201       print(" o Successes: %.2f%% (%s/%s) " %(self.get_success_rate(),self.n_successes,self.weight))
0202       print(" o Skipped: %.2f%% (%s/%s) " %(self.get_skiped_rate(),self.n_skiped,self.weight))
0203       print(" o Missing objects: %s" %(self.n_missing_objs))
0204
0205   def get_skiped_rate(self):
0206     if self.weight == 0: return 0
0207     return 100.*self.n_skiped/self.weight
0208   def get_fail_rate(self):
0209     if self.weight == 0:return 0
0210     return 100.*self.n_fails/self.weight
0211
0212   def get_success_rate(self):
0213     if self.weight == 0:return 1
0214     return 100.*self.n_successes/self.weight
0215
0216   def get_null_rate(self):
0217     if self.weight == 0:return 0
0218     return 100.*self.n_nulls/self.weight
0219
0220   def __get_full_path(self):
0221     #print "Mother is %s" %self.mother_dir
0222     if len(self.mother_dir)==0:
0223       return self.name
0224     return join(self.mother_dir,self.name)
0225
0226   def __create_on_disk(self):
0227     if not exists(self.mother_dir) and len(self.mother_dir)!=0:
0228       makedirs(self.mother_dir)
0229     full_path=self.__get_full_path()
0230     if not exists(full_path) and len(full_path)>0:
0231       makedirs(full_path)
0232
0233   def get_summary_chart_name(self):
0234     return join(self.__get_full_path(),"summary_chart.png")
0235
0236   def __create_pie_image(self):
0237     self.__create_on_disk()
0238     vals=[]
0239     colors=[]
0240     for n,col in zip((self.n_fails,self.n_nulls,self.n_successes,self.n_skiped),(ROOT.kRed,ROOT.kYellow,ROOT.kGreen,ROOT.kBlue)):
0241       if n!=0:
0242         vals.append(n)
0243         colors.append(col)
0244     valsa=array('f',vals)
0245     colorsa=array('i',colors)
0246     can = ROOT.TCanvas("cpie","TPie test",100,100);
0247     try:
0248       pie = TPie("ThePie",self.name,len(vals),valsa,colorsa);
0249       label_n=0
0250       if self.n_fails!=0:
0251         pie.SetEntryLabel(label_n, "Fail: %.1f(%i)" %(self.get_fail_rate(),self.n_fails) );
0252         label_n+=1
0253       if self.n_nulls!=0:
0254         pie.SetEntryLabel(label_n, "Null: %.1f(%i)" %(self.get_null_rate(),self.n_nulls) );
0255         label_n+=1
0256       if self.n_successes!=0:
0257         pie.SetEntryLabel(label_n, "Success: %.1f(%i)" %(self.get_success_rate(),self.n_successes) );
0258       if self.n_skiped!=0:
0259         pie.SetEntryLabel(label_n, "Skipped: %.1f(%i)" %(self.get_skiped_rate(),self.n_skiped));
0260       pie.SetY(.52);
0261       pie.SetAngularOffset(0.);
0262       pie.SetLabelsOffset(-.3);
0263       #pie.SetLabelFormat("#splitline{%val (%perc)}{%txt}");
0264       pie.Draw("3d  nol");
0265       can.Print(self.get_summary_chart_name());
0266     except:
0267       print("self.name = %s" %self.name)
0268       print("len(vals) = %s (vals=%s)" %(len(vals),vals))
0269       print("valsa = %s" %valsa)
0270       print("colorsa = %s" %colorsa)
0271
0272   def prune(self,expandable_dir):
0273     """Eliminate from the tree the directory the expandable ones.
0274     """
0275     #print "pruning %s" %self.name
0276     exp_index=-1
0277     counter=0
0278     for subdir in self.subdirs:
0279       # Eliminate any trace of the expandable path in the mother directories
0280       # for depths higher than 1
0281       subdir.mother_dir=subdir.mother_dir.replace("/"+expandable_dir,"")
0282       if subdir.name==expandable_dir:
0283         exp_index=counter
0284       counter+=1
0285
0286     # Did we find an expandable?
0287     if exp_index>=0:
0288       exp_dir=self.subdirs[exp_index]
0289       for subsubdir in exp_dir.subdirs:
0290         #print "*******",subsubdir.mother_dir,
0291         subsubdir.mother_dir=subsubdir.mother_dir.replace("/"+expandable_dir,"")
0292         while "//" in subsubdir.mother_dir:
0293           print(subsubdir.mother_dir)
0294           subsubdir.mother_dir=subsubdir.mother_dir.replace("//","/")
0295         #print "*******",subsubdir.mother_dir
0296         self.subdirs.append(subsubdir)
0297
0298         for comp in exp_dir.comparisons:
0299           comp.mother_dir=comp.mother_dir.replace("/"+expandable_dir,"")
0300           while "//" in comp.mother_dir:
0301               comp.mother_dir
0302               comp.mother_dir=comp.mother_dir.replace("/")
0303           if not comp in self.comparisons:  #in case not to  append same comparisons few times
0304               self.comparisons.append(comp)  # add a comparison
0305               self.n_comp_fails = exp_dir.n_comp_fails  #copy to-be removed directory
0306               self.n_comp_nulls = exp_dir.n_comp_nulls  # numbers to parent directory
0307               self.n_comp_successes = exp_dir.n_comp_successes
0308               self.n_comp_skiped = exp_dir.n_comp_skiped
0309
0310       del self.subdirs[exp_index]
0311       self.prune(expandable_dir)
0312
0313     for subdir in self.subdirs:
0314       subdir.prune(expandable_dir)
0315
0316   def __repr__(self):
0317     if self.is_empty():
0318       return "%s seems to be empty. Please check!" %self.name
0319     content="%s , Rates: Success %.2f%%(%s) - Fail %.2f%%(%s) - Null %.2f%%(%s)\n" %(self.name,self.get_success_rate(),self.n_successes,self.get_fail_rate(),self.n_fails,self.get_null_rate(),self.n_nulls)
0320     for subdir in self.subdirs:
0321       content+=" %s\n" % subdir
0322     for comp in self.comparisons:
0323       content+=" %s\n" % comp
0324     return content
0325
0326 #-------------------------------------------------------------------------------
0327 from multiprocessing import Process
0328 def print_multi_threaded(canvas,img_name):
0329     canvas.Print(img_name)
0330
0331 tcanvas_print_processes=[]
0332 #-------------------------------------------------------------------------------
0333
0334 class Comparison(Weighted):
0335   canvas_xsize=500
0336   canvas_ysize=400
0337   def __init__(self,name,mother_dir,h1,h2,stat_test,draw_success=False,do_pngs=False, skip=False):
0338     self.name=name
0339     self.png_name="placeholder.png"
0340     self.mother_dir=mother_dir
0341     self.img_name=""
0342     #self.draw_success=draw_success
0343     Weighted.__init__(self,name)
0344
0345     stat_test.set_operands(h1,h2)
0346     if skip:
0347         self.status = SKIPED
0348         self.test_name=stat_test.name
0349         self.test_name=stat_test.name
0350         self.test_thr=stat_test.threshold
0351         self.rank = 0
0352     else:
0353         self.status=stat_test.get_status()
0354         self.rank=stat_test.get_rank()
0355         self.test_name=stat_test.name
0356         self.test_thr=stat_test.threshold
0357         self.do_pngs=do_pngs
0358         self.draw_success=draw_success or not do_pngs
0359         if ((self.status==FAIL or self.status==NULL or self.status == SKIPED or self.draw_success) and self.do_pngs):
0360             self.__make_image(h1,h2)
0361         #self.__make_image(h1,h2)
0362
0363   def __make_img_dir(self):
0364     if not exists(self.mother_dir):
0365       makedirs(self.mother_dir)
0366
0367   def __get_img_name(self):
0368     #self.__make_img_dir()
0369     #print "MOTHER: ",self.mother_dir
0370     self.img_name="%s/%s.png"%(self.mother_dir,self.name)
0371     self.img_name=self.img_name.replace("Run summary","")
0372     self.img_name=self.img_name.replace("/","_")
0373     self.img_name=self.img_name.strip("_")
0374     #print "IMAGE NAME: %s " %self.img_name
0375     return self.img_name
0376
0377   def tcanvas_slow(self,canvas):
0378     #print "About to print %s" %self.img_name
0379     #print_multi_threaded(canvas,self.img_name)
0380     #print "-->Printed"
0381
0382     p = Process(target=print_multi_threaded, args=(canvas,self.img_name))
0383     p.start()
0384     tcanvas_print_processes.append(p)
0385     n_proc=len(tcanvas_print_processes)
0386     if n_proc>3:
0387       p_to_remove=[]
0388       for iprocess in range(0,n_proc):
0389         p=tcanvas_print_processes[iprocess]
0390         p.join()
0391         p_to_remove.append(iprocess)
0392
0393       adjustment=0
0394       for iprocess in p_to_remove:
0395         tcanvas_print_processes.pop(iprocess-adjustment)
0396         adjustment+=1
0397
0398   def __make_image(self,obj1,obj2):
0399     self.img_name=self.__get_img_name()
0400     if self.rank==-1:
0401       return 0
0402
0403     canvas=ROOT.TCanvas(self.name,self.name,Comparison.canvas_xsize,Comparison.canvas_ysize)
0404     objs=(obj1,obj2)
0405
0406     # Add some specifics for the graphs
0407     obj1.SetTitle(self.name)
0408
0409     if obj1.GetNbinsY()!=0 and not "2" in obj1.ClassName() :
0410       obj1 .SetLineWidth(2)
0411       obj2 .SetLineWidth(2)
0412
0413       obj1.SetMarkerStyle(8)
0414       obj1.SetMarkerSize(.8)
0415
0416       obj2.SetMarkerStyle(8)
0417       obj2.SetMarkerSize(.8)
0418
0419       obj1.SetMarkerColor(ROOT.kBlue)
0420       obj1.SetLineColor(ROOT.kBlue)
0421
0422       obj2.SetMarkerColor(ROOT.kRed)
0423       obj2.SetLineColor(ROOT.kRed)
0424
0425       obj1.Draw("EP")
0426       #Statsbox
0427       obj2.Draw("HistSames")
0428       #ROOT.gPad.Update()
0429       #if 'stats' in map(lambda o: o.GetName(),list(ROOT.gPad.GetListOfPrimitives())):
0430         #st = gPad.GetPrimitive("stats")
0431         #st.SetY1NDC(0.575)
0432         #st.SetY2NDC(0.735)
0433         #st.SetLineColor(ROOT.kRed)
0434         #st.SetTextColor(ROOT.kRed)
0435         #print st
0436     else:
0437       obj1.Draw("Colz")
0438       ROOT.gPad.Update()
0439       #if 'stats' in map(lambda o: o.GetName(),list(ROOT.gPad.GetListOfPrimitives())):
0440         #st = ROOT.gPad.GetPrimitive("stats")
0441         #st.SetY1NDC(0.575)
0442         #st.SetY2NDC(0.735)
0443         #st.SetLineColor(ROOT.kRed)
0444         #st.SetTextColor(ROOT.kRed)
0445         #print st
0446       obj2.Draw("ColSame")
0447
0448     # Put together the TLatex for the stat test if possible
0449     color=ROOT.kGreen+2 # which is green, as everybody knows
0450     if self.status==FAIL:
0451       print("This comparison failed %f" %self.rank)
0452       color=ROOT.kRed
0453     elif self.status==NULL:
0454       color=ROOT.kYellow
0455     elif self.status==SKIPED:
0456       color=ROOT.kBlue #check if kBlue exists ;)
0457
0458     lat_text="#scale[.7]{#color[%s]{%s: %2.2f}}" %(color,self.test_name,self.rank)
0459     lat=ROOT.TLatex(.1,.91,lat_text)
0460     lat.SetNDC()
0461     lat.Draw()
0462
0463     # Put also the stats together!
0464     n1=obj1.GetEntries()
0465     if n1> 100000:
0466       n1="%e"%n1
0467     else:
0468       n1="%s"%n1
0469     n2=obj2.GetEntries()
0470     if n2> 100000:
0471       n2="%e"%n2
0472     else:
0473       n2="%s"%n2
0474
0475     lat_text1="#scale[.7]{#color[%s]{Entries: %s}}" %(obj1.GetLineColor(),n1)
0476     lat1=ROOT.TLatex(.3,.91,lat_text1)
0477     lat1.SetNDC()
0478     lat1.Draw()
0479
0480
0481     lat_text2="#scale[.7]{#color[%s]{Entries: %s}}" %(obj2.GetLineColor(),n2)
0482     lat2=ROOT.TLatex(.6,.91,lat_text2)
0483     lat2.SetNDC()
0484     lat2.Draw()
0485
0486
0487     self.tcanvas_slow(canvas)
0488
0489   def __repr__(self):
0490     return "%s , (%s=%s). IMG=%s. status=%s" %(self.name,self.test_name,self.rank,self.img_name,self.status)
0491
0492 #-------------------------------------------------------------------------------