Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2024-11-26 02:34:35

0001 ################################################################################
0002 # RelMon: a tool for automatic Release Comparison                              
0003 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0004 #
0005 #
0006 #                                                                              
0007 # Danilo Piparo CERN - danilo.piparo@cern.ch                                   
0008 #                                                                              
0009 ################################################################################
0010 
0011 from builtins import range
0012 from copy import deepcopy
0013 from os import chdir,getcwd,makedirs
0014 from os.path import abspath,exists,join, basename
0015 from re import sub,search
0016 from re import compile as recompile
0017 from sys import exit,stderr,version_info
0018 from threading import Thread,activeCount
0019 from time import sleep
0020 if version_info[0]==2:
0021   from urllib2  import Request,build_opener,urlopen
0022 else:
0023   from urllib.request  import Request,build_opener,urlopen
0024 
0025 import sys
0026 argv=sys.argv
0027 import ROOT
0028 sys.argv=argv
0029 
0030 ROOT.gROOT.SetBatch(True)
0031 
0032 from .authentication import X509CertOpen
0033 from .dirstructure import Comparison,Directory,tcanvas_print_processes
0034 from .utils import Chi2,KS,BinToBin,Statistical_Tests,literal2root
0035 
0036 #-------------------------------------------------------------------------------  
0037 
0038 class Error(Exception):
0039     """Base class for exceptions in this module."""
0040     pass
0041 
0042 class DQM_DB_Communication(Error):
0043     """Exception occurs in case of problems of communication with the server.
0044     """
0045     def __init__(self,msg):
0046         self.msg = msg
0047 
0048 class InvalidNumberOfArguments(Error):
0049 
0050     def __init__(self,msg):
0051         self.msg = msg
0052 
0053 #-----------------------------------------------------------------------------    
0054 
0055 class DQMcommunicator(object):
0056 
0057     """Communicate with the DQM Document server"""
0058 
0059     #-----------------------------------------------------------------------------
0060 
0061     base_dir='/data/json/archive/'
0062 
0063     def __init__(self,
0064                  server,
0065                  is_private=False,
0066                  ident="DQMToJson/1.0 python/%d.%d.%d" % version_info[:3]):
0067         self.ident = ident
0068         self.server = server
0069         self.is_private = is_private
0070         self.DQMpwd=DQMcommunicator.base_dir
0071         self.prevDQMpwd=self.DQMpwd
0072         self.opener=None
0073         if not self.is_private:
0074             self.opener=build_opener(X509CertOpen())
0075     #-----------------------------------------------------------------------------
0076 
0077     def open_url(self,url):
0078         url=url.replace(' ','%20')
0079         datareq = Request(url)
0080         datareq.add_header('User-agent', self.ident)    
0081         url_obj=0
0082         if not self.is_private:
0083             url_obj=self.opener.open(datareq)   
0084             #url_obj=build_opener(X509CertOpen()).open(datareq) 
0085         else:
0086             url_obj=urlopen(datareq)
0087 
0088         return url_obj
0089 
0090     #-----------------------------------------------------------------------------
0091 
0092     def get_data(self, full_url):
0093         #print "getting data from %s" %full_url
0094         data = self.open_url(full_url).read()
0095 
0096         data = sub("-inf", '0', data)
0097         data = sub("\\s+inf", '0', data)
0098         data = sub("\\s+nan", '0', data)
0099         data = sub('""(CMSSW.*?)""', '"\\1"', data)
0100 
0101         return data
0102 
0103     #-----------------------------------------------------------------------------
0104 
0105     def ls_url(self, url):
0106         url=url.replace(" ","%20")
0107         url=self.server+url
0108         #print "listing "+url
0109         form_folder={}
0110         raw_folder=None
0111         try:
0112             raw_folder=eval(self.get_data(url))
0113         except:
0114             print("Retrying..")
0115             for ntrials in range(5):
0116                 try:
0117                     if ntrials!=0:
0118                         sleep(2)
0119                     #raw_folder=loads(self.get_data(url))
0120                     raw_folder=eval(self.get_data(url))
0121                     break
0122                 except:
0123                     print("Could not fetch %s. Retrying" %url)
0124 
0125         #raw_folder=loads(self.get_data(url))
0126         for content_dict in raw_folder["contents"]:      
0127             if "subdir" in content_dict:
0128                 form_folder[content_dict["subdir"]]={"type":'dir'}
0129             elif "obj" in content_dict:
0130                 properties=content_dict["properties"]
0131                 obj_name=content_dict["obj"]
0132                 obj_type=properties["type"]
0133                 obj_kind=properties["kind"]
0134                 obj_as_string=''
0135                 if "rootobj" in content_dict:
0136                     obj_as_string=content_dict["rootobj"]
0137                 form_folder[obj_name]={'type':obj_type,'obj_as_string':obj_as_string,"kind":obj_kind}
0138         #for k,v in form_folder.items():
0139             #print "* %s --> %s" %(k,v["type"])
0140 
0141         return form_folder        
0142 
0143     #-----------------------------------------------------------------------------
0144 
0145     def ls(self, url='', fetch_root=False):
0146         if len(url)==0:
0147             url=join(self.DQMpwd,url)
0148 
0149         form_folder={}   
0150 
0151         if fetch_root:
0152             url='%s?rootcontent=1'%url
0153         form_folder=self.ls_url(url)
0154 
0155         return form_folder
0156 
0157     #-----------------------------------------------------------------------------
0158 
0159     def cd(self, *args):
0160         len_args=len(args)
0161         full_url=""
0162         if len_args!=1 and len_args!=3:
0163             raise InvalidNumberOfArguments
0164         if len_args==3:
0165             dataset, run, folder = args    
0166             full_url='%s/data/json/archive/%s/%s/%s' % (self.server, dataset, run, folder)
0167         if len_args==1:
0168             folder=args[0]
0169             if folder==self.DQMpwd:
0170                 full_url=self.DQMpwd
0171             elif folder=="..":
0172                 full_url=self.DQMpwd[:self.DQMpwd.rfind("/")]
0173             elif folder=="-":
0174                 full_url=self.oldDQMpwd
0175             elif folder=="":
0176                 full_url=DQMcommunicator.base_dir
0177             else:
0178                 full_url=self.DQMpwd+"/"+folder
0179 
0180         full_url=full_url.replace(' ','%20')
0181         #print "cd: "+full_url
0182 
0183         self.oldDQMpwd=self.DQMpwd
0184         self.DQMpwd=full_url   
0185         #print "In %s" %self.DQMpwd
0186 
0187     #-----------------------------------------------------------------------------
0188 
0189     def get_samples(self, samples_string="*"):
0190         """
0191         A sample contains, among the other things, a data type, a dataset name 
0192         and a run.
0193         """
0194         full_url='%s/data/json/samples?match=%s' % (self.server, samples_string)
0195         samples_dict=eval(self.get_data(full_url))
0196         return samples_dict["samples"]
0197 
0198     #-----------------------------------------------------------------------------
0199 
0200     def get_datasets_list(self, dataset_string=""):
0201         samples_list=self.get_samples(dataset_string)    
0202         datasets_list=[]
0203         for sample in samples_list:
0204             temp_datasets_list =  map(lambda item:item["dataset"] ,sample['items'])
0205             for temp_dataset in temp_datasets_list:
0206                 if not temp_dataset in datasets_list:
0207                     datasets_list.append(temp_dataset)
0208         return datasets_list
0209 
0210     #-----------------------------------------------------------------------------
0211 
0212     def get_RelVal_CMSSW_versions(self,query):
0213         """Get the available cmssw versions for the relvals.
0214         """
0215         relvals_list=self.get_datasets_list(query)
0216         # The samples are of the form /RelValTHISISMYFAVOURITECHANNEL/CMSSW_VERSION/GEN-SIM-WHATEVER-RECO
0217         cmssw_versions_with_duplicates=map (lambda x: x.split("/")[2],relvals_list)
0218         return list(set(cmssw_versions_with_duplicates))
0219 
0220     #-----------------------------------------------------------------------------    
0221 
0222     def get_runs_list(self, dataset_string):
0223         slash="/"
0224         while(dataset_string.endswith(slash) or dataset_string.beginswith(slash)):
0225             dataset_string=dataset_string.strip("/")
0226         samples_list=self.get_samples(dataset_string)
0227         runlist=[]
0228         # Get all the runs in all the items which are in every sample
0229         map( lambda sample: map (lambda item: runlist.append(item['run']), sample['items']), samples_list)
0230         return runlist
0231 
0232     #-----------------------------------------------------------------------------  
0233 
0234     def get_dataset_runs(self,dataset_string):
0235         dataset_runs={}
0236         for dataset in self.get_datasets_list(dataset_string):
0237             dataset_runs[dataset]=self.get_runs_list(dataset)
0238         return dataset_runs
0239 
0240     #-----------------------------------------------------------------------------  
0241 
0242     def get_common_runs(self,dataset_string1,dataset_string2):
0243         set1=set(self.get_runs_list(dataset_string1))
0244         set2=set(self.get_runs_list(dataset_string2))
0245         set1.intersection_update(set2)
0246         return list (set2)
0247 
0248     #-----------------------------------------------------------------------------  
0249 
0250     def get_root_objects_list(self, url=""):
0251         if len(url)==0:
0252             url=self.DQMpwd
0253         else:
0254             url="/"+url    
0255         url = url.replace(" ","%20")
0256         objects=[]
0257         for name,description in self.ls(url,True).items():     
0258             if "dir" not in description["type"]  and "ROOT" in description["kind"]:
0259                 objects.append(literal2root(description["obj_as_string"],description["type"]))
0260         return objects
0261 
0262     #-----------------------------------------------------------------------------  
0263 
0264     def get_root_objects(self, url=""):
0265         if len(url)==0:
0266             url=self.DQMpwd
0267         else:
0268             url=self.server+"/"+url    
0269         url = url.replace(" ","%20")
0270         objects={}
0271         for name,description in self.ls(url,True).items():     
0272             if "dir" not in description["type"] and "ROOT" in description["kind"]:
0273                 objects[name]=literal2root(description["obj_as_string"],description["type"])
0274         return objects
0275 
0276      #-------------------------------------------------------------------------------
0277 
0278     def get_root_objects_list_recursive(self, url=""):
0279         null_url = (len(url)==0)    
0280         if len(url)==0:
0281             url=self.DQMpwd
0282         else:
0283             url="/"+url    
0284         url = url.replace(" ","%20")      
0285         if not null_url: 
0286             self.cd(url)
0287         objects=[]
0288         for name,description in self.ls("",True).items():     
0289             if "dir" in description["type"]:
0290                 objects+=self.get_root_objects_list_recursive(name)
0291                 self.cd("..")
0292             elif  "ROOT" in description["kind"]:
0293                 objects.append(literal2root(description["obj_as_string"],description["type"]))
0294         if not null_url: 
0295             self.cd("..")
0296         return objects
0297 
0298      #-------------------------------------------------------------------------------
0299 
0300     def get_root_objects_names_list_recursive(self, url="",present_url=""):
0301         null_url = (len(url)==0)
0302         if (not null_url):
0303             if len(present_url)==0:
0304                 present_url=url
0305             else:
0306                 present_url+="_%s"%url
0307         if len(url)==0:
0308             url=self.DQMpwd
0309         else:
0310             url="/"+url    
0311         url = url.replace(" ","%20")
0312         if not null_url:
0313             self.cd(url)
0314         objects_names=[]
0315         for name,description in self.ls("",False).items():     
0316             if "dir" in description["type"]:        
0317                 objects_names+=self.get_root_objects_names_list_recursive(name,present_url)
0318                 self.cd("..")
0319             elif  "ROOT" in description["kind"]:
0320                 objects_names.append("%s_%s"%(present_url,name))
0321         if not null_url: 
0322             self.cd("..")
0323         return objects_names
0324 
0325      #-------------------------------------------------------------------------------
0326 
0327     def get_root_objects_recursive(self, url="",present_url=""):
0328         null_url = (len(url)==0)
0329         if (not null_url):
0330             if len(present_url)==0:
0331                 present_url=url
0332             else:
0333                 present_url+="_%s"%url
0334         if len(url)==0:
0335             url=self.DQMpwd
0336         else:
0337             url="/"+url    
0338         url = url.replace(" ","%20")
0339         #if not null_url:
0340         self.cd(url)
0341         objects={}
0342         for name,description in self.ls("",True).items():     
0343             if "dir" in description["type"]:
0344                 objects.update(self.get_root_objects_recursive(name,present_url))
0345                 self.cd("..")
0346             elif  "ROOT" in description["kind"]:
0347                 objects["%s_%s"%(present_url,name)]=literal2root(description["obj_as_string"],description["type"])
0348         #if not null_url:
0349         self.cd("..")
0350         return objects
0351 
0352 #-------------------------------------------------------------------------------
0353 
0354 class DirID(object):
0355     """Structure used to identify a directory in the walked tree,
0356     It carries the name and depth information.
0357     """
0358     def __init__(self,name,depth,mother=""):
0359         self.name=name
0360         self.compname=recompile(name)
0361         self.mother=mother
0362         self.depth=depth
0363     def __eq__(self,dirid):
0364         depth2=dirid.depth
0365         compname2=dirid.compname
0366         name2=dirid.name
0367         is_equal = False
0368         #if self.name in name2 or name2 in self.name:
0369         if search(self.compname,name2)!=None or search(compname2,self.name)!=None:
0370             is_equal = self.depth*depth2 <0 or self.depth==depth2
0371         if len(self.mother)*len(dirid.mother)>0:
0372             is_equal = is_equal and self.mother==dirid.mother
0373         return is_equal
0374 
0375     def __repr__(self):
0376         return "Directory %s at level %s" %(self.name,self.depth)
0377 
0378 #-------------------------------------------------------------------------------
0379 class DirFetcher(Thread):
0380     """ Fetch the content of the single "directory" in the dqm.
0381     """
0382     def __init__ (self,comm,directory):
0383         Thread.__init__(self)
0384         self.comm = comm
0385         self.directory = directory
0386         self.contents=None    
0387     def run(self):
0388         self.contents = self.comm.ls(self.directory,True)
0389 
0390 #-------------------------------------------------------------------------------
0391 
0392 class DirWalkerDB(Thread):
0393     """An interface to the DQM document db. It is threaded to compensate the 
0394     latency introduced by the finite response time of the server.
0395     """
0396     def __init__ (self,comm1,comm2,base1,base2,directory,depth=0,do_pngs=True,stat_test="KS",test_threshold=.5,black_list=[]):
0397         Thread.__init__(self)
0398         self.comm1 = deepcopy(comm1)
0399         self.comm2 = deepcopy(comm2)
0400         self.base1,self.base2 = base1,base2
0401         self.directory = directory
0402         self.depth=depth
0403         self.do_pngs=do_pngs
0404         self.test_threshold=test_threshold
0405         self.stat_test=stat_test
0406         self.black_list=black_list
0407         # name of the thread
0408         self.name+="_%s" %directory.name
0409 
0410     def run(self):
0411 
0412         this_dir=DirID(self.directory.name,self.depth)
0413         if this_dir in self.black_list: 
0414             print("Skipping %s since blacklisted!" %this_dir)
0415             return 0 
0416 
0417         self.depth+=1
0418 
0419         the_test=Statistical_Tests[self.stat_test](self.test_threshold)
0420         #print "Test %s with threshold %s" %(self.stat_test,self.test_threshold)
0421 
0422         directory1=self.base1+"/"+self.directory.mother_dir+"/"+self.directory.name
0423         directory2=self.base2+"/"+self.directory.mother_dir+"/"+self.directory.name
0424 
0425         fetchers =(DirFetcher(self.comm1,directory1),DirFetcher(self.comm2,directory2))
0426         for fetcher in fetchers:
0427             fetcher.start()
0428         for fetcher in fetchers:  
0429             fetcher.join()
0430 
0431         contents1 = fetchers[0].contents
0432         contents2 = fetchers[1].contents
0433         set1= set(contents1.keys())
0434         set2= set(contents2.keys())  
0435 
0436         walkers=[]
0437         self_directory_directories=self.directory.subdirs
0438         self_directory_comparisons=self.directory.comparisons
0439         contents_names=list(set1.intersection(set2))
0440 
0441         for name in contents_names:
0442             content = contents1[name]
0443             if "dir" in content["type"]:
0444                 #if this_dir not in DirWalker.white_list:continue              
0445                 subdir=Directory(name,join(self.directory.mother_dir,self.directory.name))        
0446                 dirwalker=DirWalkerDB(self.comm1,self.comm2,self.base1,self.base2,subdir,self.depth,
0447                                       self.do_pngs,self.stat_test,self.test_threshold,self.black_list)
0448                 dirwalker.start()
0449                 walkers.append(dirwalker)
0450                 n_threads=activeCount()
0451                 if n_threads>5:
0452                     #print >> stderr, "Threads that are running: %s. Joining them." %(n_threads)    
0453                     dirwalker.join()
0454             elif content["kind"]=="ROOT":
0455 #   print directory1,name
0456                 comparison=Comparison(name,
0457                                       join(self.directory.mother_dir,self.directory.name),
0458                                       literal2root(content["obj_as_string"],content["type"]),
0459                                       literal2root(contents2[name]["obj_as_string"],content["type"]),
0460                                       deepcopy(the_test),
0461                                       do_pngs=self.do_pngs)
0462                 self_directory_comparisons.append(comparison)
0463 
0464 
0465         for walker in walkers:
0466             walker.join()
0467             walker_directory=walker.directory
0468             if not walker_directory.is_empty():
0469                 self_directory_directories.append(walker_directory)
0470 
0471 #-------------------------------------------------------------------------------
0472 
0473 class DQMRootFile(object):
0474     """ Class acting as interface between the user and the harvested DQMRootFile.  
0475     It skips the directories created by the DQM infrastructure so to provide an
0476     interface as similar as possible to a real direcory structure and to the 
0477     directory structure provided by the db interface.
0478     """
0479     def __init__(self,rootfilename):
0480         dqmdatadir="DQMData"
0481         self.rootfile=ROOT.TFile(rootfilename)
0482         self.rootfilepwd=self.rootfile.GetDirectory(dqmdatadir)
0483         self.rootfileprevpwd=self.rootfile.GetDirectory(dqmdatadir)
0484         if self.rootfilepwd == None:
0485             print("Directory %s does not exist: skipping. Is this a custom rootfile?" %dqmdatadir)
0486             self.rootfilepwd=self.rootfile
0487             self.rootfileprevpwd=self.rootfile
0488 
0489     def __is_null(self,directory,name):
0490         is_null = not directory
0491         if is_null:
0492             print("Directory %s does not exist!" %name, file=stderr)
0493         return is_null
0494 
0495     def ls(self,directory_name=""):
0496         contents={}
0497         directory=None
0498         if len(directory_name)==0:
0499             directory=self.rootfilepwd      
0500 
0501         directory=self.rootfilepwd.GetDirectory(directory_name)    
0502         if self.__is_null(directory,directory_name):
0503             return contents
0504 
0505         for key in directory.GetListOfKeys():
0506             contents[key.GetName()]=key.GetClassName()
0507         return contents
0508 
0509     def cd(self,directory_name):
0510         """Change the current TDirectoryFile. The familiar "-" and ".." directories 
0511         can be accessed as well.
0512         """
0513         if directory_name=="-":
0514             tmp=self.rootfilepwd
0515             self.rootfilepwd=self.rootfileprevpwd
0516             self.rootfileprevpwd=tmp
0517         if directory_name=="..":
0518             #print "Setting prevpwd"
0519             self.rootfileprevpwd=self.rootfilepwd
0520             #print "The mom"
0521             mom=self.rootfilepwd.GetMotherDir()
0522             #print "In directory +%s+" %self.rootfilepwd
0523             #print "Deleting the TFileDir"
0524             if "Run " not in self.rootfilepwd.GetName():
0525                 self.rootfilepwd.Delete()
0526             #print "Setting pwd to mom"
0527             self.rootfilepwd=mom
0528         else:
0529             new_directory=self.rootfilepwd.GetDirectory(directory_name)
0530             if not self.__is_null(new_directory,directory_name):
0531                 self.rootfileprevpwd=self.rootfilepwd
0532                 self.rootfilepwd=new_directory
0533 
0534     def getObj(self,objname):
0535         """Get a TObject from the rootfile.
0536         """
0537         obj=self.rootfilepwd.Get(objname)
0538         if not self.__is_null(obj,objname):
0539             return obj
0540 
0541 #-------------------------------------------------------------------------------
0542 
0543 class DirWalkerFile(object):
0544     def __init__(self, name, topdirname,rootfilename1, rootfilename2, run=-1, black_list=[], stat_test="KS", test_threshold=.5,draw_success=True,do_pngs=False, black_list_histos=[]):
0545         self.name=name
0546         self.dqmrootfile1=DQMRootFile(abspath(rootfilename1))
0547         self.dqmrootfile2=DQMRootFile(abspath(rootfilename2))
0548         self.run=run
0549         self.stat_test=Statistical_Tests[stat_test](test_threshold)
0550         self.workdir=getcwd()
0551         self.black_list=black_list
0552         self.directory=Directory(topdirname)
0553         #print "DIRWALKERFILE %s %s" %(draw_success,do_pngs)
0554         self.directory.draw_success=draw_success
0555         self.directory.do_pngs=do_pngs
0556         self.black_list_histos = black_list_histos
0557         self.different_histograms = {}
0558         self.filename1 = basename(rootfilename2)
0559         self.filename2 = basename(rootfilename1)
0560 
0561     def __del__(self):
0562         chdir(self.workdir)
0563 
0564     def cd(self,directory_name, on_disk=False, regexp=False,):
0565         if regexp == True:
0566             if len(directory_name)!=0:
0567                 if on_disk:
0568                     if not exists(directory_name):
0569                         makedirs(directory_name)
0570                         chdir(directory_name)  
0571                 tmp = self.dqmrootfile2.ls().keys()
0572                 for elem in tmp:
0573                     if "Run" in elem:
0574                         next_dir = elem
0575                 self.dqmrootfile2.cd(next_dir)
0576                 tmp = self.dqmrootfile1.ls().keys()
0577                 for elem in tmp:
0578                     if "Run" in elem:
0579                         next_dir = elem
0580                 self.dqmrootfile1.cd(next_dir)
0581         else:
0582             if len(directory_name)!=0:
0583                 if on_disk:
0584                     if not exists(directory_name):
0585                         makedirs(directory_name)
0586                         chdir(directory_name)
0587                 self.dqmrootfile2.cd(directory_name)
0588                 self.dqmrootfile1.cd(directory_name)
0589 
0590     def ls(self,directory_name=""):
0591         """Return common objects to the 2 files.
0592         """
0593         contents1=self.dqmrootfile1.ls(directory_name)
0594         contents2=self.dqmrootfile2.ls(directory_name)
0595         #print "cont1: %s"%(contents1)
0596         #print "cont2: %s"%(contents2)
0597         contents={}
0598         self.different_histograms['file1']= {}
0599         self.different_histograms['file2']= {}
0600         keys = [key for key in contents2.keys() if key in contents1] #set of all possible contents from both files
0601         #print " ## keys: %s" %(keys)
0602         for key in keys:  #iterate on all unique keys
0603             if contents1[key]!=contents2[key]:
0604                 diff_file1 = set(contents1.keys()) - set(contents2.keys()) #set of contents that file1 is missing
0605                 diff_file2 = set(contents2.keys()) - set(contents1.keys()) #--'-- that file2 is missing
0606                 for key1 in diff_file1:
0607                     obj_type = contents1[key1]
0608                     if obj_type == "TDirectoryFile":
0609                         self.different_histograms['file1'][key1] = contents1[key1] #if direcory
0610                         #print "\n Missing inside a dir: ", self.ls(key1)
0611                         #contents[key] = contents1[key1]
0612                     if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : #if histogram
0613                         continue
0614                     self.different_histograms['file1'][key1] = contents1[key1]
0615                 for key1 in diff_file2:
0616                     obj_type = contents2[key1]
0617                     if obj_type == "TDirectoryFile":
0618                         self.different_histograms['file2'][key1] = contents2[key1] #if direcory
0619                         #print "\n Missing inside a dir: ", self.ls(key1)
0620                         #contents[key] = contents2[key1]
0621                     if obj_type[:2]!="TH" and obj_type[:3]!="TPr" : #if histogram
0622                         continue
0623                     self.different_histograms['file2'][key1] = contents2[key1]
0624             contents[key]=contents1[key]
0625         return contents
0626 
0627     def getObjs(self,name):
0628         h1=self.dqmrootfile1.getObj(name)
0629         h2=self.dqmrootfile2.getObj(name)
0630         return h1,h2
0631 
0632     def __fill_single_dir(self,dir_name,directory,mother_name="",depth=0):
0633         #print "MOTHER NAME  = +%s+" %mother_name
0634      #print "About to study %s (in dir %s)" %(dir_name,getcwd())
0635 
0636         # see if in black_list
0637         this_dir=DirID(dir_name,depth)
0638         #print "  ## this_dir: %s"%(this_dir)
0639         if this_dir in self.black_list: 
0640             #print "Directory %s skipped because black-listed" %dir_name
0641             return 0        
0642 
0643         depth+=1
0644 
0645         self.cd(dir_name)
0646         #if dir_name == 'HLTJETMET':
0647         #    print self.ls()
0648 
0649         #print "Test %s with thre %s" %(self.stat_test.name, self.stat_test.threshold)
0650 
0651         contents=self.ls()
0652         if depth==1:
0653             n_top_contents=len(contents)
0654 
0655         #print contents
0656         cont_counter=1
0657         comparisons=[]
0658         for name,obj_type in contents.items():
0659             if obj_type=="TDirectoryFile":        
0660                 #We have a dir, launch recursion!
0661                 #Some feedback on the progress
0662                 if depth==1:
0663                     print("Studying directory %s, %s/%s" %(name,cont_counter,n_top_contents))
0664                     cont_counter+=1          
0665 
0666                 #print "Studying directory",name
0667                 # ok recursion on!
0668                 subdir=Directory(name)
0669                 subdir.draw_success=directory.draw_success
0670                 subdir.do_pngs=directory.do_pngs
0671                 self.__fill_single_dir(name,subdir,join(mother_name,dir_name),depth)
0672                 if not subdir.is_empty():
0673                     if depth==1:
0674                         print(" ->Appending %s..." %name, end=' ')
0675                     directory.subdirs.append(subdir)
0676                     if depth==1:
0677                         print("Appended.")
0678             elif name != '':
0679                 # We have probably an histo. Let's make the plot and the png.        
0680                 if obj_type[:2]!="TH" and obj_type[:3]!="TPr" :
0681                     continue
0682                 h1,h2=self.getObjs(name)
0683                 #print "COMPARISON : +%s+%s+" %(mother_name,dir_name)
0684                 path = join(mother_name,dir_name,name)
0685                 if path in self.black_list_histos:
0686                     print("  Skipping %s" %(path))
0687                     directory.comparisons.append(Comparison(name,
0688                                         join(mother_name,dir_name),
0689                                         h1,h2,
0690                                         deepcopy(self.stat_test),
0691                                         draw_success=directory.draw_success,
0692                                         do_pngs=directory.do_pngs, skip=True))
0693                 else:
0694                     directory.comparisons.append(Comparison(name,
0695                                           join(mother_name,dir_name),
0696                                           h1,h2,
0697                                           deepcopy(self.stat_test),
0698                                           draw_success=directory.draw_success,
0699                                           do_pngs=directory.do_pngs, skip=False))
0700                     directory.filename1 = self.filename1
0701                     directory.filename2 = self.filename2
0702                     directory.different_histograms['file1'] = self.different_histograms['file1']
0703                     directory.different_histograms['file2'] = self.different_histograms['file2']
0704 
0705         self.cd("..")
0706 
0707     def walk(self):
0708         # Build the top dir in the rootfile first
0709         rundir=""
0710         if self.run<0:
0711             # change dir in the first one...
0712             #print  self.ls().keys()
0713             first_run_dir = ""
0714             try:
0715                 first_run_dir = list(filter(lambda k: "Run " in k, self.ls().keys()))[0]
0716             except:
0717                 print("\nRundir not there: Is this a generic rootfile?\n")
0718             rundir=first_run_dir
0719             try:
0720                 self.run= int(rundir.split(" ")[1])
0721             except:
0722                 print("Setting run number to 0")
0723                 self.run= 0
0724         else:
0725             rundir="Run %s"%self.run
0726 
0727         try:
0728             self.cd(rundir, False, True) #True -> for checking the Rundir in case of different runs
0729         except:
0730             print("\nRundir not there: Is this a generic rootfile?\n")
0731 
0732         # Let's rock!
0733         self.__fill_single_dir(self.directory.name,self.directory)
0734         print("Finished")
0735         n_left_threads=len(tcanvas_print_processes)
0736         if n_left_threads>0:
0737             print("Waiting for %s threads to finish..." %n_left_threads)
0738             for p in tcanvas_print_processes:
0739                 p.join()  
0740 
0741 #-------------------------------------------------------------------------------
0742 
0743 class DirWalkerFile_thread_wrapper(Thread):
0744     def __init__(self, walker):
0745         Thread.__init__(self)
0746         self.walker=walker
0747     def run(self):
0748         self.walker.walk()
0749 
0750 #-------------------------------------------------------------------------------
0751 
0752 def string2blacklist(black_list_str):
0753     black_list=[]
0754     # replace the + with " ":
0755     black_list_str=black_list_str.replace("__"," ")
0756     if len(black_list_str)>0:
0757         for ele in black_list_str.split(","):
0758             dirname,level=ele.split("@")
0759             level=int(level)
0760             dirid=None
0761             if "/" not in dirname:
0762                 dirid=DirID(dirname,level)
0763             else:
0764                 mother,daughter=dirname.split("/")
0765                 dirid=DirID(daughter,level,mother)
0766             if not dirid in black_list:
0767                 black_list.append(dirid)
0768 
0769     return black_list
0770 
0771 #-------------------------------------------------------------------------------
0772