RelMon/scripts/ValidationMatrix.py

0001 #! /usr/bin/env python3
0002 ################################################################################
0003 # RelMon: a tool for automatic Release Comparison
0004 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
0005 #
0006 #
0007 #
0008 # Danilo Piparo CERN - danilo.piparo@cern.ch
0009 #
0010 ################################################################################
0011
0012 from __future__ import print_function
0013 from builtins import range
0014 from optparse import OptionParser
0015
0016 import os
0017 import pickle
0018 import glob
0019 from re import search
0020 from subprocess import call,PIPE
0021 from multiprocessing import Pool
0022 from sys import exit
0023
0024 import sys
0025 argv=sys.argv
0026 sys.argv=[]
0027 if "RELMON_SA" in os.environ:
0028   import definitions as definitions
0029   from dqm_interfaces import DirWalkerFile,string2blacklist,DirWalkerFile_thread_wrapper
0030   from dirstructure import Directory
0031   from directories2html import directory2html,make_summary_table
0032   from utils import ask_ok, unpickler, make_files_pairs
0033 else:
0034   import Utilities.RelMon.definitions as definitions
0035   from Utilities.RelMon.dqm_interfaces import DirWalkerFile,string2blacklist,DirWalkerFile_thread_wrapper
0036   from Utilities.RelMon.dirstructure import Directory
0037   from Utilities.RelMon.directories2html import directory2html,make_summary_table
0038   from Utilities.RelMon.utils import ask_ok, unpickler, make_files_pairs
0039 sys.argv=argv
0040
0041 #-------------------------------------------------------------------------------
0042
0043 def name2sample(filename):
0044   namebase=os.path.basename(filename)
0045   return namebase.split("__")[1]
0046
0047 def name2version(filename):
0048   namebase=os.path.basename(filename)
0049   return namebase.split("__")[2]
0050
0051 def name2run(filename):
0052   namebase=os.path.basename(filename)
0053   return namebase.split("__")[0].split("_")[2]
0054
0055 def name2runskim(filename):
0056   run=name2run(filename)
0057   skim=name2version(filename).split("_")[-1]
0058   # remove skim version
0059   if "-v" in skim:
0060     skim = skim[:skim.rfind('-v')]
0061   return "%s_%s"%(run,skim)
0062
0063 def name2globaltag(filename):
0064   namebase = os.path.basename(filename)
0065   return namebase.split("__")[2].split("-")[1] #returns GT from file basename
0066
0067 #-------------------------------------------------------------------------------
0068
0069 def guess_params(ref_filenames,test_filenames):
0070
0071   if len(ref_filenames)*len(test_filenames)==0:
0072     print("Empty reference and test filenames lists!")
0073     return [],"",""
0074
0075   samples=[]
0076   ref_versions=[]
0077   test_versions=[]
0078
0079   for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
0080
0081     ref_sample=name2sample(ref)
0082     ref_version=name2version(ref)
0083     test_sample=name2sample(test)
0084     test_version=name2version(test)
0085
0086     print("  ## sample 1: %s vs sample 2: %s"%(ref_sample, test_sample))
0087
0088     if ref_sample!=test_sample:
0089       print("Files %s and %s do not seem to be relative to the same sample." %(ref, test))
0090     #  exit(2)
0091
0092     # Slightly modify for data
0093     if search("20[01]",ref_version)!=None:
0094       ref_sample+=ref_version.split("_")[-1]
0095     samples.append(ref_sample)
0096
0097     # append the versions
0098     ref_versions.append(ref_version)
0099     test_versions.append(test_version)
0100
0101   # Check if ref and test versions are always the same.
0102   ref_versions=list(set(ref_versions))
0103   test_versions=list(set(test_versions))
0104
0105   #for versions in ref_versions,test_versions:
0106     #if len(versions)!=1:
0107       #print "More than one kind of CMSSW version selected (%s)" %versions
0108       #exit(2)
0109
0110   cmssw_version1=ref_versions[0]
0111   cmssw_version2=test_versions[0]
0112
0113   return samples,cmssw_version1,cmssw_version2
0114
0115
0116 #-------------------------------------------------------------------------------
0117
0118 def check_root_files(names_list):
0119   for name in names_list:
0120     if not name.endswith(".root"):
0121       print("File %s does not seem to be a rootfile. Please check.")
0122       return False
0123   return True
0124
0125 #-------------------------------------------------------------------------------
0126
0127 def add_to_blacklist(blacklist, pattern, target, blist_piece):
0128   int_pattern=pattern
0129   int_pattern=pattern.strip()
0130   flip_condition=False
0131   if int_pattern[0]=='!':
0132     int_pattern=int_pattern[1:]
0133     flip_condition=True
0134
0135   condition = search(int_pattern,target)!=None
0136   if flip_condition:
0137     condition = not condition
0138
0139   if condition:
0140     #print "Found %s in %s" %(pattern,target)
0141     if blacklist!="": # if not the first, add a comma
0142       blacklist+=","
0143     blacklist+=blist_piece
0144   #else:
0145     #print "  NOT Found %s in %s" %(pattern,target)
0146   return blacklist
0147
0148 #-------------------------------------------------------------------------------
0149
0150 def guess_blacklists(samples,ver1,ver2,hlt):
0151   """Build a blacklist for each sample accordind to a set of rules
0152   """
0153   blacklists={}
0154   for sample in samples:
0155     blacklists[sample]="FED@1,AlcaBeamMonitor@1,HLT@1,AlCaReco@1"
0156
0157     # HLT
0158     if hlt: #HLT
0159       blacklists[sample]+=",AlCaEcalPi0@2"
0160       if not search("2010+|2011+|2012+|2015+",ver1):
0161         print("We are treating MC files for the HLT")
0162         for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
0163           blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
0164       else:
0165         print("We are treating Data files for the HLT")
0166         # at the moment it does not make sense since hlt is ran already
0167
0168     else: #RECO
0169       #Monte Carlo
0170       if not search("2010+|2011+|2012+",ver1):
0171         print("We are treating MC files")
0172
0173         for pattern,blist in definitions.mc_pattern_blist_pairs:
0174           blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
0175 #          print "MC RECO"
0176           #print blacklists[sample]
0177
0178       # Data
0179       else:
0180         print("We are treating Data files:")
0181         blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"
0182         for pattern,blist in definitions.data_pattern_blist_pairs:
0183           blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
0184 #         print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])
0185
0186
0187   return blacklists
0188
0189 #-------------------------------------------------------------------------------
0190
0191 def get_roofiles_in_dir(directory):
0192   print(directory)
0193   files_list = [s for s in os.listdir(directory) if s.endswith(".root")]
0194   files_list_path=map(lambda s: os.path.join(directory,s), files_list)
0195
0196   return files_list_path
0197
0198 #-------------------------------------------------------------------------------
0199
0200 def get_filenames_from_pool(all_samples):
0201
0202   # get a list of the files
0203   files_list=get_roofiles_in_dir(all_samples)
0204
0205   if len(files_list)==0:
0206     print("Zero files found in directory %s!" %all_samples)
0207     return [],[]
0208
0209   # Are they an even number?
0210   for name in files_list:
0211     print("* ",name)
0212   if len(files_list)%2!=0:
0213     print("The numbuer of file is not even... Trying to recover a catastrophe.")
0214
0215   files_list=make_files_pairs(files_list)
0216
0217   # Try to couple them according to their sample
0218   ref_filenames=[]
0219   test_filenames=[]
0220   #files_list.sort(key=name2version)
0221   #files_list.sort(key=name2sample)
0222   #files_list.sort(key=name2run)
0223   for iname in range(len(files_list)):
0224     filename=files_list[iname]
0225     if iname%2==0:
0226       ref_filenames.append(filename)
0227     else:
0228       test_filenames.append(filename)
0229
0230   print("The guess would be the following:")
0231   for ref,test in zip(ref_filenames,test_filenames):
0232     refbasedir=os.path.dirname(ref)
0233     testbasedir=os.path.dirname(test)
0234     dir_to_print=refbasedir
0235     if refbasedir!=testbasedir:
0236       dir_to_print="%s and %s" %(refbasedir,testbasedir)
0237     print("* Directory: %s " %dir_to_print)
0238     refname=os.path.basename(ref)
0239     testname=os.path.basename(test)
0240     print("  o %s" %refname)
0241     print("  o %s" %testname)
0242
0243   #is_ok=ask_ok("Is that ok?")
0244   #if not is_ok:
0245     #print "Manual input needed then!"
0246     #exit(2)
0247
0248
0249   return ref_filenames,test_filenames
0250
0251
0252 #-------------------------------------------------------------------------------
0253
0254 def get_clean_fileanames(ref_samples,test_samples):
0255   # Process the samples starting from the names
0256   ref_filenames=list(map(lambda s:s.strip(),ref_samples.split(",")))
0257   test_filenames=list(map(lambda s:s.strip(),test_samples.split(",")))
0258
0259   if len(ref_filenames)!=len(test_filenames):
0260     print("The numebr of reference and test files does not seem to be the same. Please check.")
0261     exit(2)
0262
0263   if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
0264     exit(2)
0265   return ref_filenames,test_filenames
0266
0267 #-------------------------------------------------------------------------------
0268
0269 def count_alive_processes(p_list):
0270   return len([p for p in p_list if p.returncode==None])
0271
0272 #-------------------------------------------------------------------------------
0273
0274 def call_compare_using_files(args):
0275   """Creates shell command to compare two files using compare_using_files.py
0276   script and calls it."""
0277   sample, ref_filename, test_filename, options = args
0278   gt = name2globaltag(ref_filename)
0279   blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
0280   command = " compare_using_files.py "
0281   command+= "%s %s " %(ref_filename,test_filename)
0282   command+= " -C -R "
0283   if options.do_pngs:
0284     command+= " -p "
0285   command+= " -o %s_%s " %(sample, gt)
0286   # Change threshold to an experimental and empirical value of 10^-5
0287   command+= " --specify_run "
0288   if options.stat_test in ["Bin2Bin", "BinToBin"]:
0289     options.test_threshold = 0.9999
0290   command+= " -t %s " %options.test_threshold
0291   command+= " -s %s " %options.stat_test
0292
0293   # Inspect the HLT directories
0294   if options.hlt:
0295     command+=" -d HLT "
0296
0297   if options.hash_name:
0298     command += " --hash_name "
0299
0300   if options.blacklist_file:
0301     command += " --use_black_file "
0302
0303   if options.standalone:
0304     command += " --standalone "
0305   if len(blacklists[sample]) >0:
0306     command+= '-B %s ' %blacklists[sample]
0307   print("\nExecuting --  %s" %command)
0308
0309   process=call([x for x in command.split(" ") if len(x)>0])
0310   return process
0311
0312
0313 #--------------------------------------------------------------------------------
0314
0315 def do_comparisons_threaded(options):
0316
0317   n_processes= int(options.n_processes)
0318
0319   ref_filenames=[]
0320   test_filenames=[]
0321
0322   if len(options.all_samples)>0:
0323     ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)
0324   else:
0325     ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
0326
0327   # make the paths absolute
0328   ref_filenames=list(map(os.path.abspath,ref_filenames))
0329   test_filenames=list(map(os.path.abspath,test_filenames))
0330
0331   samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
0332
0333   if len(samples)==0:
0334     print("No Samples found... Quitting")
0335     return 0
0336
0337 #  blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)
0338
0339   # Launch the single comparisons
0340   original_dir=os.getcwd()
0341
0342   outdir=options.out_dir
0343   if len(outdir)==0:
0344     print("Creating automatic outdir:", end=' ')
0345     outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
0346     print(outdir)
0347   if len(options.input_dir)==0:
0348     print("Creating automatic indir:", end=' ')
0349     options.input_dir=outdir
0350     print(options.input_dir)
0351
0352   if not os.path.exists(outdir):
0353     os.mkdir(outdir)
0354   os.chdir(outdir)
0355
0356   # adjust the number of threads
0357   n_comparisons=len(ref_filenames)
0358   if n_comparisons < n_processes:
0359     print("Less comparisons than possible processes: reducing n processes to", end=' ')
0360     n_processes=n_comparisons
0361   #elif n_processes/n_comparisons == 0:
0362     #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",
0363     #original_nprocesses=n_processes
0364     #first=True
0365     #n_bunches=0
0366     #while first or n_processes > original_nprocesses:
0367       #n_processes=n_comparisons/2
0368       #if n_comparisons%2 !=0:
0369         #n_processes+=1
0370       #first=False
0371
0372     #print n_processes
0373   #print n_processes
0374
0375   # Test if we treat data
0376   skim_name=""
0377   if search("20[01]",cmssw_version1)!=None:
0378     skim_name=cmssw_version1.split("_")[-1]
0379
0380   running_subprocesses=[]
0381   process_counter=0
0382   #print ref_filenames
0383
0384   ## Compare all pairs of root files
0385   pool = Pool(n_processes)
0386   args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
0387   pool.map(call_compare_using_files, args_iterable)
0388   # move the pickles on the top, hack
0389   os.system("mv */*pkl .")
0390
0391   os.chdir("..")
0392 #-------------------------------------------------------------------------------
0393 def do_reports(indir):
0394   #print indir
0395   os.chdir(indir)
0396   pkl_list=[x for x in os.listdir("./") if ".pkl" in x]
0397   running_subprocesses=[]
0398   n_processes=int(options.n_processes)
0399   process_counter=0
0400   for pklfilename in pkl_list:
0401     command = "compare_using_files.py "
0402     command+= "-R "
0403     if options.do_pngs:
0404       command+= " -p "
0405     command+= "-P %s " %pklfilename
0406     command+= "-o %s " %pklfilename[:-4]
0407     print("Executing %s" %command)
0408     process=call([x for x in command.split(" ") if len(x)>0])
0409     process_counter+=1
0410     # add it to the list
0411     running_subprocesses.append(process)
0412     if process_counter>=n_processes:
0413       process_counter=0
0414       for p in running_subprocesses:
0415         #print "Waiting for %s" %p.name
0416         p.wait()
0417
0418   os.chdir("..")
0419
0420 #-------------------------------------------------------------------------------
0421 def do_html(options, hashing_flag, standalone):
0422
0423   if options.reports:
0424     print("Preparing reports for the single files...")
0425     do_reports(options.input_dir)
0426   # Do the summary page
0427   aggregation_rules={}
0428   aggregation_rules_twiki={}
0429   # check which aggregation rules are to be used
0430   if options.hlt:
0431     print("Aggregating directories according to HLT rules")
0432     aggregation_rules=definitions.aggr_pairs_dict['HLT']
0433     aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
0434   else:
0435     aggregation_rules=definitions.aggr_pairs_dict['reco']
0436     aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
0437   table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone)
0438
0439   # create summary html file
0440   ofile = open("RelMonSummary.html","w")
0441   ofile.write(table_html)
0442   ofile.close()
0443
0444 #-------------------------------------------------------------------------------
0445
0446 if __name__ == "__main__":
0447
0448   #-----------------------------------------------------------------------------
0449   ref_samples=""
0450   test_samples=""
0451   all_samples=""
0452   n_processes=1
0453   out_dir=""
0454   in_dir=""
0455   n_threads=1 # do not change this
0456   run=-1
0457   stat_test="Chi2"
0458   test_threshold=0.00001
0459   hlt=False
0460   #-----------------------------------------------------------------------------
0461
0462
0463   parser = OptionParser(usage="usage: %prog [options]")
0464
0465   parser.add_option("-R","--ref_samples ",
0466                     action="store",
0467                     dest="ref_samples",
0468                     default=ref_samples,
0469                     help="The samples that act as reference (comma separated list)")
0470
0471   parser.add_option("-T","--test_samples",
0472                     action="store",
0473                     dest="test_samples",
0474                     default=test_samples,
0475                     help="The samples to be tested (comma separated list)")
0476
0477   parser.add_option("-a","--all_samples",
0478                     action="store",
0479                     dest="all_samples",
0480                     default=all_samples,
0481                     help="EXPERIMENTAL: Try to sort all samples selected (wildacrds) and organise a comparison")
0482
0483   parser.add_option("-o","--out_dir",
0484                     action="store",
0485                     dest="out_dir",
0486                     default=out_dir,
0487                     help="The outdir other than <Version1>VS<Version2>")
0488
0489   parser.add_option("-p","--do_pngs",
0490                     action="store_true",
0491                     dest="do_pngs",
0492                     default=False,
0493                     help="EXPERIMENTAL!!! Do the pngs of the comparison (takes 50%% of the total running time) \n(default is %s)" %False)
0494
0495   parser.add_option("-r","--run ",
0496                     action="store",
0497                     dest="run",
0498                     default=run,
0499                     help="The run to be checked \n(default is %s)" %run)
0500
0501   parser.add_option("-t","--test_threshold",
0502                     action="store",
0503                     dest="test_threshold",
0504                     default=test_threshold,
0505                     help="Threshold for the statistical test \n(default is %s)" %test_threshold)
0506
0507   parser.add_option("-s","--stat_test",
0508                     action="store",
0509                     dest="stat_test",
0510                     default=stat_test,
0511                     help="Statistical test (KS or Chi2) \n(default is %s)" %stat_test)
0512
0513   parser.add_option("-N","--numberOfProcesses",
0514                     action="store",
0515                     dest="n_processes",
0516                     default=n_processes,
0517                     help="Number of parallel processes to be run. Be Polite! \n(default is %s)" %n_processes)
0518
0519   parser.add_option("--HLT",
0520                     action="store_true",
0521                     dest="hlt",
0522                     default=False,
0523                     help="Analyse HLT histograms\n(default is %s)" %hlt)
0524
0525   parser.add_option("-i","--input_dir",
0526                     action="store",
0527                     dest="input_dir",
0528                     default=in_dir,
0529                     help="Input directory for html creation \n(default is %s)" %in_dir)
0530
0531   parser.add_option("--reports",
0532                     action="store_true",
0533                     dest="reports",
0534                     default=False,
0535                     help="Do the reports for the pickles \n(default is %s)" %in_dir)
0536 ##---HASHING---##
0537   parser.add_option("--hash_name",
0538                     action="store_true",
0539                     dest="hash_name",
0540                     default=False,
0541                     help="Set if you want to minimize & hash the output HTML files.")
0542 ##--Blacklist File --##
0543   parser.add_option("--use_black_file",
0544                     action="store_true",
0545                     dest="blacklist_file",
0546                     default=False,
0547                     help="Use a black list file of histograms located @ /RelMon/data")
0548 ##-- USE CSS files in web access, for stand-alone usage --##
0549   parser.add_option("--standalone",
0550                   action="store_true",
0551                   dest="standalone",
0552                   default=False,
0553                   help="Define that using RelMon in standalone method. Makes CSS files accessible over HTTP")
0554
0555   (options, args) = parser.parse_args()
0556
0557   if len(options.test_samples)*len(options.ref_samples)+len(options.all_samples)==0 and len(options.input_dir)==0:
0558     print("No samples given as input.")
0559     parser.print_help()
0560     exit(2)
0561
0562   if len(options.all_samples)>0 or (len(options.ref_samples)*len(options.test_samples)>0):
0563     do_comparisons_threaded(options)
0564   if len(options.input_dir)>0:
0565     do_html(options, options.hash_name, options.standalone)
0566
0567
0568
0569
0570
0571
0572
0573
0574
0575
0576
0577