File indexing completed on 2023-03-17 11:26:50
0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012 from __future__ import print_function
0013 def getInfoFromFilename(filename):
0014 prefix,sample,cmssw_release,tier = filename[:-5].split("__")[:5]
0015 run=int(prefix.split("_")[-1][1:])
0016 return run,sample,cmssw_release,tier
0017
0018 from sys import argv,exit
0019 import os
0020
0021
0022
0023 stat_test="Chi2"
0024 test_threshold=1e-5
0025
0026
0027
0028
0029 dir_name=""
0030 outdir_name=""
0031
0032 compare=False
0033 report=False
0034
0035 do_pngs=False
0036
0037 black_list_str=""
0038
0039
0040
0041 from optparse import OptionParser
0042
0043 parser = OptionParser(usage="usage: %prog file1 file2 [options]")
0044
0045
0046
0047
0048
0049
0050
0051
0052 parser.add_option("-d","--dir_name",
0053 action="store",
0054 dest="dir_name",
0055 default=dir_name,
0056 help="The 'directory' to be checked in the DQM \n(default is %s)" %dir_name)
0057
0058 parser.add_option("-o","--outdir_name",
0059 action="store",
0060 dest="outdir_name",
0061 default=outdir_name,
0062 help="The directory where the output will be stored \n(default is %s)" %outdir_name)
0063
0064 parser.add_option("-p","--do_pngs",
0065 action="store_true",
0066 dest="do_pngs",
0067 default=False,
0068 help="Do the pngs of the comparison (takes 50%% of the total running time) \n(default is %s)" %False)
0069
0070 parser.add_option("--no_successes",
0071 action="store_true",
0072 dest="no_successes",
0073 default=False,
0074 help="Do not draw successes. Default is False.")
0075
0076 parser.add_option("-P","--pickle",
0077 action="store",
0078 dest="pklfile",
0079 default="",
0080 help="Pkl file of the dir structure ")
0081
0082 parser.add_option("--sample",
0083 action="store",
0084 dest="sample",
0085 default="Sample",
0086 help="The name of the sample to be displayed")
0087
0088 parser.add_option("--metas",
0089 action="store",
0090 dest="metas",
0091 default="",
0092 help="The Metas describing the two files (separated by @@@)")
0093
0094 parser.add_option("-t","--test_threshold",
0095 action="store",
0096 dest="test_threshold",
0097 default=test_threshold,
0098 help="Threshold for the statistical test \n(default is %s)" %test_threshold)
0099
0100 parser.add_option("-s","--stat_test",
0101 action="store",
0102 dest="stat_test",
0103 default=stat_test,
0104 help="Statistical test (KS or Chi2) \n(default is %s)" %stat_test)
0105
0106 parser.add_option("-C","--compare",
0107 action="store_true",
0108 dest="compare",
0109 default=compare,
0110 help="Make the comparison \n(default is %s)" %compare)
0111
0112 parser.add_option("-R","--Report",
0113 action="store_true",
0114 dest="report",
0115 default=report,
0116 help="Make the html report \n(default is %s)" %report)
0117
0118 parser.add_option("--specify_run",
0119 action="store_true",
0120 dest="specify_run",
0121 default=False,
0122 help="Append the run number to the output dir for data")
0123
0124
0125 parser.add_option("-B","--black_list",
0126 action="store",
0127 dest="black_list",
0128 default=black_list_str,
0129 help="Blacklist elements. form is name@hierarchy_level (i.e. HLT@1) \n(default is %s)" %black_list_str)
0130
0131
0132 parser.add_option("--hash_name",
0133 action="store_true",
0134 dest="hash_name",
0135 default=False,
0136 help="Set if you want to minimize & hash the output HTML files.")
0137
0138 parser.add_option("--use_black_file",
0139 action="store_true",
0140 dest="blacklist_file",
0141 default=False,
0142 help="Use a black list file of histograms located @ /RelMon/data")
0143
0144 parser.add_option("--standalone",
0145 action="store_true",
0146 dest="standalone",
0147 default=False,
0148 help="Makes CSS files accessible over HTTP")
0149
0150 def blackListedHistos():
0151
0152 if "RELMON_SA" in os.environ:
0153 black_list_file="../data/blacklist.txt"
0154 else:
0155 black_list_file="%s/src/Utilities/RelMon/data/blacklist.txt"%(os.environ["CMSSW_BASE"])
0156 bListFile = open(black_list_file,'r')
0157 black_listed_histograms = bListFile.read()
0158 bListFile.close()
0159
0160 histogramArray = black_listed_histograms.split("\n")
0161 histogramArray.remove("")
0162 newarray = []
0163 for elem in histogramArray:
0164 tmp = elem.split("/")
0165 tmp.insert(1,"Run summary")
0166 newarray.append(("/").join(tmp))
0167 return newarray
0168
0169
0170 (options, args) = parser.parse_args()
0171
0172 if len(args)!=2 and options.compare:
0173 print("Wrong number of RootFiles specified (%s)" %len(args))
0174 print(args)
0175
0176
0177 original_pickle_name=""
0178 if options.compare:
0179
0180 if "RELMON_SA" in os.environ:
0181 import definitions
0182 from dqm_interfaces import DirID,DirWalkerFile,string2blacklist
0183 from dirstructure import Directory
0184 else:
0185 import Utilities.RelMon.definitions as definitions
0186 from Utilities.RelMon.dqm_interfaces import DirID,DirWalkerFile,string2blacklist
0187 from Utilities.RelMon.dirstructure import Directory
0188
0189 import pickle
0190 from os import mkdir,chdir,getcwd
0191 from os.path import exists
0192
0193
0194
0195 rootfilename1,rootfilename2 = args
0196
0197 run1=-1
0198 sample1=''
0199 cmssw_release1=''
0200 tier1=''
0201 run2=-1
0202 sample2=''
0203 cmssw_release2=''
0204 tier2=''
0205
0206 if options.metas=='':
0207 run1,sample1,cmssw_release1,tier1= getInfoFromFilename(rootfilename1)
0208 run2,sample2,cmssw_release2,tier2= getInfoFromFilename(rootfilename2)
0209 else:
0210 print("Reading meta from commandline")
0211 sample1=sample2=options.sample
0212 cmssw_release1,cmssw_release2=options.metas.split('@@@')
0213 options.standalone = True
0214
0215
0216 if sample1!=sample2:
0217 print("I am puzzled. Did you choose two different samples?")
0218
0219 sample = sample1
0220
0221
0222 if run1!=run2:
0223 print("I am puzzled. Did you choose two different runs?")
0224
0225 run=run1
0226
0227 fulldirname=options.outdir_name
0228 if len(fulldirname)==0:
0229 fulldirname=options.dir_name
0230 if len(fulldirname)==0:
0231 fulldirname="%s_%s_%s" %(sample1,cmssw_release1,cmssw_release2)
0232
0233
0234 black_list=string2blacklist(options.black_list)
0235
0236 if options.blacklist_file:
0237 black_listed = blackListedHistos()
0238 else:
0239 black_listed = []
0240
0241
0242
0243 print("Analysing Histograms located in directory %s at: " %options.dir_name)
0244 for filename in rootfilename1,rootfilename2:
0245 print(" o %s" %filename)
0246
0247
0248 if len(black_list)>0:
0249 print("We have a Blacklist:")
0250 for dirid in black_list:
0251 print(" o %s" %dirid)
0252
0253
0254 directory=Directory(options.dir_name)
0255 dirwalker=DirWalkerFile(fulldirname,
0256 options.dir_name,
0257 rootfilename1,rootfilename2,
0258 run,
0259 black_list,
0260 options.stat_test,
0261 options.test_threshold,
0262 not options.no_successes,
0263 options.do_pngs,
0264 set(black_listed)
0265 )
0266
0267
0268 outdir_name=options.outdir_name
0269 if run>1 and options.specify_run:
0270 outdir_name+="_%s" %run
0271 fulldirname+="_%s" %run
0272 print("+"*30)
0273 print("Output Directory will be ", outdir_name)
0274 options.outdir_name=outdir_name
0275 if not exists(outdir_name) and len(outdir_name )>0:
0276 mkdir(outdir_name)
0277 if len(outdir_name)>0:
0278 chdir(outdir_name)
0279 dirwalker.walk()
0280
0281 run = dirwalker.run
0282
0283
0284
0285 directory=dirwalker.directory
0286
0287
0288 directory.meta.sample1=sample1
0289 directory.meta.sample2=sample2
0290 directory.meta.run1=run1
0291 directory.meta.run2=run2
0292 directory.meta.release1=cmssw_release1
0293 directory.meta.release2=cmssw_release2
0294 directory.meta.tier1=tier1
0295 directory.meta.tier2=tier2
0296
0297
0298 directory.print_report(verbose=True)
0299
0300
0301 directory.prune("Run summary")
0302
0303
0304 original_pickle_name="%s.pkl" %fulldirname
0305 print("Pickleing the directory as %s in dir %s" %(original_pickle_name,getcwd()))
0306 output = open(original_pickle_name,"wb")
0307 pickle.dump(directory, output, -1)
0308 output.close()
0309
0310
0311 if options.report:
0312
0313 if "RELMON_SA" in os.environ:
0314 from directories2html import directory2html
0315 from dirstructure import Directory
0316 else:
0317 from Utilities.RelMon.directories2html import directory2html
0318 from Utilities.RelMon.dirstructure import Directory
0319
0320 from os.path import exists
0321 from os import chdir,mkdir
0322 import os
0323 import pickle
0324
0325 pickle_name=options.pklfile
0326 if len(options.pklfile)==0:
0327 pickle_name=original_pickle_name
0328
0329 print("Reading directory from %s" %(pickle_name))
0330 ifile=open(pickle_name,"rb")
0331 directory=pickle.load(ifile)
0332 ifile.close()
0333
0334 if not options.compare:
0335 if not os.path.exists(options.outdir_name):
0336 mkdir(options.outdir_name)
0337
0338 if exists(options.outdir_name) and len(directory.name)==0:
0339 chdir(options.outdir_name)
0340
0341
0342 print("Calculating stats for the directory...")
0343 directory.calcStats()
0344
0345 print("Producing html...")
0346 directory2html(directory, options.hash_name, options.standalone)
0347
0348 if not (options.report or options.compare):
0349 print("Neither comparison nor report to be executed. A typo?")
0350
0351