Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2024-04-06 11:57:13 

0001 #!/usr/bin/env python3
0002 import sys
0003 argv = sys.argv
0004 sys.argv = argv[:1]
0005 
0006 import argparse
0007 import os
0008 import json
0009 import yaml
0010 import csv
0011 import ROOT
0012 
0013 def log(log_type="",text=""):
0014     #########################################################################################################################
0015     #Logger:
0016     #  INFO    = Informative text
0017     #  WARNING = Notify user about unpredictable changes or missing files which do not result in abort
0018     #  ERROR   = Error in logic results in abort. Can be fixed by user (missing input, settings clash ...)
0019     #  FATAL   = Fatal error results in abort. Cannot be fixed by user (the way how input is produced has changed or bug ...)
0020     #########################################################################################################################
0021 
0022     v = int(sys.version_info[0])
0023     source = "mkLumiAveragedPlots:       "
0024     text = str(text)
0025     if v == 3:
0026         if "i" in log_type:
0027             print(source,"[INFO]     ",text)
0028         elif "n" in log_type:
0029             print("                  ",text)
0030         elif "w" in log_type:
0031             print(source,"[WARNING]  ",text)
0032         elif "e" in log_type:
0033             print(source,"[ERROR]    ",text)
0034         elif "f" in log_type:
0035             print(source,"[FATAL]    ",text)
0036         else:
0037             print(text) 
0038 
0039 def isNumber(this):
0040     try:
0041         int(this)
0042         return True
0043     except ValueError:
0044         return False 
0045 
0046 def decodeLine(line,index,type):
0047     if "txt" in type:
0048         return line.strip('\n').split(" ")[index]
0049     elif "csv" in type:
0050         return line.strip('\r\n').replace("\t"," ").split(" ")[index]
0051 
0052 def getLumiPerIoV(start_run,end_run = 0):
0053     #################################
0054     #Returns luminosity per IoV
0055     lumi = 0.0
0056     foundStartRun = False
0057     if end_run > 0:
0058         for lumifile in lumiPerRun:
0059             f = open(lumifile, "r")
0060             for line in f:
0061                 if int(decodeLine(line,0,lumifile)) in range(start_run,end_run):
0062                     #log(line.strip('\n').split(" ")[0],lumi,line.strip('\n').split(" ")[1])
0063                     lumi += float(decodeLine(line,1,lumifile))
0064                 if int(decodeLine(line,0,lumifile)) == start_run:
0065                     foundStartRun = True 
0066             f.close()
0067     elif end_run < 0:
0068         for lumifile in lumiPerRun:
0069             f = open(lumifile, "r")
0070             for line in f:
0071                 if int(decodeLine(line,0,lumifile)) >= start_run:
0072                     #log(line.strip('\n').split(" ")[0],lumi,line.strip('\n').split(" ")[1])
0073                     lumi += float(decodeLine(line,1,lumifile)) 
0074                 if int(decodeLine(line,0,lumifile)) == start_run:
0075                     foundStartRun = True   
0076             f.close()  
0077     elif end_run == 0: 
0078         for lumifile in lumiPerIoV:
0079             f = open(lumifile, "r")
0080             for line in f:
0081                 first = decodeLine(line,0,lumifile)
0082                 try:
0083                     if int(first) == start_run:
0084                         lumi = float(decodeLine(line,1,lumifile))
0085                         foundStartRun = True 
0086                 except ValueError:
0087                     if str(first) == str(start_run):
0088                         lumi = float(decodeLine(line,1,lumifile))
0089                         foundStartRun = True
0090             f.close()
0091 
0092     if not foundStartRun:
0093         if lumi == 0.0:
0094             log("w","Lumi per IoV: "+str(start_run)+" not found.")
0095             return 11112
0096     return lumi
0097 
0098 def getLumiPerRun(run):
0099     ###############################################################
0100     #Return luminosity per run as stored in given lumiPerRun files.
0101     #Return code 11112 if run number was not found in the list.
0102     ###############################################################
0103 
0104     lumi = 0.0
0105     foundRun = False
0106     for lumifile in lumiPerRun:
0107         f = open(lumifile, "r")
0108         for line in f:
0109             if int(decodeLine(line,0,lumifile)) == run:
0110                 #log(line.strip('\n').split(" ")[0],lumi,line.strip('\n').split(" ")[1])
0111                 lumi = float(decodeLine(line,1,lumifile)) 
0112                 foundRun = True
0113         f.close() 
0114     if not foundRun: 
0115         log("w","Lumi per run: "+str(run)+" not found.")
0116         return 11112
0117     return lumi 
0118 
0119 def getTuples(inputDir, filterNumbers=[]):
0120     ########################################
0121     #This applies for DATA only.
0122     #PV:
0123     #DMR:
0124     ######################################## 
0125 
0126     tuples = []
0127     if isDMR:
0128         valid_files = 0
0129         for dirpath,dirs,files in os.walk(inputDir):
0130             if len(dirs) == 0: dirs = [""] #needed for finalization jobs 
0131             for n_dir in dirs:
0132                 if len(filterNumbers)!=0 and (n_dir not in filterNumbers or not isNumber(n_dir)): continue
0133                 _number = 0
0134                 if isNumber(n_dir):
0135                     _number = int(n_dir)
0136                     n_dir = n_dir+"/"
0137                 else:
0138                     if len(n_dir) > 0: 
0139                         _number = n_dir  
0140                         n_dir = n_dir+"/" 
0141                     else: 
0142                         _number = os.path.join(dirpath,"OfflineValidationSummary.root")
0143                 if os.path.isfile(os.path.join(dirpath,n_dir+"OfflineValidationSummary.root")): 
0144                     test_root = ROOT.TFile(os.path.join(dirpath,n_dir+"OfflineValidationSummary.root"),"READ")
0145                     if test_root.IsZombie():
0146                         log("w",os.path.join(dirpath,n_dir+"OfflineValidationSummary.root")+" is ZOMBIE!")
0147                         tuples.append({'file' : "invalid.root",
0148                                        'number' : _number,
0149                                        'lumi' : 11111
0150                                      })
0151                         continue
0152                     test_root.Close()
0153                     tuples.append({'file' : os.path.join(dirpath,n_dir+"OfflineValidationSummary.root"),
0154                                    'number' : _number,
0155                                    'lumi' : 0 
0156                                  })
0157                     valid_files += 1
0158                 else:
0159                     log("w",os.path.join(dirpath,n_dir+"OfflineValidationSummary.root")+" NOT found! Directory is empty?")
0160                     tuples.append({'file' : "invalid.root",
0161                                    'number' : _number,
0162                                    'lumi' : 11111
0163                                  }) 
0164             if (valid_files < 2 and config['mode'] != "finalize") or (valid_files < 1 and config['mode'] == "finalize"):
0165                 log("f","Check input directory. Less than two valid files to merge recognized.")
0166                 sys.exit(0)   
0167     
0168     elif isPV:
0169         for dirpath,dirs,files in os.walk(inputDir):
0170             for n_file in files:
0171                 if "PVValidation" in n_file: #if pass this condition then file exists TODO: check for zombie
0172                     if isNumber((n_file.split("_")[-1]).split(".")[0]) and os.path.isfile(os.path.join(dirpath,n_file)):
0173                         tuples.append({'file' : os.path.join(dirpath,n_file),
0174                                        'number' : int((n_file.split("_")[-1]).split(".")[0]),
0175                                        'lumi' : 0
0176                                      })
0177                     else:
0178                         log("f","Format for run by run PV results NOT recognised!")
0179                         sys.exit(0)
0180        
0181     #sort list based on IoV number and calculate luminosity per IoV (from lumiPerRun or directly from lumiPerIoV)
0182     if config['mode'] != "finalize":
0183         tuples.sort(key=lambda tuple: tuple['number'])
0184     if len(lumiPerRun)!=0:
0185         if isDMR:
0186             valid_tuples = []
0187             for ituple,tuple in enumerate(tuples):
0188                 if ituple != len(tuples)-1 or len(config['validation']['firstFromNext']) != 0:
0189                     if int(tuple['lumi']) != 11111: #input is not empty/zombie
0190                         if ituple == len(tuples)-1:
0191                             tuple['lumi'] += getLumiPerIoV(tuples[ituple]['number'],int(config['validation']['firstFromNext'][0]))
0192                         else:  
0193                             tuple['lumi'] += getLumiPerIoV(tuples[ituple]['number'],tuples[ituple+1]['number'])  
0194                     else:
0195                         _ituple = ituple
0196                         while _ituple > 0 and int(tuples[_ituple-1]['lumi']) in [11111,11112]:
0197                             _ituple -= 1
0198                         if _ituple != 0:
0199                             if ituple == len(tuples)-1:
0200                                 dLumi = getLumiPerIoV(tuples[ituple]['number'],int(config['validation']['firstFromNext'][0]))
0201                             else: 
0202                                 dLumi = getLumiPerIoV(tuples[ituple]['number'],tuples[ituple+1]['number'])
0203                             if int(dLumi) not in [11111,11112]: #TODO loop until dLumi is OK 
0204                                 tuples[_ituple-1]['lumi'] += dLumi
0205                             else:
0206                                 dLumi = 0. 
0207                             log("w","Effectively adding luminosity of IoV: "+str(tuples[ituple]['number'])+"(missing file with "+str(dLumi)+") to IoV: "+str(tuples[_ituple-1]['number']))
0208                         else:
0209                             __ituple = ituple
0210                             while __ituple < len(tuples)-1 and int(tuples[__ituple+1]['lumi']) in [11111,11112]:
0211                                 __ituple += 1
0212                             if __ituple !=  len(tuples)-1:
0213                                 dLumi = getLumiPerIoV(tuples[ituple]['number'],tuples[ituple+1]['number'])
0214                                 if int(dLumi) not in [11111,11112]: #TODO loop until dLumi is OK
0215                                     tuples[__ituple+1]['lumi'] += dLumi
0216                                 else:
0217                                     dLumi = 0. 
0218                                 log("w","Effectively adding luminosity of IoV: "+str(tuples[ituple]['number'])+"(missing file with "+str(dLumi)+") to IoV: "+str(tuples[__ituple+1]['number']))
0219                 else:
0220                     if skipLast:
0221                         log("i","User requested to skip IoV: "+str(tuples[ituple]['number']))
0222                     else:
0223                         if int(tuple['lumi']) != 11111: #input is not empty/zombie
0224                             tuple['lumi'] += getLumiPerIoV(tuples[ituple]['number'],-1)
0225                         else:
0226                             _ituple = ituple
0227                             while _ituple > 0 and int(tuples[_ituple-1]['lumi']) in [11111,11112]:
0228                                 _ituple -= 1
0229                             if _ituple != 0:
0230                                 dLumi = getLumiPerIoV(tuples[ituple]['number'],-1) 
0231                                 if int(dLumi) not in [11111,11112]: #TODO loop until dLumi is OK
0232                                     tuples[_ituple-1]['lumi'] += dLumi
0233                                 else:
0234                                     dLumi = 0.  
0235                                 log("w","Effectively adding luminosity of IoV: "+str(tuples[ituple]['number'])+"(missing file with "+str(dLumi)+") to IoV: "+str(tuples[_ituple-1]['number']))
0236                             else:
0237                                 log("w","No more IOVs in the list to add luminosity from missing IOV"+str(tuples[ituple]['number'])+".")
0238 
0239             if skipLast:
0240                 valid_tuples = [ tuple for ituple,tuple in enumerate(tuples) if int(tuple['lumi']) not in [11111,11112] and ituple != len(tuples)-1 ]
0241             else:
0242                 valid_tuples = [ tuple for tuple in tuples if int(tuple['lumi']) not in [11111,11112] ]   
0243             tuples = valid_tuples
0244  
0245         elif isPV:
0246             for ituple,tuple in enumerate(tuples):
0247                 tuple['lumi'] = getLumiPerRun(tuple['number'])
0248            
0249     elif len(lumiPerIoV)!=0 and isDMR:
0250         #This will work for finalization jobs as well  
0251         valid_tuples = []
0252         for ituple,tuple in enumerate(tuples):
0253             if tuple['lumi'] != 11111: #empty input will not contribute to total average
0254                 tuple['lumi'] = getLumiPerIoV(tuple['number'])
0255                 valid_tuples.append(tuple)
0256         tuples = valid_tuples 
0257 
0258     return tuples
0259 
0260 def getTuplesMC(inputDir):
0261     #####################################################################
0262     #This applies for MC only.
0263     #Define different periods inside input object directory:
0264     #PV: PVValidation_<period>_<part>.root
0265     #DMR: OfflineValidationSummary.root (period hidden in subdirectory)
0266     #####################################################################
0267 
0268     tuples = {}
0269     if isPV:
0270         for dirpath,dirs,files in os.walk(inputDir):
0271             for file in files:
0272                 if "PVValidation" in file:
0273                     if len(file.split("_")) == 3:
0274                         period = file.split("_")[1]
0275                         if period not in tuples: 
0276                             tuples[period] = []
0277                             tuples[period].append({ 'file' : os.path.join(dirpath,file),
0278                                                     'lumi' : 1
0279                                                  })
0280                         else:
0281                             tuples[period].append({ 'file' : os.path.join(dirpath,file),
0282                                                     'lumi' : 1
0283                                                  }) 
0284                     elif len(file.split("_")) == 2:
0285                         if ".root" in file.split("_")[1]:
0286                             period = file.split("_")[1].replace(".root","")
0287                             if period not in tuples:
0288                                 tuples[period] = []  
0289                                 tuples[period].append({ 'file' : os.path.join(dirpath,file),
0290                                                         'lumi' : 1
0291                                                      })
0292                             else:
0293                                 tuples[period].append({ 'file' : os.path.join(dirpath,file),
0294                                                         'lumi' : 1
0295                                                      }) 
0296                         else:
0297                             log("e","No extension found for PV MC input files.")
0298                             sys.exit(0)    
0299                     else:
0300                         log("w","Incorrect format for <period> tag inside PV MC input-file names. Char \"_\" is not allowed inside tag.")
0301     elif isDMR:
0302         inputFile = os.path.join(inputDir,"OfflineValidationSummary.root")
0303         period = (inputDir.split("/")[-2])
0304         tuples[period] = []
0305         if os.path.isfile(inputFile): 
0306             tuples[period].append({ 'file' : inputFile, 
0307                                    'lumi' : 1})
0308         else:
0309             tuples[period].append({ 'file' : "invalid.root", 
0310                                     'lumi' : 11111}) 
0311 
0312     return tuples    
0313             
0314 
0315 def makeAveragedFile(tuples,intLumi,objName=""):
0316     ####################################################################
0317     #Use (compiled) haddws executable to weight every histogram in given
0318     #rootfile by particular weight = lumi/intLumi and merge histograms 
0319     #for each run/IoV. Skip files with 0.0 lumi (uncertified runs).
0320     ####################################################################
0321 
0322     fileArgument = ""
0323     weightArgument = ""
0324     haddwsDir = ""
0325     #cmd_bare = ""
0326 
0327     countTest = 0.0
0328     for ituple,tuple in enumerate(tuples):
0329         if str(tuple['lumi']) not in ['11111','11112']: #will not include 11111=non-existing input,11112=non-existing lumi record
0330             if tuple['lumi']/intLumi > -1.:#1e-6: #will not include 'almost zero' lumi input (not digestible by haddws)  
0331                 fileArgument += tuple['file']+" "
0332                 weightArgument += str(format(tuple['lumi']/intLumi,'.20f'))+" "
0333                 #cmd_bare += tuple['file']+" "+str(format(tuple['lumi']/intLumi,'.20f'))+"\n" 
0334                 countTest += tuple['lumi']/intLumi
0335         else:
0336             log("i","Not including IOV "+str(tuple['number'])+" for weighting: "+str(tuple['lumi']))
0337 
0338     if countTest < 1.0:
0339         log("w","Normalization factor: "+str(format(countTest,'.20f'))+" is less than 1.")
0340     if len(weightArgument.split(" ")) != len(fileArgument.split(" ")):
0341         log("e","There is "+str(len(fileArgument.split(" ")))+"rootfiles but "+str(len(weightArgument.split(" ")))+" weights.")
0342     else:
0343         log("i","Real number of files to merge is: "+str(len(fileArgument.split(" "))-1)) 
0344 
0345     #f = open("haddws_command.txt", "w")
0346     #f.write(cmd_bare)
0347     #f.close()
0348     #sys.exit(0)
0349 
0350     cmd = haddwsDir+"haddws "+fileArgument+weightArgument+" > haddws.out 2> haddws.err"
0351     log("i","Running haddws.C")
0352     os.system(cmd)
0353 
0354     #Store result file in output
0355     outFileName = ""
0356     if isDMR:
0357         if objName != "" and not objName.startswith("_"): objName = "_"+objName 
0358         outFileName = os.path.join(config['output'],"OfflineValidationSummary"+objName+".root") 
0359     elif isPV:
0360         outFileName = os.path.join(config['output'],"result"+objName+".root") 
0361     if os.path.isfile("./result.root"): 
0362         os.system("mv result.root "+outFileName)
0363 
0364 def getIntLumi(tuples):
0365     ################################
0366     #Calculate integrated luminosity
0367     ################################
0368 
0369     intLumi = 0.0
0370     for tuple in tuples:
0371         if tuple['lumi'] not in [11111,11112]:  
0372             intLumi += tuple['lumi'] 
0373     return intLumi
0374 
0375 def parser():
0376     sys.argv = argv
0377     parser = argparse.ArgumentParser(description = "run the python plots for the AllInOneTool validations", formatter_class=argparse.RawTextHelpFormatter)
0378     parser.add_argument("config", metavar='config', type=str, action="store", help="Averager AllInOneTool config (json/yaml format)")
0379     parser.add_argument("-b", "--batch", action = "store_true", help ="Batch mode")
0380 
0381     #sys.argv.append('-b')
0382     #ROOT.gROOT.SetBatch()
0383     return parser.parse_args()
0384   
0385 if __name__ == '__main__':
0386     ############################################################
0387     #Main code:
0388     #  - parse local config
0389     #  - run sanity checks
0390     #  - prepare average file (PV or DMR) using external C macro
0391     #  - (and) or run plotting macros
0392     ############################################################
0393 
0394     args = parser()
0395     with open(args.config, "r") as configFile:
0396         if args.config.split(".")[-1] == "json":
0397             config = json.load(configFile)
0398         elif args.config.split(".")[-1] == "yaml":
0399             config = yaml.load(configFile, Loader=yaml.Loader)
0400         else:
0401             raise Exception("Unknown config extension '{}'. Please use json/yaml format!".format(args.config.split(".")[-1]))
0402 
0403     log(' ----- All-in-one Averager -----')
0404     log(' type:   '+config['type'])
0405     log(' mode:   '+config['mode'])
0406     log(' isData: '+str(config['isData']))
0407     log(' isMC:   '+str(not config['isData']))
0408     if config['mode'] == "finalize":
0409         nFiles = len(config['validation']['mergeFile'])
0410     elif config['mode'] == "plot":
0411         nFiles = len(config['plot']['inputData'])+len(config['plot']['inputMC'])
0412     else: 
0413         nFiles = len(config['validation']['IOV'])  
0414     log(' nFiles: '+str(nFiles))
0415     log(' -------------------------------')
0416 
0417     #BASIC SANITY CHECKS
0418     ######################## 
0419     #Input directory checks
0420     ########################
0421     inputDirData = []
0422     inputDirMC   = []
0423     IOVs         = []
0424     if config['mode'] == "merge":
0425         #DATA
0426         if config['isData']:
0427             if len(config['validation']['IOV']) == 0:
0428                 log("f","No input DATA found. List of IOVs needs to contain at least one number.")
0429                 sys.exit(0)
0430             elif len(config['validation']['IOV']) != 0:
0431                 _dir = config['validation']['mergeFile'].replace("{}","")
0432                 if os.path.isdir(_dir):
0433                     log("i","Will average "+str(len(config['validation']['IOV']))+" DATA files from directory(ies): ")
0434                     log("i",_dir)
0435                 subDirMissing = False
0436                 for IOV in config['validation']['IOV']:
0437                     IOVs.append(str(IOV))
0438                     if not os.path.isdir(config['validation']['mergeFile'].replace("{}",str(IOV))):
0439                         log("f","Subdir not found "+str(IOV)+"!")
0440                         subDirMissing = True
0441                 if subDirMissing:
0442                     sys.exit(0)
0443                 else:
0444                     inputDirData.append(_dir)
0445         #MC 
0446         elif not config['isData']:
0447             if len(config['validation']['IOV']) != 1 \
0448             or (len(config['validation']['IOV']) == 1 and str(config['validation']['IOV'][0]) != "1"):
0449                 log("f","MC validation configuration error: IOV!=1")
0450                 sys.exit(0)
0451             else:
0452                 IOVs.append(str(config['validation']['IOV'][0]))
0453             if len(config['validation']['mergeFile']) == 0:
0454                 log("f", "No input MC found.")
0455                 sys.exit(0)
0456             else:
0457                 log("i","Will scale and merge "+str(len(config['validation']['mergeFile']))+" MC directory(ies): ")
0458                 for _dir in config['validation']['mergeFile']:
0459                     basedir = _dir.replace("{}","")
0460                     subdir = _dir.replace("{}",str(config['validation']['IOV'][0]))
0461                     if os.path.isdir(basedir) and os.path.isdir(subdir):
0462                         log("i",subdir)
0463                         inputDirMC.append(subdir)
0464                     else:
0465                         log("f","Directory not found "+subdir)
0466                         sys.exit(0)
0467     elif config['mode'] == "finalize":
0468         #DATA FINALIZE
0469         if config['isData']:
0470             if len(config['validation']['mergeFile']) == 0:
0471                 log("f", "No files found to finalize.")
0472                 sys.exit(0)
0473             log("i","Will finalize average job for "+str(len(config['validation']['mergeFile']))+" parts:")
0474             for partFile in config['validation']['mergeFile']:
0475                 if os.path.isdir(partFile): 
0476                     inputDirData.append(partFile)
0477                     log("i","---> "+partFile)  
0478                 else: log("w","Missing partial input: "+partFile)
0479             if len(inputDirData) != len(config['validation']['mergeFile']):
0480                 log("e","Some input was not found for DATA finalization job.")
0481                 sys.exit(0)   
0482         #NO FINALIZE FOR MC 
0483         elif not config['isData']:
0484             log("f", "Nothing to finalize for MC.")
0485             sys.exit(0)
0486     elif config['mode'] == "plot":
0487         if len(config['plot']['inputData'])+len(config['plot']['inputMC']) == 0:
0488                 log("f", "No files found for plotting!")
0489                 sys.exit(0)
0490         log("i", "Will attempt to plot objects from "+str(nFiles)+" source files.") 
0491         for inputDir in config['plot']['inputData']:
0492             if os.path.isdir(inputDir):
0493                 inputDirData.append(inputDir)
0494         for inputDir in config['plot']['inputMC']:
0495             if os.path.isdir(inputDir):
0496                 inputDirMC.append(inputDir)  
0497                   
0498                 
0499     #######################
0500     # JSON lumifiles checks
0501     #######################
0502     isPV  = (config['type'] == "PV")
0503     isDMR = (config['type'] == "DMR")
0504     if config['mode'] != "plot":
0505         lumiPerRun = config['validation']['lumiPerRun']
0506         lumiPerIoV = config['validation']['lumiPerIoV']
0507         lumiMC     = config['validation']['lumiMC']
0508     else:
0509         lumiPerRun = []
0510         lumiPerIoV = []
0511         lumiMC     = []
0512     if config['mode'] != "plot":
0513         if isPV and len(lumiPerIoV)!=0: 
0514             log("w","Are you sure you have run PV validation in IoV by IoV mode?")
0515         if (isPV or isDMR) and len(lumiPerRun)==0 and len(lumiPerIoV)==0 and len(inputDirData) != 0:
0516             log("e","Use option 'lumiPerRun' or 'lumiPerIoV' to specify luminosity files.")
0517             sys.exit(0)
0518         if (isPV or isDMR) and len(lumiPerIoV)==0:
0519             if len(lumiPerRun) != 0:
0520                 if config['mode'] == "finalize":
0521                     log("i","Integrated Luminosity per intermediate files found.")
0522                 else:
0523                     log("w","Lumi per run list will be processed to get Lumi per IoV list (applies for DATA).")
0524             elif len(inputDirMC) != 0 and len(inputDirData) == 0:
0525                 log("i", "MC will be scaled per period to given integrated luminosity.") 
0526         if len(inputDirMC) != 0 and len(lumiMC) == 0:
0527             log("w","MC object(s) found on input but lumi per period not specified.")
0528         _lumiMC = {}
0529         for formatLumi in lumiMC:
0530             if "::" in formatLumi:
0531                 try:
0532                     float(formatLumi.split("::")[-1])
0533                 except ValueError:
0534                     log("e","Wrong lumi per period for MC formatting. USAGE: <object>::<merge>::<lumi> or <merge>::<lumi> for single MC object (alignment).")
0535                     sys.exit(0)
0536                 if len(formatLumi.split("::")) == 2:
0537                     _lumiMC[formatLumi.split("::")[0]] = { 'lumi' : float(formatLumi.split("::")[-1]), 'group' : 0}
0538                 elif len(formatLumi.split("::")) == 3:
0539                     _lumiMC[formatLumi.split("::")[1]] = { 'lumi' : float(formatLumi.split("::")[-1]), 'group' : formatLumi.split("::")[0]}
0540             else:
0541                 log("e","Wrong lumi per period for MC formatting. USAGE: <object>::<merge>::<lumi> or <merge>::<lumi> for single MC object (alignment).")
0542                 sys.exit(0)
0543         lumiMC = _lumiMC
0544         skipLast = False
0545         if isDMR and config['isData']: 
0546             if len(config['validation']['firstFromNext']) == 0: skipLast = True 
0547 
0548     ######################
0549     #Style optional checks
0550     ######################
0551     if config['mode'] == "plot": 
0552         if len(config['plot']['colors']) < len(config['plot']['objects']):
0553             log("e","Please specify color code for each object.")
0554             sys.exit(0)
0555         if len(config['plot']['styles']) < len(config['plot']['objects']):
0556             log("e","Please specify line style for each object.")
0557             sys.exit(0) 
0558         if len(config['plot']['objects']) != 0:
0559             for obj in config['plot']['objects']:
0560                 if len(obj.split(" ")) != 1:
0561                     log("e","No space in object name is allowed. Use \"_\" instead.")
0562                     sys.exit(0)
0563         if len(config['plot']['objects']) != 0 and len(config['plot']['labels']) == 0:
0564             log("i","Object labels will be generated automatically.")
0565         if 'plotGlobal' not in config.keys():
0566             log("w","Global plotting settings not found. Fallback to default.")
0567             config['plotGlobal'] = {}  
0568  
0569     ##########################
0570     #Stat plot settings checks
0571     ##########################
0572     if config['mode'] == "plot":
0573         if config['plot']['showMeanError'] and not config['plot']['showMean']:
0574             log("w","Cannot show mean error without showing mean.")
0575             config['plot']['showMeanError'] = False
0576         if config['plot']['showRMSError'] and not config['plot']['showRMS']:
0577             log("w","Cannot show RMS error without showing RMS.")
0578             config['plot']['showRMSError'] = False
0579         if config['plot']['useFitError'] and not config['plot']['useFit']:
0580             log("w","Cannot show fit parameters error without fitting.")
0581             config['plot']['useFitError'] = False
0582 
0583     #################
0584     #Printout details
0585     #################
0586     whichValidation = ""
0587     if isDMR: whichValidation = "DMR"
0588     elif isPV: whichValidation = "PV"
0589 
0590     ##########################################################################################
0591     #DATA: PERFORM INITIAL MERGE OR FINALIZE
0592     #PV: EACH FOLDER CONSIDERED TO BE ONE OBJECT, DMR: (MULTIPLE) OBJECTS DECIDED IMPLICITELY
0593     ##########################################################################################
0594     tuples_total = []
0595     if config['mode'] == "merge": 
0596         for inputDir in inputDirData:
0597             ###################################################################################
0598             #Get list of python dictionaries:  
0599             #{ 'file' : file name, 'number' : IoV/run number, 'lumi' : corresponding luminosity} 
0600             ####################################################################################  
0601             tuples = getTuples(inputDir,IOVs)
0602             intLumi = getIntLumi(tuples)
0603             if len(tuples) == 0:
0604                 log("e","Zero "+whichValidation+" files to merge detected.")
0605                 sys.exit(0)
0606             log("i","Attempting to average "+str(len(tuples))+" "+whichValidation+" files with (partial) integrated luminosity "+str(intLumi)+".")
0607             outFileName = ""
0608             objName = ""
0609             if isDMR:
0610                 outFileName = os.path.join(config['output'],"OfflineValidationSummary"+objName+".root")
0611             elif isPV:
0612                 objName = inputDir.split("/")[-1] 
0613                 outFileName = os.path.join(config['output'],"result"+objName+".root")
0614             tuples_total.append({ 'file' : outFileName,
0615                                   'lumi' : intLumi 
0616                                })
0617             makeAveragedFile(tuples,intLumi,objName)
0618     elif config['mode'] == "finalize":
0619         tuples = []
0620         intLumi = 0.0 
0621         for inputDir in inputDirData:
0622             ####################################################################################################
0623             #Get list of python dictionaries:  
0624             #{ 'file' : previously merged file, 'number' : [], 'lumi' : corresponding lumi from previous jobs} 
0625             ####################################################################################################
0626             tupleFinal = getTuples(inputDir)
0627             tuples += tupleFinal
0628             intLumi += getIntLumi(tupleFinal)
0629         if len(tuples) == 0:
0630                 log("e","Zero final "+whichValidation+" files to merge detected.")
0631                 sys.exit(0)
0632         log("i","Attempting to average final "+str(len(tuples))+" "+whichValidation+" files with overall integrated luminosity "+str(intLumi)+".")
0633         objName = ""
0634         makeAveragedFile(tuples,intLumi,objName)
0635 
0636     ###################################################
0637     #DATA: CREATE LUMI FILE FOR FINALIZATION JOB
0638     ###################################################
0639     if config['mode'] == "merge":
0640         with open(os.path.join(config['output'],'lumiPerFile.csv'), 'a') as csvfile:
0641             csvwriter = csv.writer(csvfile, delimiter=' ')
0642             for tuple in tuples_total:
0643                 csvwriter.writerow([tuple['file'],str(tuple['lumi'])])
0644 
0645     #################################################################
0646     #MC: FIRST DEFINE MC GROUPS 
0647     #    THEN MERGE ALL MC IN GROUP WITH CORRESPONDING LUMI SCALE
0648     #################################################################
0649     tupleGroupsMC = {}
0650     if config['mode'] == "merge":
0651         for inputDir in inputDirMC:
0652             tuples = getTuplesMC(inputDir)
0653             for period, _list in tuples.items():
0654                 tuple = _list[0] 
0655                 if period in lumiMC.keys(): 
0656                     tuple['lumi'] = lumiMC[period]['lumi']   
0657                     log("i","Group N."+str(lumiMC[period]['group'])+" <-- "+str(tuple['lumi']))  
0658                     if lumiMC[period]['group'] not in tupleGroupsMC.keys():
0659                         tupleGroupsMC[lumiMC[period]['group']] = []
0660                         tupleGroupsMC[lumiMC[period]['group']].append(tuple)
0661                     else:
0662                         tupleGroupsMC[lumiMC[period]['group']].append(tuple)
0663                 else:
0664                     log("w","Period "+str(period)+" not recognised in lumiMC list.")
0665         for group, tuples in tupleGroupsMC.items():
0666             log("i","Detected MC N."+str(group)+" group to be merged.")
0667             intLumi = getIntLumi(tuples)
0668             makeAveragedFile(tuples,intLumi,"_merged"+str(group))
0669 
0670     ##############################
0671     #PLOT:
0672     ##############################
0673     if isDMR and config['mode'] == "plot":
0674         #import plotting class  
0675         from Alignment.OfflineValidation.TkAlAllInOneTool.DMRplotter import DMRplotter
0676  
0677         #initialize plotting class with proper options
0678         plotInfo = {}
0679         plotInfo['outputDir'] = config['output']
0680         for key in ['objects','labels','colors','styles', \
0681                     'useFit','useFitError','showMean','showMeanError','showRMS','showRMSError']: 
0682             plotInfo[key] = config['plot'][key]
0683         if 'plotGlobal' in config.keys():
0684             if 'CMSlabel' in config['plotGlobal'].keys():
0685                 plotInfo['CMSlabel'] = config['plotGlobal']['CMSlabel']
0686             else:
0687                 plotInfo['CMSlabel'] = ""
0688             if 'Rlabel' in config['plotGlobal'].keys():
0689                 plotInfo['Rlabel'] = config['plotGlobal']['Rlabel']
0690             else:
0691                 plotInfo['Rlabel'] = "single muon (2016+2017+2018)" 
0692         plotter = DMRplotter(plotInfo)
0693 
0694         #add input files
0695         for inputDir in inputDirData:
0696             plotter.addDATA(inputDir)
0697         for inputDir in inputDirMC:
0698             plotter.addDirMC(inputDir)
0699 
0700         #plot&save
0701         plotter.plot()
0702 
0703     log("i","All done.")
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!