RecoMuon/test/muonReleaseSummary.py

0001 #! /usr/bin/env python3
0002
0003 import os
0004 import shutil
0005 import sys
0006 import fileinput
0007 import string
0008 import userparams
0009
0010 ##############################################################
0011 # Input parameters
0012 #
0013
0014 print("Reading input parameters")
0015
0016 #
0017 #specify macros used to plot here
0018 #
0019 macro='macro/TrackValHistoPublisher.C'
0020 macroSeed='macro/SeedValHistoPublisher.C'
0021 macroReco='macro/RecoValHistoPublisher.C'
0022 macroIsol='macro/IsoValHistoPublisher.C'
0023 macroMuonReco='macro/RecoMuonValHistoPublisher.C'
0024
0025
0026 ##############################################################
0027 # Helper functions
0028 #
0029 def GetGuiRepository(param):
0030     splitrelease=param['Release'].split('_')
0031     genericrelease='_'.join(splitrelease[0:3])+'_x/'
0032     dqmguirepository=param['DqmGuiBaseRepo']+genericrelease
0033     return dqmguirepository
0034
0035 def GetLabel(params):
0036     label = params['Condition']
0037     if (params['Condition'] == 'STARTUP'):
0038         label = 'START'
0039     elif (params['Condition']=='MC'):
0040         label ='MC_52_V1'
0041     label+=params['Label']
0042     if (params['FastSim']):
0043         label += '_FastSim'
0044
0045     label += '-' + params['Version']
0046
0047     if (params['PileUp']!='no'):
0048         label = 'PU' + params['PileUp']+ '_' + label
0049
0050     return label
0051
0052
0053 def GetTag(params):
0054     tag = params['Condition']
0055     if (params['PileUp']=='no'):
0056         tag += '_noPU'
0057     else:
0058         tag += '_PU'+params['PileUp']
0059     if (params['FastSim']):
0060         tag +='_FSIM'
0061     return tag
0062
0063
0064
0065 def GetFastSimSuffix(params):
0066     if (params['FastSim']):
0067         return 'FS'
0068     else:
0069         return ''
0070
0071
0072
0073
0074 def replace(map, filein, fileout):
0075     replace_items = map.items()
0076     while True:
0077         line = filein.readline()
0078         if not line: break
0079         for old, new in replace_items:
0080             line = string.replace(line, old, new)
0081         fileout.write(line)
0082     fileout.close()
0083     filein.close()
0084
0085 def myrootsubmit(cfgfile):
0086     command='root -b -q -l '+ cfgfile + '.C' + '>  macro.' + cfgfile +'.log'
0087     print('>> Executing \'' + command + '\'')
0088     os.system(command)
0089     print('   ... executed \'' + command + '\'');
0090
0091 def downloadfile(url):
0092     print('   + Downloading "' + url + '"...')
0093     stream = os.popen('/usr/bin/curl -k -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY -w "%{http_code}" '+ url)
0094     output=stream.readlines()
0095     if output[0]=='200':
0096         print('   + OK!')
0097         return True
0098     else:
0099         print('   + ERROR! ' + str(output[0]))
0100         print("Skipping " + url)
0101         print("Please check the name of the file in the repository: "+GetGuiRepository(userparams.NewParams))
0102         sys.exit('Exiting...');
0103         # return False
0104
0105 def GetSamplePath(params, sample):
0106     return params['Release']+'/'+GetTag(params)+'/'+sample
0107
0108 def GetLocalSampleName(params, sample):
0109     return GetSamplePath(params, sample) + '/val.' + sample + '.root'
0110
0111
0112 def createSampleDirectory(params, sample):
0113     path = GetSamplePath(params, sample)
0114     print('>> Creating directory ' + path + '...')
0115     if(os.path.exists(path)==False):
0116         os.makedirs(path)
0117     else:
0118         print('NOTE: Directory \'' + path + '\' already exists')
0119
0120     return path
0121
0122
0123
0124 def getSampleFiles(params, sample):
0125     path = GetSamplePath(params,sample)
0126     print('>> Creating directory ' + path + '...')
0127     if(os.path.exists(path)==False):
0128         os.makedirs(path)
0129     else:
0130         print('NOTE: Directory \'' + path + '\' already exists')
0131
0132     print('\n')
0133
0134     # Check if pdfs have already been produced
0135     checkFile = path + '/general_tpToTkmuAssociation' + GetFastSimSuffix(params)+'.pdf'
0136     print('>> Checking if ' + checkFile + ' already exists...')
0137     if (os.path.isfile(checkFile)==True):
0138         print('   + Files of type ' + checkFile + ' exist already.')
0139         print('     Delete them first, if you really want to overwrite them')
0140         return
0141
0142     print('   + The file does not exist so we continue')
0143     localsample=GetLocalSampleName(params, sample)
0144     sampleOnWeb=userparams.WebRepository+'/'+ localsample
0145
0146     if (os.path.isfile(localsample)==True):
0147         print('   + ' + params['Type'] + 'sample file found at: ' +localsample + '. Using that one')
0148     elif (userparams.NewParams['GetFilesFrom']=='GUI'):
0149         theGuiSample = sample
0150
0151         if (params['Condition'].find('PRE_LS1')!=-1 or params['Condition'].find('POSTLS1')!=-1):
0152             if (userparams.NewParams['FastSim']| userparams.RefParams['FastSim']):
0153                 if ((sample=="RelValTTbar")|(sample=="RelValJpsiMM")|(sample=="RelValZMM")):
0154                     #theGuiPostFixLS1 = "_UPGpostls1_14"
0155                     theGuiPostFixLS1 = "_13"
0156                     theGuiSample = theGuiSample+theGuiPostFixLS1
0157                 print("Sample is "+theGuiSample)
0158             else:
0159                 # Temporary fix due to the different names used for JPsiMM and TTbar samples in the DQM GUI
0160                 if(sample=="RelValZpMM_2250_13TeV_Tauola"):
0161                     print("Subfix not added")
0162                     theGuiPostFixLS1 = ""
0163                 elif ((sample=="RelValTTbar")|(sample=="RelValJpsiMM")|(sample=="RelValZMM")):
0164                     #theGuiPostFixLS1 = "_UPGpostls1_14"
0165                     theGuiPostFixLS1 = "_13"
0166                 else:
0167                     theGuiPostFixLS1 = "_UP15"
0168
0169             theGuiSample = theGuiSample+theGuiPostFixLS1
0170
0171         guiFileName='DQM_V0001_R000000001__'+theGuiSample+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0172         guiFullURL=GetGuiRepository(params)+guiFileName
0173         print(">> Downloading file from the GUI: " + guiFullURL)
0174         #os.system('wget --ca-directory $X509_CERT_DIR/ --certificate=$X509_USER_PROXY --private-key=$X509_USER_PROXY '+GetGuiRepository(params)+guiFileName)
0175         #os.system('/usr/bin/curl -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY '+ guiFullURL)
0176         if (downloadfile(guiFullURL)==True):
0177             print('   + Moving ' + guiFileName + ' to ' + localsample)
0178             shutil.move(guiFileName,localsample)
0179
0180     elif (params['GetFilesFrom']=='CASTOR'):
0181         print('   + Getting new file from castor')
0182         params['Condition']=params['Condition']+GetFastSimSuffix(params)
0183         os.system('rfcp '+params['CastorRepository']+'/'+params['Release']+'_'+params['Condition']+'_'+sample+'_val.'+sample+'.root '+localsample)
0184     elif ((params['GetFilesFrom']=='WEB') & (os.path.isfile(sampleOnWeb))) :
0185         print("NOTE: New file found at: "+newSample+' -> Copy that one')
0186         os.system('cp '+sampleOnWeb+' '+path)
0187     else:
0188         print('*** WARNING: no signal file was found')
0189
0190
0191 def getReplaceMap(newparams, refparams, sample, datatype, cfgkey, cfgfile):
0192     newLocalSample=GetLocalSampleName(newparams, sample)
0193     refLocalSample=GetLocalSampleName(refparams, sample)
0194     replace_map = { 'DATATYPE': datatype,
0195                     'NEW_FILE':newLocalSample,
0196                     'REF_FILE':refLocalSample,
0197                     'REF_LABEL':sample,
0198                     'NEW_LABEL': sample,
0199                     'REF_RELEASE':refparams['Release'],
0200                     'NEW_RELEASE':newparams['Release'],
0201                     'REFSELECTION':GetTag(refparams),
0202                     'NEWSELECTION':GetTag(newparams),
0203                     cfgkey: cfgfile
0204                     }
0205     return replace_map
0206
0207
0208 ##############################################################
0209 # Main program
0210 #
0211
0212 # Initial checks
0213 # + Copying missing parameters from New to Ref
0214 print('>> Checking input parameters...')
0215 for i in userparams.NewParams:
0216     if i in userparams.RefParams:
0217         print('   + ' + i + ' set to:')
0218         print('     -' + str(userparams.NewParams[i]) + ' (new)')
0219         print('     -' + str(userparams.RefParams[i]) + ' (ref)')
0220     else:
0221         userparams.RefParams[i]=userparams.NewParams[i]
0222         print('   + ' + i + ' set to ' + str(userparams.NewParams[i]) + ' (both)')
0223
0224 # + Things needed by GUI
0225 if ((userparams.NewParams['GetFilesFrom']=='GUI')|(userparams.RefParams['GetFilesFrom']=='GUI')):
0226     if os.getenv('X509_USER_PROXY','') == '':
0227         print("ERROR: It seems you did not configure your environment to be able")
0228         print("       to download files from the GUI. Your should follow these steps:")
0229         print(" > source /afs/cern.ch/project/gd/LCG-share/sl5/etc/profile.d/grid_env.csh")
0230         print(" > voms-proxy-init")
0231         quit()
0232
0233 if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0234     userparams.ValidateDQM=False
0235
0236
0237 # Choose samples based on input parameters
0238 #print('>> Selecting samples...')
0239 #if (userparams.NewParams['Condition']=='MC'):
0240 #    userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValSingleMuPt1000','RelValTTbar']
0241 #    if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0242 #        userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValTTbar']
0243 #elif (userparams.NewParams['Condition']=='STARTUP'):
0244 #    userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValSingleMuPt1000','RelValTTbar','RelValZMM','RelValJpsiMM']
0245 #    if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0246 #        userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValTTbar']
0247 #if ((userparams.NewParams['Condition'].find("PRE_LS1")!=-1)|(userparams.RefParams['Condition'].find("POSTS1")!=-1)):
0248 #    #userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValSingleMuPt1000','RelValTTbar','RelValZMM','RelValJpsiMM','RelValZpMM_2250_13TeV_Tauola']
0249 #    userparams.samples= ['RelValZpMM_2250_13TeV_Tauola']
0250 #if ((userparams.NewParams['PileUp'] != 'no')|(userparams.RefParams['PileUp']!='no')):
0251 #    userparams.samples= ['RelValTTbar']
0252 #    if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0253 #        userparams.samples= ['RelValTTbar']
0254
0255 # Itereate over userparams.samples
0256 print('>> Samples selected:')
0257 for sample in userparams.samples:
0258     print('   + ' + sample)
0259
0260 print('>> Processing userparams.samples:')
0261 for sample in userparams.samples :
0262     print('###################################################')
0263     print('# ' + sample)
0264     print('###################################################')
0265     print('\n')
0266
0267
0268
0269     # Create the directories to store the userparams.samples and pdfs
0270     newpath=createSampleDirectory(userparams.NewParams, sample)
0271     refpath=createSampleDirectory(userparams.RefParams, sample)
0272     print('\n')
0273
0274     getSampleFiles(userparams.NewParams, sample)
0275     getSampleFiles(userparams.RefParams, sample)
0276
0277
0278     print('   + Producing macro files...')
0279     cfgFileName=sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0280     hltcfgFileName='HLT'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0281     seedcfgFileName='DQMSEED'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0282     recocfgFileName='DQMRECO'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0283     recomuoncfgFileName='RECO'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0284     isolcfgFileName='ISOL'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0285
0286
0287     if os.path.isfile(GetLocalSampleName(userparams.RefParams,sample)):
0288         replace_map = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'TrackValHistoPublisher', cfgFileName)
0289         if (userparams.ValidateHLT):
0290             replace_map_HLT = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'HLT', 'TrackValHistoPublisher', hltcfgFileName)
0291         if (userparams.ValidateDQM):
0292             replace_map_DIST = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0293             replace_map_SEED = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0294         if (userparams.ValidateISO):
0295             replace_map_ISOL = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0296         if (userparams.ValidateRECO):
0297             replace_map_RECO = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0298             replace_map_RECO['IS_FSIM']=''
0299     else:
0300         print("No reference file found at: ", refpath)
0301         replace_map = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'TrackValHistoPublisher', cfgFileName)
0302         if (userparams.ValidateHLT):
0303             replace_map_HLT = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'HLT', 'TrackValHistoPublisher', hltcfgFileName)
0304         if (userparams.ValidateDQM):
0305             replace_map_DIST = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0306             replace_map_SEED = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0307         if (userparams.ValidateISO):
0308             replace_map_ISOL = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0309         if (userparams.ValidateRECO):
0310             replace_map_RECO = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0311             replace_map_RECO['IS_FSIM']=''
0312
0313     templatemacroFile = open(macro, 'r')
0314     macroFile = open(cfgFileName+'.C' , 'w' )
0315     replace(replace_map, templatemacroFile, macroFile)
0316
0317     if (userparams.ValidateHLT):
0318         templatemacroFile = open(macro, 'r')
0319         hltmacroFile = open(hltcfgFileName+'.C' , 'w' )
0320         replace(replace_map_HLT, templatemacroFile, hltmacroFile)
0321
0322     if (userparams.ValidateDQM):
0323         templatemacroFile = open(macroReco, 'r')
0324         recomacroFile = open(recocfgFileName+'.C' , 'w' )
0325         replace(replace_map_DIST, templatemacroFile, recomacroFile)
0326         templatemacroFile = open(macroSeed, 'r')
0327         seedmacroFile = open(seedcfgFileName+'.C' , 'w' )
0328         replace(replace_map_SEED, templatemacroFile, seedmacroFile)
0329
0330     if (userparams.ValidateISO):
0331         templatemacroFile = open(macroIsol, 'r')
0332         isolmacroFile = open(isolcfgFileName+'.C' , 'w' )
0333         replace(replace_map_ISOL, templatemacroFile, isolmacroFile)
0334
0335     if (userparams.ValidateRECO):
0336         templatemacroFile = open(macroMuonReco, 'r')
0337         recomuonmacroFile = open(recomuoncfgFileName+'.C' , 'w' )
0338         replace(replace_map_RECO, templatemacroFile, recomuonmacroFile)
0339
0340     if(userparams.Submit):
0341         myrootsubmit(cfgFileName)
0342         if (userparams.ValidateHLT):
0343             myrootsubmit(hltcfgFileName)
0344         if (userparams.ValidateDQM):
0345             myrootsubmit(recocfgFileName)
0346             myrootsubmit(seedcfgFileName)
0347         if (userparams.ValidateISO):
0348             myrootsubmit(isolcfgFileName)
0349             if (userparams.NewParams['FastSim']&userparams.RefParams['FastSim']):
0350                 shutil.move(newpath+'/MuonIsolationV_inc.pdf',newpath+'/MuonIsolationV_inc_FS.pdf')
0351         if (userparams.ValidateRECO):
0352             myrootsubmit(recomuoncfgFileName)
0353             if (userparams.NewParams['FastSim']&userparams.RefParams['FastSim']):
0354                 if (os.path.isfile(newpath+'/RecoMuonV.pdf') == True):
0355                     os.rename(newpath+'/RecoMuonV.pdf',newpath+'/RecoMuonV_FS.pdf')
0356                 else:
0357                     print('ERROR: Could not find "' + newpath + '/RecoMuonV.pdf')
0358
0359     if(userparams.Publish):
0360         newpath = GetSamplePath(userparams.NewParams,sample)
0361         newlocalsample = GetLocalSampleName(userparams.NewParams, sample)
0362         newdir=userparams.WebRepository + '/' + newpath
0363         print('>> Publishing to ' + newdir + '...')
0364         if(os.path.exists(newdir)==False):
0365             os.system('ssh '+userparams.User+'@lxplus.cern.ch mkdir -p ' + newdir)
0366             # os.makedirs(newdir)
0367         # os.system('rm '+ newlocalsample)
0368         # os.system('scp -r '+newpath+'/* ' + newdir)
0369         os.system('scp -r '+newpath+'/* '+userparams.User+'@lxplus.cern.ch:' + newdir)
0370         print('Newpath is' + newlocalsample + ' and ' + newpath)