RecoMuon/test/muonReleaseSummary.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os
0005 import shutil
0006 import sys
0007 import fileinput
0008 import string
0009 import userparams
0010
0011 ##############################################################
0012 # Input parameters
0013 #
0014
0015 print("Reading input parameters")
0016
0017 #
0018 #specify macros used to plot here
0019 #
0020 macro='macro/TrackValHistoPublisher.C'
0021 macroSeed='macro/SeedValHistoPublisher.C'
0022 macroReco='macro/RecoValHistoPublisher.C'
0023 macroIsol='macro/IsoValHistoPublisher.C'
0024 macroMuonReco='macro/RecoMuonValHistoPublisher.C'
0025
0026
0027 ##############################################################
0028 # Helper functions
0029 #
0030 def GetGuiRepository(param):
0031     splitrelease=param['Release'].split('_')
0032     genericrelease='_'.join(splitrelease[0:3])+'_x/'
0033     dqmguirepository=param['DqmGuiBaseRepo']+genericrelease
0034     return dqmguirepository
0035
0036 def GetLabel(params):
0037     label = params['Condition']
0038     if (params['Condition'] == 'STARTUP'):
0039         label = 'START'
0040     elif (params['Condition']=='MC'):
0041         label ='MC_52_V1'
0042     label+=params['Label']
0043     if (params['FastSim']):
0044         label += '_FastSim'
0045
0046     label += '-' + params['Version']
0047
0048     if (params['PileUp']!='no'):
0049         label = 'PU' + params['PileUp']+ '_' + label
0050
0051     return label
0052
0053
0054 def GetTag(params):
0055     tag = params['Condition']
0056     if (params['PileUp']=='no'):
0057         tag += '_noPU'
0058     else:
0059         tag += '_PU'+params['PileUp']
0060     if (params['FastSim']):
0061         tag +='_FSIM'
0062     return tag
0063
0064
0065
0066 def GetFastSimSuffix(params):
0067     if (params['FastSim']):
0068         return 'FS'
0069     else:
0070         return ''
0071
0072
0073
0074
0075 def replace(map, filein, fileout):
0076     replace_items = map.items()
0077     while True:
0078         line = filein.readline()
0079         if not line: break
0080         for old, new in replace_items:
0081             line = string.replace(line, old, new)
0082         fileout.write(line)
0083     fileout.close()
0084     filein.close()
0085
0086 def myrootsubmit(cfgfile):
0087     command='root -b -q -l '+ cfgfile + '.C' + '>  macro.' + cfgfile +'.log'
0088     print('>> Executing \'' + command + '\'')
0089     os.system(command)
0090     print('   ... executed \'' + command + '\'');
0091
0092 def downloadfile(url):
0093     print('   + Downloading "' + url + '"...')
0094     stream = os.popen('/usr/bin/curl -k -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY -w "%{http_code}" '+ url)
0095     output=stream.readlines()
0096     if output[0]=='200':
0097         print('   + OK!')
0098         return True
0099     else:
0100         print('   + ERROR! ' + str(output[0]))
0101         print("Skipping " + url)
0102         print("Please check the name of the file in the repository: "+GetGuiRepository(userparams.NewParams))
0103         sys.exit('Exiting...');
0104         # return False
0105
0106 def GetSamplePath(params, sample):
0107     return params['Release']+'/'+GetTag(params)+'/'+sample
0108
0109 def GetLocalSampleName(params, sample):
0110     return GetSamplePath(params, sample) + '/val.' + sample + '.root'
0111
0112
0113 def createSampleDirectory(params, sample):
0114     path = GetSamplePath(params, sample)
0115     print('>> Creating directory ' + path + '...')
0116     if(os.path.exists(path)==False):
0117         os.makedirs(path)
0118     else:
0119         print('NOTE: Directory \'' + path + '\' already exists')
0120
0121     return path
0122
0123
0124
0125 def getSampleFiles(params, sample):
0126     path = GetSamplePath(params,sample)
0127     print('>> Creating directory ' + path + '...')
0128     if(os.path.exists(path)==False):
0129         os.makedirs(path)
0130     else:
0131         print('NOTE: Directory \'' + path + '\' already exists')
0132
0133     print('\n')
0134
0135     # Check if pdfs have already been produced
0136     checkFile = path + '/general_tpToTkmuAssociation' + GetFastSimSuffix(params)+'.pdf'
0137     print('>> Checking if ' + checkFile + ' already exists...')
0138     if (os.path.isfile(checkFile)==True):
0139         print('   + Files of type ' + checkFile + ' exist already.')
0140         print('     Delete them first, if you really want to overwrite them')
0141         return
0142
0143     print('   + The file does not exist so we continue')
0144     localsample=GetLocalSampleName(params, sample)
0145     sampleOnWeb=userparams.WebRepository+'/'+ localsample
0146
0147     if (os.path.isfile(localsample)==True):
0148         print('   + ' + params['Type'] + 'sample file found at: ' +localsample + '. Using that one')
0149     elif (userparams.NewParams['GetFilesFrom']=='GUI'):
0150         theGuiSample = sample
0151
0152         if (params['Condition'].find('PRE_LS1')!=-1 or params['Condition'].find('POSTLS1')!=-1):
0153             if (userparams.NewParams['FastSim']| userparams.RefParams['FastSim']):
0154                 if ((sample=="RelValTTbar")|(sample=="RelValJpsiMM")|(sample=="RelValZMM")):
0155                     #theGuiPostFixLS1 = "_UPGpostls1_14"
0156                     theGuiPostFixLS1 = "_13"
0157                     theGuiSample = theGuiSample+theGuiPostFixLS1
0158                 print("Sample is "+theGuiSample)
0159             else:
0160                 # Temporary fix due to the different names used for JPsiMM and TTbar samples in the DQM GUI
0161                 if(sample=="RelValZpMM_2250_13TeV_Tauola"):
0162                     print("Subfix not added")
0163                     theGuiPostFixLS1 = ""
0164                 elif ((sample=="RelValTTbar")|(sample=="RelValJpsiMM")|(sample=="RelValZMM")):
0165                     #theGuiPostFixLS1 = "_UPGpostls1_14"
0166                     theGuiPostFixLS1 = "_13"
0167                 else:
0168                     theGuiPostFixLS1 = "_UP15"
0169
0170             theGuiSample = theGuiSample+theGuiPostFixLS1
0171
0172         guiFileName='DQM_V0001_R000000001__'+theGuiSample+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0173         guiFullURL=GetGuiRepository(params)+guiFileName
0174         print(">> Downloading file from the GUI: " + guiFullURL)
0175         #os.system('wget --ca-directory $X509_CERT_DIR/ --certificate=$X509_USER_PROXY --private-key=$X509_USER_PROXY '+GetGuiRepository(params)+guiFileName)
0176         #os.system('/usr/bin/curl -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY '+ guiFullURL)
0177         if (downloadfile(guiFullURL)==True):
0178             print('   + Moving ' + guiFileName + ' to ' + localsample)
0179             shutil.move(guiFileName,localsample)
0180
0181     elif (params['GetFilesFrom']=='CASTOR'):
0182         print('   + Getting new file from castor')
0183         params['Condition']=params['Condition']+GetFastSimSuffix(params)
0184         os.system('rfcp '+params['CastorRepository']+'/'+params['Release']+'_'+params['Condition']+'_'+sample+'_val.'+sample+'.root '+localsample)
0185     elif ((params['GetFilesFrom']=='WEB') & (os.path.isfile(sampleOnWeb))) :
0186         print("NOTE: New file found at: "+newSample+' -> Copy that one')
0187         os.system('cp '+sampleOnWeb+' '+path)
0188     else:
0189         print('*** WARNING: no signal file was found')
0190
0191
0192 def getReplaceMap(newparams, refparams, sample, datatype, cfgkey, cfgfile):
0193     newLocalSample=GetLocalSampleName(newparams, sample)
0194     refLocalSample=GetLocalSampleName(refparams, sample)
0195     replace_map = { 'DATATYPE': datatype,
0196                     'NEW_FILE':newLocalSample,
0197                     'REF_FILE':refLocalSample,
0198                     'REF_LABEL':sample,
0199                     'NEW_LABEL': sample,
0200                     'REF_RELEASE':refparams['Release'],
0201                     'NEW_RELEASE':newparams['Release'],
0202                     'REFSELECTION':GetTag(refparams),
0203                     'NEWSELECTION':GetTag(newparams),
0204                     cfgkey: cfgfile
0205                     }
0206     return replace_map
0207
0208
0209 ##############################################################
0210 # Main program
0211 #
0212
0213 # Initial checks
0214 # + Copying missing parameters from New to Ref
0215 print('>> Checking input parameters...')
0216 for i in userparams.NewParams:
0217     if i in userparams.RefParams:
0218         print('   + ' + i + ' set to:')
0219         print('     -' + str(userparams.NewParams[i]) + ' (new)')
0220         print('     -' + str(userparams.RefParams[i]) + ' (ref)')
0221     else:
0222         userparams.RefParams[i]=userparams.NewParams[i]
0223         print('   + ' + i + ' set to ' + str(userparams.NewParams[i]) + ' (both)')
0224
0225 # + Things needed by GUI
0226 if ((userparams.NewParams['GetFilesFrom']=='GUI')|(userparams.RefParams['GetFilesFrom']=='GUI')):
0227     if os.getenv('X509_USER_PROXY','') == '':
0228         print("ERROR: It seems you did not configure your environment to be able")
0229         print("       to download files from the GUI. Your should follow these steps:")
0230         print(" > source /afs/cern.ch/project/gd/LCG-share/sl5/etc/profile.d/grid_env.csh")
0231         print(" > voms-proxy-init")
0232         quit()
0233
0234 if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0235     userparams.ValidateDQM=False
0236
0237
0238 # Choose samples based on input parameters
0239 #print('>> Selecting samples...')
0240 #if (userparams.NewParams['Condition']=='MC'):
0241 #    userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValSingleMuPt1000','RelValTTbar']
0242 #    if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0243 #        userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValTTbar']
0244 #elif (userparams.NewParams['Condition']=='STARTUP'):
0245 #    userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValSingleMuPt1000','RelValTTbar','RelValZMM','RelValJpsiMM']
0246 #    if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0247 #        userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValTTbar']
0248 #if ((userparams.NewParams['Condition'].find("PRE_LS1")!=-1)|(userparams.RefParams['Condition'].find("POSTS1")!=-1)):
0249 #    #userparams.samples= ['RelValSingleMuPt10','RelValSingleMuPt100','RelValSingleMuPt1000','RelValTTbar','RelValZMM','RelValJpsiMM','RelValZpMM_2250_13TeV_Tauola']
0250 #    userparams.samples= ['RelValZpMM_2250_13TeV_Tauola']
0251 #if ((userparams.NewParams['PileUp'] != 'no')|(userparams.RefParams['PileUp']!='no')):
0252 #    userparams.samples= ['RelValTTbar']
0253 #    if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0254 #        userparams.samples= ['RelValTTbar']
0255
0256 # Itereate over userparams.samples
0257 print('>> Samples selected:')
0258 for sample in userparams.samples:
0259     print('   + ' + sample)
0260
0261 print('>> Processing userparams.samples:')
0262 for sample in userparams.samples :
0263     print('###################################################')
0264     print('# ' + sample)
0265     print('###################################################')
0266     print('\n')
0267
0268
0269
0270     # Create the directories to store the userparams.samples and pdfs
0271     newpath=createSampleDirectory(userparams.NewParams, sample)
0272     refpath=createSampleDirectory(userparams.RefParams, sample)
0273     print('\n')
0274
0275     getSampleFiles(userparams.NewParams, sample)
0276     getSampleFiles(userparams.RefParams, sample)
0277
0278
0279     print('   + Producing macro files...')
0280     cfgFileName=sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0281     hltcfgFileName='HLT'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0282     seedcfgFileName='DQMSEED'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0283     recocfgFileName='DQMRECO'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0284     recomuoncfgFileName='RECO'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0285     isolcfgFileName='ISOL'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0286
0287
0288     if os.path.isfile(GetLocalSampleName(userparams.RefParams,sample)):
0289         replace_map = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'TrackValHistoPublisher', cfgFileName)
0290         if (userparams.ValidateHLT):
0291             replace_map_HLT = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'HLT', 'TrackValHistoPublisher', hltcfgFileName)
0292         if (userparams.ValidateDQM):
0293             replace_map_DIST = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0294             replace_map_SEED = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0295         if (userparams.ValidateISO):
0296             replace_map_ISOL = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0297         if (userparams.ValidateRECO):
0298             replace_map_RECO = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0299             replace_map_RECO['IS_FSIM']=''
0300     else:
0301         print("No reference file found at: ", refpath)
0302         replace_map = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'TrackValHistoPublisher', cfgFileName)
0303         if (userparams.ValidateHLT):
0304             replace_map_HLT = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'HLT', 'TrackValHistoPublisher', hltcfgFileName)
0305         if (userparams.ValidateDQM):
0306             replace_map_DIST = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0307             replace_map_SEED = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0308         if (userparams.ValidateISO):
0309             replace_map_ISOL = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0310         if (userparams.ValidateRECO):
0311             replace_map_RECO = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0312             replace_map_RECO['IS_FSIM']=''
0313
0314     templatemacroFile = open(macro, 'r')
0315     macroFile = open(cfgFileName+'.C' , 'w' )
0316     replace(replace_map, templatemacroFile, macroFile)
0317
0318     if (userparams.ValidateHLT):
0319         templatemacroFile = open(macro, 'r')
0320         hltmacroFile = open(hltcfgFileName+'.C' , 'w' )
0321         replace(replace_map_HLT, templatemacroFile, hltmacroFile)
0322
0323     if (userparams.ValidateDQM):
0324         templatemacroFile = open(macroReco, 'r')
0325         recomacroFile = open(recocfgFileName+'.C' , 'w' )
0326         replace(replace_map_DIST, templatemacroFile, recomacroFile)
0327         templatemacroFile = open(macroSeed, 'r')
0328         seedmacroFile = open(seedcfgFileName+'.C' , 'w' )
0329         replace(replace_map_SEED, templatemacroFile, seedmacroFile)
0330
0331     if (userparams.ValidateISO):
0332         templatemacroFile = open(macroIsol, 'r')
0333         isolmacroFile = open(isolcfgFileName+'.C' , 'w' )
0334         replace(replace_map_ISOL, templatemacroFile, isolmacroFile)
0335
0336     if (userparams.ValidateRECO):
0337         templatemacroFile = open(macroMuonReco, 'r')
0338         recomuonmacroFile = open(recomuoncfgFileName+'.C' , 'w' )
0339         replace(replace_map_RECO, templatemacroFile, recomuonmacroFile)
0340
0341     if(userparams.Submit):
0342         myrootsubmit(cfgFileName)
0343         if (userparams.ValidateHLT):
0344             myrootsubmit(hltcfgFileName)
0345         if (userparams.ValidateDQM):
0346             myrootsubmit(recocfgFileName)
0347             myrootsubmit(seedcfgFileName)
0348         if (userparams.ValidateISO):
0349             myrootsubmit(isolcfgFileName)
0350             if (userparams.NewParams['FastSim']&userparams.RefParams['FastSim']):
0351                 shutil.move(newpath+'/MuonIsolationV_inc.pdf',newpath+'/MuonIsolationV_inc_FS.pdf')
0352         if (userparams.ValidateRECO):
0353             myrootsubmit(recomuoncfgFileName)
0354             if (userparams.NewParams['FastSim']&userparams.RefParams['FastSim']):
0355                 if (os.path.isfile(newpath+'/RecoMuonV.pdf') == True):
0356                     os.rename(newpath+'/RecoMuonV.pdf',newpath+'/RecoMuonV_FS.pdf')
0357                 else:
0358                     print('ERROR: Could not find "' + newpath + '/RecoMuonV.pdf')
0359
0360     if(userparams.Publish):
0361         newpath = GetSamplePath(userparams.NewParams,sample)
0362         newlocalsample = GetLocalSampleName(userparams.NewParams, sample)
0363         newdir=userparams.WebRepository + '/' + newpath
0364         print('>> Publishing to ' + newdir + '...')
0365         if(os.path.exists(newdir)==False):
0366             os.system('ssh '+userparams.User+'@lxplus.cern.ch mkdir -p ' + newdir)
0367             # os.makedirs(newdir)
0368         # os.system('rm '+ newlocalsample)
0369         # os.system('scp -r '+newpath+'/* ' + newdir)
0370         os.system('scp -r '+newpath+'/* '+userparams.User+'@lxplus.cern.ch:' + newdir)
0371         print('Newpath is' + newlocalsample + ' and ' + newpath)