File indexing completed on 2024-11-26 02:34:37
0001
0002
0003 import os
0004 import shutil
0005 import sys
0006 import fileinput
0007 import string
0008 import userparams
0009
0010
0011
0012
0013
0014 print("Reading input parameters")
0015
0016
0017
0018
0019 macro='macro/TrackValHistoPublisher.C'
0020 macroSeed='macro/SeedValHistoPublisher.C'
0021 macroReco='macro/RecoValHistoPublisher.C'
0022 macroIsol='macro/IsoValHistoPublisher.C'
0023 macroMuonReco='macro/RecoMuonValHistoPublisher.C'
0024
0025
0026
0027
0028
0029 def GetGuiRepository(param):
0030 splitrelease=param['Release'].split('_')
0031 genericrelease='_'.join(splitrelease[0:3])+'_x/'
0032 dqmguirepository=param['DqmGuiBaseRepo']+genericrelease
0033 return dqmguirepository
0034
0035 def GetLabel(params):
0036 label = params['Condition']
0037 if (params['Condition'] == 'STARTUP'):
0038 label = 'START'
0039 elif (params['Condition']=='MC'):
0040 label ='MC_52_V1'
0041 label+=params['Label']
0042 if (params['FastSim']):
0043 label += '_FastSim'
0044
0045 label += '-' + params['Version']
0046
0047 if (params['PileUp']!='no'):
0048 label = 'PU' + params['PileUp']+ '_' + label
0049
0050 return label
0051
0052
0053 def GetTag(params):
0054 tag = params['Condition']
0055 if (params['PileUp']=='no'):
0056 tag += '_noPU'
0057 else:
0058 tag += '_PU'+params['PileUp']
0059 if (params['FastSim']):
0060 tag +='_FSIM'
0061 return tag
0062
0063
0064
0065 def GetFastSimSuffix(params):
0066 if (params['FastSim']):
0067 return 'FS'
0068 else:
0069 return ''
0070
0071
0072
0073
0074 def replace(map, filein, fileout):
0075 replace_items = map.items()
0076 while True:
0077 line = filein.readline()
0078 if not line: break
0079 for old, new in replace_items:
0080 line = string.replace(line, old, new)
0081 fileout.write(line)
0082 fileout.close()
0083 filein.close()
0084
0085 def myrootsubmit(cfgfile):
0086 command='root -b -q -l '+ cfgfile + '.C' + '> macro.' + cfgfile +'.log'
0087 print('>> Executing \'' + command + '\'')
0088 os.system(command)
0089 print(' ... executed \'' + command + '\'');
0090
0091 def downloadfile(url):
0092 print(' + Downloading "' + url + '"...')
0093 stream = os.popen('/usr/bin/curl -k -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY -w "%{http_code}" '+ url)
0094 output=stream.readlines()
0095 if output[0]=='200':
0096 print(' + OK!')
0097 return True
0098 else:
0099 print(' + ERROR! ' + str(output[0]))
0100 print("Skipping " + url)
0101 print("Please check the name of the file in the repository: "+GetGuiRepository(userparams.NewParams))
0102 sys.exit('Exiting...');
0103
0104
0105 def GetSamplePath(params, sample):
0106 return params['Release']+'/'+GetTag(params)+'/'+sample
0107
0108 def GetLocalSampleName(params, sample):
0109 return GetSamplePath(params, sample) + '/val.' + sample + '.root'
0110
0111
0112 def createSampleDirectory(params, sample):
0113 path = GetSamplePath(params, sample)
0114 print('>> Creating directory ' + path + '...')
0115 if(os.path.exists(path)==False):
0116 os.makedirs(path)
0117 else:
0118 print('NOTE: Directory \'' + path + '\' already exists')
0119
0120 return path
0121
0122
0123
0124 def getSampleFiles(params, sample):
0125 path = GetSamplePath(params,sample)
0126 print('>> Creating directory ' + path + '...')
0127 if(os.path.exists(path)==False):
0128 os.makedirs(path)
0129 else:
0130 print('NOTE: Directory \'' + path + '\' already exists')
0131
0132 print('\n')
0133
0134
0135 checkFile = path + '/general_tpToTkmuAssociation' + GetFastSimSuffix(params)+'.pdf'
0136 print('>> Checking if ' + checkFile + ' already exists...')
0137 if (os.path.isfile(checkFile)==True):
0138 print(' + Files of type ' + checkFile + ' exist already.')
0139 print(' Delete them first, if you really want to overwrite them')
0140 return
0141
0142 print(' + The file does not exist so we continue')
0143 localsample=GetLocalSampleName(params, sample)
0144 sampleOnWeb=userparams.WebRepository+'/'+ localsample
0145
0146 if (os.path.isfile(localsample)==True):
0147 print(' + ' + params['Type'] + 'sample file found at: ' +localsample + '. Using that one')
0148 elif (userparams.NewParams['GetFilesFrom']=='GUI'):
0149 theGuiSample = sample
0150
0151 if (params['Condition'].find('PRE_LS1')!=-1 or params['Condition'].find('POSTLS1')!=-1):
0152 if (userparams.NewParams['FastSim']| userparams.RefParams['FastSim']):
0153 if ((sample=="RelValTTbar")|(sample=="RelValJpsiMM")|(sample=="RelValZMM")):
0154
0155 theGuiPostFixLS1 = "_13"
0156 theGuiSample = theGuiSample+theGuiPostFixLS1
0157 print("Sample is "+theGuiSample)
0158 else:
0159
0160 if(sample=="RelValZpMM_2250_13TeV_Tauola"):
0161 print("Subfix not added")
0162 theGuiPostFixLS1 = ""
0163 elif ((sample=="RelValTTbar")|(sample=="RelValJpsiMM")|(sample=="RelValZMM")):
0164
0165 theGuiPostFixLS1 = "_13"
0166 else:
0167 theGuiPostFixLS1 = "_UP15"
0168
0169 theGuiSample = theGuiSample+theGuiPostFixLS1
0170
0171 guiFileName='DQM_V0001_R000000001__'+theGuiSample+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0172 guiFullURL=GetGuiRepository(params)+guiFileName
0173 print(">> Downloading file from the GUI: " + guiFullURL)
0174
0175
0176 if (downloadfile(guiFullURL)==True):
0177 print(' + Moving ' + guiFileName + ' to ' + localsample)
0178 shutil.move(guiFileName,localsample)
0179
0180 elif (params['GetFilesFrom']=='CASTOR'):
0181 print(' + Getting new file from castor')
0182 params['Condition']=params['Condition']+GetFastSimSuffix(params)
0183 os.system('rfcp '+params['CastorRepository']+'/'+params['Release']+'_'+params['Condition']+'_'+sample+'_val.'+sample+'.root '+localsample)
0184 elif ((params['GetFilesFrom']=='WEB') & (os.path.isfile(sampleOnWeb))) :
0185 print("NOTE: New file found at: "+newSample+' -> Copy that one')
0186 os.system('cp '+sampleOnWeb+' '+path)
0187 else:
0188 print('*** WARNING: no signal file was found')
0189
0190
0191 def getReplaceMap(newparams, refparams, sample, datatype, cfgkey, cfgfile):
0192 newLocalSample=GetLocalSampleName(newparams, sample)
0193 refLocalSample=GetLocalSampleName(refparams, sample)
0194 replace_map = { 'DATATYPE': datatype,
0195 'NEW_FILE':newLocalSample,
0196 'REF_FILE':refLocalSample,
0197 'REF_LABEL':sample,
0198 'NEW_LABEL': sample,
0199 'REF_RELEASE':refparams['Release'],
0200 'NEW_RELEASE':newparams['Release'],
0201 'REFSELECTION':GetTag(refparams),
0202 'NEWSELECTION':GetTag(newparams),
0203 cfgkey: cfgfile
0204 }
0205 return replace_map
0206
0207
0208
0209
0210
0211
0212
0213
0214 print('>> Checking input parameters...')
0215 for i in userparams.NewParams:
0216 if i in userparams.RefParams:
0217 print(' + ' + i + ' set to:')
0218 print(' -' + str(userparams.NewParams[i]) + ' (new)')
0219 print(' -' + str(userparams.RefParams[i]) + ' (ref)')
0220 else:
0221 userparams.RefParams[i]=userparams.NewParams[i]
0222 print(' + ' + i + ' set to ' + str(userparams.NewParams[i]) + ' (both)')
0223
0224
0225 if ((userparams.NewParams['GetFilesFrom']=='GUI')|(userparams.RefParams['GetFilesFrom']=='GUI')):
0226 if os.getenv('X509_USER_PROXY','') == '':
0227 print("ERROR: It seems you did not configure your environment to be able")
0228 print(" to download files from the GUI. Your should follow these steps:")
0229 print(" > source /afs/cern.ch/project/gd/LCG-share/sl5/etc/profile.d/grid_env.csh")
0230 print(" > voms-proxy-init")
0231 quit()
0232
0233 if (userparams.NewParams['FastSim']|userparams.RefParams['FastSim']):
0234 userparams.ValidateDQM=False
0235
0236
0237
0238
0239
0240
0241
0242
0243
0244
0245
0246
0247
0248
0249
0250
0251
0252
0253
0254
0255
0256 print('>> Samples selected:')
0257 for sample in userparams.samples:
0258 print(' + ' + sample)
0259
0260 print('>> Processing userparams.samples:')
0261 for sample in userparams.samples :
0262 print('###################################################')
0263 print('# ' + sample)
0264 print('###################################################')
0265 print('\n')
0266
0267
0268
0269
0270 newpath=createSampleDirectory(userparams.NewParams, sample)
0271 refpath=createSampleDirectory(userparams.RefParams, sample)
0272 print('\n')
0273
0274 getSampleFiles(userparams.NewParams, sample)
0275 getSampleFiles(userparams.RefParams, sample)
0276
0277
0278 print(' + Producing macro files...')
0279 cfgFileName=sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0280 hltcfgFileName='HLT'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0281 seedcfgFileName='DQMSEED'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0282 recocfgFileName='DQMRECO'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0283 recomuoncfgFileName='RECO'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0284 isolcfgFileName='ISOL'+sample+'_'+userparams.NewParams['Release']+'_'+userparams.RefParams['Release']
0285
0286
0287 if os.path.isfile(GetLocalSampleName(userparams.RefParams,sample)):
0288 replace_map = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'TrackValHistoPublisher', cfgFileName)
0289 if (userparams.ValidateHLT):
0290 replace_map_HLT = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'HLT', 'TrackValHistoPublisher', hltcfgFileName)
0291 if (userparams.ValidateDQM):
0292 replace_map_DIST = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0293 replace_map_SEED = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0294 if (userparams.ValidateISO):
0295 replace_map_ISOL = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0296 if (userparams.ValidateRECO):
0297 replace_map_RECO = getReplaceMap(userparams.NewParams, userparams.RefParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0298 replace_map_RECO['IS_FSIM']=''
0299 else:
0300 print("No reference file found at: ", refpath)
0301 replace_map = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'TrackValHistoPublisher', cfgFileName)
0302 if (userparams.ValidateHLT):
0303 replace_map_HLT = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'HLT', 'TrackValHistoPublisher', hltcfgFileName)
0304 if (userparams.ValidateDQM):
0305 replace_map_DIST = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0306 replace_map_SEED = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0307 if (userparams.ValidateISO):
0308 replace_map_ISOL = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0309 if (userparams.ValidateRECO):
0310 replace_map_RECO = getReplaceMap(userparams.NewParams, userparams.NewParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0311 replace_map_RECO['IS_FSIM']=''
0312
0313 templatemacroFile = open(macro, 'r')
0314 macroFile = open(cfgFileName+'.C' , 'w' )
0315 replace(replace_map, templatemacroFile, macroFile)
0316
0317 if (userparams.ValidateHLT):
0318 templatemacroFile = open(macro, 'r')
0319 hltmacroFile = open(hltcfgFileName+'.C' , 'w' )
0320 replace(replace_map_HLT, templatemacroFile, hltmacroFile)
0321
0322 if (userparams.ValidateDQM):
0323 templatemacroFile = open(macroReco, 'r')
0324 recomacroFile = open(recocfgFileName+'.C' , 'w' )
0325 replace(replace_map_DIST, templatemacroFile, recomacroFile)
0326 templatemacroFile = open(macroSeed, 'r')
0327 seedmacroFile = open(seedcfgFileName+'.C' , 'w' )
0328 replace(replace_map_SEED, templatemacroFile, seedmacroFile)
0329
0330 if (userparams.ValidateISO):
0331 templatemacroFile = open(macroIsol, 'r')
0332 isolmacroFile = open(isolcfgFileName+'.C' , 'w' )
0333 replace(replace_map_ISOL, templatemacroFile, isolmacroFile)
0334
0335 if (userparams.ValidateRECO):
0336 templatemacroFile = open(macroMuonReco, 'r')
0337 recomuonmacroFile = open(recomuoncfgFileName+'.C' , 'w' )
0338 replace(replace_map_RECO, templatemacroFile, recomuonmacroFile)
0339
0340 if(userparams.Submit):
0341 myrootsubmit(cfgFileName)
0342 if (userparams.ValidateHLT):
0343 myrootsubmit(hltcfgFileName)
0344 if (userparams.ValidateDQM):
0345 myrootsubmit(recocfgFileName)
0346 myrootsubmit(seedcfgFileName)
0347 if (userparams.ValidateISO):
0348 myrootsubmit(isolcfgFileName)
0349 if (userparams.NewParams['FastSim']&userparams.RefParams['FastSim']):
0350 shutil.move(newpath+'/MuonIsolationV_inc.pdf',newpath+'/MuonIsolationV_inc_FS.pdf')
0351 if (userparams.ValidateRECO):
0352 myrootsubmit(recomuoncfgFileName)
0353 if (userparams.NewParams['FastSim']&userparams.RefParams['FastSim']):
0354 if (os.path.isfile(newpath+'/RecoMuonV.pdf') == True):
0355 os.rename(newpath+'/RecoMuonV.pdf',newpath+'/RecoMuonV_FS.pdf')
0356 else:
0357 print('ERROR: Could not find "' + newpath + '/RecoMuonV.pdf')
0358
0359 if(userparams.Publish):
0360 newpath = GetSamplePath(userparams.NewParams,sample)
0361 newlocalsample = GetLocalSampleName(userparams.NewParams, sample)
0362 newdir=userparams.WebRepository + '/' + newpath
0363 print('>> Publishing to ' + newdir + '...')
0364 if(os.path.exists(newdir)==False):
0365 os.system('ssh '+userparams.User+'@lxplus.cern.ch mkdir -p ' + newdir)
0366
0367
0368
0369 os.system('scp -r '+newpath+'/* '+userparams.User+'@lxplus.cern.ch:' + newdir)
0370 print('Newpath is' + newlocalsample + ' and ' + newpath)