Back to home page

Project CMSSW displayed by LXR
 
 

    

File indexing completed on 2024-11-26 02:34:38 

0001 #! /usr/bin/env python3
0002 
0003 import os
0004 import shutil
0005 import sys
0006 import fileinput
0007 import string
0008 import new_userparams
0009 
0010 ##############################################################
0011 # Input parameters
0012 #
0013 
0014 print("Reading input parameters") 
0015 
0016 #
0017 #specify macros used to plot here
0018 #
0019 macro='macro/new_TrackValHistoPublisher.C'
0020 macroSeed='macro/SeedValHistoPublisher.C'
0021 macroReco='macro/RecoValHistoPublisher.C'
0022 macroIsol='macro/IsoValHistoPublisher.C'
0023 macroMuonReco='macro/RecoMuonValHistoPublisher.C'
0024 
0025 
0026 ##############################################################
0027 # Helper functions
0028 #
0029 def GetGuiRepository(param):
0030     splitrelease=param['Release'].split('_')
0031     genericrelease='_'.join(splitrelease[0:3])+'_x/'
0032     dqmguirepository=param['DqmGuiBaseRepo']+genericrelease
0033     return dqmguirepository
0034 
0035 def GetEosRepository(param):
0036     splitrelease=param['Release'].split('_')
0037     genericrelease='_'.join(splitrelease[0:3])+'_x/'
0038     eosRepository=param['EOSBaseRepository']+genericrelease
0039     return eosRepository
0040 
0041 def GetLabel(params):
0042     label = params['Condition']
0043     if (params['Condition'] == 'STARTUP'):
0044         label = 'START'
0045     elif (params['Condition']=='MC'):
0046         label ='MC_52_V1'
0047     label+=params['Label']
0048 
0049     if (params['FastSim']):
0050         label += '_FastSim'
0051 
0052     label += '-' + params['Version']
0053    
0054     if (params['PileUp']!='no'):
0055         label = 'PU' + params['PileUp']+ '_' + label
0056 
0057     return label
0058 
0059 
0060 def GetTag(params):
0061     tag = params['Condition']
0062     if (params['PileUp']=='no'):
0063         tag += '_noPU'
0064     else:
0065         tag += '_PU'+params['PileUp']
0066     if (params['FastSim']):
0067         tag +='_FSIM'
0068     return tag
0069 
0070 def GetFastSimSuffix(params):
0071     if (params['FastSim']):
0072         return 'FS'
0073     else:
0074         return ''
0075 
0076 def replace(map, filein, fileout):
0077     replace_items = map.items()
0078     while True:
0079         line = filein.readline()
0080         if not line: break
0081         for old, new in replace_items:
0082             line = string.replace(line, old, new)
0083         fileout.write(line)
0084     fileout.close()
0085     filein.close()
0086 
0087 def myrootsubmit(cfgfile):
0088     command='root -b -q -l '+ cfgfile + '.C' + '>  macro.' + cfgfile +'.log'
0089     print('>> Executing \'' + command + '\'')
0090     os.system(command)
0091     print('   ... executed \'' + command + '\'');
0092 
0093 def downloadfile(url):
0094     print('   + Downloading "' + url + '"...')
0095     #os.system('wget --ca-directory $X509_CERT_DIR/ --certificate=$X509_USER_PROXY --private-key=$X509_USER_PROXY '+url)
0096     stream = os.popen('/usr/bin/curl -k -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY -w "%{http_code}" '+ url)
0097     output=stream.readlines()
0098     if output[0]=='200':
0099         print('   + OK!')
0100         return True
0101     else:
0102         print('   + ERROR! ' + str(output[0]))
0103         print("Skipping " + url)
0104         print("Please check the name of the file in the repository: "+GetGuiRepository(new_userparams.NewParams))
0105         sys.exit('Exiting...');
0106         return False
0107 
0108 def GetSamplePath(params, sample):
0109     return params['Release']+'/'+GetTag(params)+'/'+sample+'_'+params['Energy']
0110 
0111 def GetLocalSampleName(params, sample):
0112     return GetSamplePath(params, sample) + '/val.' + sample + '.root'
0113 
0114 def createSampleDirectory(params, sample):
0115     path = GetSamplePath(params, sample)
0116     print('>> Creating directory ' + path + '...')
0117     if(os.path.exists(path)==False):
0118         os.makedirs(path)
0119     else:
0120         print('NOTE: Directory \'' + path + '\' already exists')
0121 
0122     return path
0123     
0124 def getSampleFiles(params, sample):
0125     path = GetSamplePath(params,sample)
0126     print('>> Creating directory ' + path + '...')
0127     if(os.path.exists(path)==False):
0128         os.makedirs(path)
0129     else:
0130         print('NOTE: Directory \'' + path + '\' already exists')
0131 
0132     print('\n')
0133 
0134     # Check if pdfs have already been produced
0135     checkFile = path + '/standAloneMuons.pdf'
0136     print('>> Checking if ' + checkFile + ' already exists...')
0137     if (os.path.isfile(checkFile)==True):
0138         print('   + Files of type ' + checkFile + ' exist already.')
0139         print('     Delete them first, if you really want to overwrite them')
0140         quit()
0141       
0142     print('   + The file does not exist so we continue')
0143     localsample=GetLocalSampleName(params, sample)
0144     sampleOnWeb=new_userparams.WebRepository+'/'+ localsample
0145 
0146     if (os.path.isfile(localsample)==True):
0147         print('   + ' + params['Type'] + ' sample file found at: ' +localsample + '. Using that one')
0148 
0149     elif (new_userparams.NewParams['GetFilesFrom']=='GUI'):
0150         theGuiSample = sample
0151         
0152         guiFileName='DQM_V0001_R000000001__'+theGuiSample+'_'+params['Energy']+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0153         guiFullURL=GetGuiRepository(params)+guiFileName
0154         print(">> Downloading file from the GUI: " + guiFullURL)
0155 
0156         if (downloadfile(guiFullURL)==True):
0157             print('   + Moving ' + guiFileName + ' to ' + localsample)
0158             shutil.move(guiFileName,localsample)
0159 
0160     elif ((params['GetFilesFrom']=='WEB') & (os.path.isfile(sampleOnWeb))) :
0161         print("NOTE: New file found at: "+newSample+' -> Copy that one')
0162         os.system('cp '+sampleOnWeb+' '+path)
0163 
0164     elif ((params['GetFilesFrom']=='EOS')) :
0165         print("creating a symbolic link to a file from eos: ")
0166         eosFileName='DQM_V0001_R000000001__'+sample+'_'+params['Energy']+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0167         eosRepository=GetEosRepository(params)
0168         sampleOnEOS=eosRepository+eosFileName
0169 
0170         if (os.path.isfile(sampleOnEOS)) :
0171             print(sampleOnEOS) 
0172             #cp_command = 'cp '+sampleOnEOS+' '+path+'/val.'+ sample+'.root'
0173             #os.system(cp_command)
0174             ln_command = 'ln -s '+sampleOnEOS+' '+path+'/val.'+ sample+'.root'
0175             os.system(ln_command)
0176         else :
0177             print("ERROR: File "+sampleOnEOS+" NOT found.")
0178             quit()
0179 
0180     else:
0181         print('*** WARNING: no signal file was found')
0182 
0183 def getReplaceMap(newparams, refparams, sample, datatype, cfgkey, cfgfile):
0184     newLocalSample=GetLocalSampleName(newparams, sample)
0185     refLocalSample=GetLocalSampleName(refparams, sample)
0186     replace_map = { 'DATATYPE': datatype,
0187                     'NEW_FILE':newLocalSample,
0188                     'REF_FILE':refLocalSample,
0189                     'REF_LABEL':sample+'_'+refparams['Energy'],
0190                     'NEW_LABEL': sample+'_'+newparams['Energy'],
0191                     'REF_RELEASE':refparams['Release'],
0192                     'NEW_RELEASE':newparams['Release'],
0193                     'REFSELECTION':GetTag(refparams),
0194                     'NEWSELECTION':GetTag(newparams),
0195                     cfgkey: cfgfile
0196                     }
0197     return replace_map
0198 
0199 ##############################################################
0200 # Main program
0201 #
0202 # Initial checks
0203 # + Copying missing parameters from New to Ref
0204 print('>> Checking input parameters...')
0205 for i in new_userparams.NewParams:
0206     if i in new_userparams.RefParams:
0207         print('   + ' + i + ' set to:')
0208         print('     -' + str(new_userparams.NewParams[i]) + ' (new)')
0209         print('     -' + str(new_userparams.RefParams[i]) + ' (ref)')
0210     else:
0211         new_userparams.RefParams[i]=new_userparams.NewParams[i]
0212         print('   + ' + i + ' set to ' + str(new_userparams.NewParams[i]) + ' (both)')
0213 
0214 # + Things needed by GUI
0215 if ((new_userparams.NewParams['GetFilesFrom']=='GUI')|(new_userparams.RefParams['GetFilesFrom']=='GUI')):
0216     if os.getenv('X509_USER_PROXY','') == '':
0217         print("ERROR: It seems you did not configure your environment to be able")
0218         print("       to download files from the GUI. Your should follow these steps:")
0219         print(" > source /cvmfs/cms.cern.ch/crab3/crab.csh")
0220         print(" > voms-proxy-init --voms cms")
0221         print(" > setenv X509_CERT_DIR $HOME/.globus")
0222         print(" > setenv X509_USER_PROXY /tmp/x509up_uVWXYZ (where VWXYZ = your unix Id on lxplus)")
0223         print(" >  or similarly for bash shell")
0224         quit()
0225 
0226 if (new_userparams.NewParams['FastSim']|new_userparams.RefParams['FastSim']):
0227     new_userparams.ValidateHLT=False
0228     new_userparams.ValidateDQM=False
0229 
0230 if (new_userparams.NewParams['HeavyIons']|new_userparams.RefParams['HeavyIons']):
0231     new_userparams.ValidateHLT=False
0232     new_userparams.ValidateRECO=False
0233     new_userparams.ValidateISO=False
0234 
0235 # Iterate over new_userparams.samples
0236 print('>> Selected samples:')
0237 for sample in new_userparams.samples:
0238     print('   + ' + sample)
0239     
0240 print('>> Processing new_userparams.samples:')
0241 for sample in new_userparams.samples :
0242     print('###################################################')
0243     print('# ' + sample)
0244     print('###################################################')
0245     print('\n')
0246 
0247     # Create the directories to store the new_userparams.samples and pdfs
0248     newpath=createSampleDirectory(new_userparams.NewParams, sample)
0249     refpath=createSampleDirectory(new_userparams.RefParams, sample)
0250     print('\n')
0251 
0252     getSampleFiles(new_userparams.NewParams, sample)
0253     getSampleFiles(new_userparams.RefParams, sample)
0254         
0255     print('   + Producing macro files...')
0256     # root macro name should not contain "-"
0257     sample_m = sample
0258     if (sample == 'RelValJpsiMuMu_Pt-8'):
0259         sample_m = 'RelValJpsiMuMu_Pt8'
0260         
0261     cfgFileName=sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0262     hltcfgFileName='HLT'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0263     seedcfgFileName='DQMSEED'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0264     recocfgFileName='DQMRECO'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0265     recomuoncfgFileName='RECO'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0266     isolcfgFileName='ISOL'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']    
0267 
0268     if os.path.isfile(GetLocalSampleName(new_userparams.RefParams,sample)):
0269         replace_map = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'new_TrackValHistoPublisher', cfgFileName)
0270         if (new_userparams.ValidateHLT):
0271             replace_map_HLT = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'HLT', 'new_TrackValHistoPublisher', hltcfgFileName)
0272         if (new_userparams.ValidateDQM):
0273             replace_map_DIST = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0274             replace_map_SEED = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0275         if (new_userparams.ValidateISO):
0276             replace_map_ISOL = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0277         if (new_userparams.ValidateRECO):
0278             replace_map_RECO = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0279             replace_map_RECO['IS_FSIM']=''
0280     else:
0281         print("No reference file found at: ", refpath)
0282         replace_map = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'new_TrackValHistoPublisher', cfgFileName)
0283         if (new_userparams.ValidateHLT):
0284             replace_map_HLT = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'HLT', 'new_TrackValHistoPublisher', hltcfgFileName)
0285         if (new_userparams.ValidateDQM):
0286             replace_map_DIST = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0287             replace_map_SEED = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0288         if (new_userparams.ValidateISO):
0289             replace_map_ISOL = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0290         if (new_userparams.ValidateRECO):
0291             replace_map_RECO = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0292             replace_map_RECO['IS_FSIM']=''
0293 
0294     templatemacroFile = open(macro, 'r')
0295     macroFile = open(cfgFileName+'.C' , 'w' )
0296     replace(replace_map, templatemacroFile, macroFile)
0297 
0298     if (new_userparams.ValidateHLT):
0299         templatemacroFile = open(macro, 'r')
0300         hltmacroFile = open(hltcfgFileName+'.C' , 'w' )
0301         replace(replace_map_HLT, templatemacroFile, hltmacroFile)
0302 
0303     if (new_userparams.ValidateDQM):
0304         templatemacroFile = open(macroReco, 'r')
0305         recomacroFile = open(recocfgFileName+'.C' , 'w' )
0306         replace(replace_map_DIST, templatemacroFile, recomacroFile)
0307         templatemacroFile = open(macroSeed, 'r')
0308         seedmacroFile = open(seedcfgFileName+'.C' , 'w' )
0309         replace(replace_map_SEED, templatemacroFile, seedmacroFile)
0310 
0311     if (new_userparams.ValidateISO):
0312         templatemacroFile = open(macroIsol, 'r')
0313         isolmacroFile = open(isolcfgFileName+'.C' , 'w' )
0314         replace(replace_map_ISOL, templatemacroFile, isolmacroFile)
0315 
0316     if (new_userparams.ValidateRECO):
0317         templatemacroFile = open(macroMuonReco, 'r')
0318         recomuonmacroFile = open(recomuoncfgFileName+'.C' , 'w' )
0319         replace(replace_map_RECO, templatemacroFile, recomuonmacroFile)
0320 
0321     if(new_userparams.Submit):
0322         myrootsubmit(cfgFileName)
0323         if (new_userparams.ValidateHLT):
0324             myrootsubmit(hltcfgFileName)
0325         if (new_userparams.ValidateDQM):
0326             myrootsubmit(recocfgFileName)
0327             myrootsubmit(seedcfgFileName)
0328         if (new_userparams.ValidateISO):
0329             myrootsubmit(isolcfgFileName)
0330             if (new_userparams.NewParams['FastSim']&new_userparams.RefParams['FastSim']):
0331                 shutil.move(newpath+'/MuonIsolationV_inc.pdf',newpath+'/MuonIsolationV_inc_FS.pdf')
0332         if (new_userparams.ValidateRECO):
0333             myrootsubmit(recomuoncfgFileName)
0334             if (new_userparams.NewParams['FastSim']&new_userparams.RefParams['FastSim']):
0335                 if (os.path.isfile(newpath+'/RecoMuonV.pdf') == True):
0336                     os.rename(newpath+'/RecoMuonV.pdf',newpath+'/RecoMuonV_FS.pdf')
0337                 else:
0338                     print('ERROR: Could not find "' + newpath + '/RecoMuonV.pdf')
0339 
0340     os.system('mkdir '+newpath+'/PDF')
0341     os.system('mv '+newpath+'/*.pdf '+newpath+'/PDF/.')
0342 
0343     if(new_userparams.Publish):
0344         newpath = GetSamplePath(new_userparams.NewParams,sample)
0345         newlocalsample = GetLocalSampleName(new_userparams.NewParams, sample)
0346         newdir=new_userparams.WebRepository + '/' + newpath
0347         print('>> Publishing to ' + newdir + '...')
0348         if(os.path.exists(newdir)==False):
0349             os.system('ssh '+new_userparams.User+'@lxplus.cern.ch mkdir -p ' + newdir)
0350             # os.makedirs(newdir)
0351             # os.system('rm '+ newlocalsample)  
0352             # os.system('scp -r '+newpath+'/* ' + newdir)
0353         os.system('scp -r '+newpath+'/* '+new_userparams.User+'@lxplus.cern.ch:' + newdir)
0354 
0355         if(new_userparams.Publish_rootfile):            
0356             os.system('scp -r '+newpath+'/val.*.root '+new_userparams.User+'@lxplus.cern.ch:' + newdir)
0357 
0358         print('New path is ' + newlocalsample + ' and ' + newpath)
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.2.1 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!