Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-08-10 22:50:04

0001 #! /usr/bin/env python3
0002 
0003 from __future__ import print_function
0004 import os
0005 import shutil
0006 import sys
0007 import fileinput
0008 import string
0009 import new_userparams
0010 
0011 ##############################################################
0012 # Input parameters
0013 #
0014 
0015 print("Reading input parameters") 
0016 
0017 #
0018 #specify macros used to plot here
0019 #
0020 macro='macro/new_TrackValHistoPublisher.C'
0021 macroSeed='macro/SeedValHistoPublisher.C'
0022 macroReco='macro/RecoValHistoPublisher.C'
0023 macroIsol='macro/IsoValHistoPublisher.C'
0024 macroMuonReco='macro/RecoMuonValHistoPublisher.C'
0025 
0026 
0027 ##############################################################
0028 # Helper functions
0029 #
0030 def GetGuiRepository(param):
0031     splitrelease=param['Release'].split('_')
0032     genericrelease='_'.join(splitrelease[0:3])+'_x/'
0033     dqmguirepository=param['DqmGuiBaseRepo']+genericrelease
0034     return dqmguirepository
0035 
0036 def GetEosRepository(param):
0037     splitrelease=param['Release'].split('_')
0038     genericrelease='_'.join(splitrelease[0:3])+'_x/'
0039     eosRepository=param['EOSBaseRepository']+genericrelease
0040     return eosRepository
0041 
0042 def GetLabel(params):
0043     label = params['Condition']
0044     if (params['Condition'] == 'STARTUP'):
0045         label = 'START'
0046     elif (params['Condition']=='MC'):
0047         label ='MC_52_V1'
0048     label+=params['Label']
0049 
0050     if (params['FastSim']):
0051         label += '_FastSim'
0052 
0053     label += '-' + params['Version']
0054    
0055     if (params['PileUp']!='no'):
0056     label = 'PU' + params['PileUp']+ '_' + label
0057 
0058     return label
0059 
0060 
0061 def GetTag(params):
0062     tag = params['Condition']
0063     if (params['PileUp']=='no'):
0064         tag += '_noPU'
0065     else:
0066         tag += '_PU'+params['PileUp']
0067     if (params['FastSim']):
0068         tag +='_FSIM'
0069     return tag
0070 
0071 def GetFastSimSuffix(params):
0072     if (params['FastSim']):
0073         return 'FS'
0074     else:
0075         return ''
0076 
0077 def replace(map, filein, fileout):
0078     replace_items = map.items()
0079     while True:
0080         line = filein.readline()
0081         if not line: break
0082         for old, new in replace_items:
0083             line = string.replace(line, old, new)
0084         fileout.write(line)
0085     fileout.close()
0086     filein.close()
0087 
0088 def myrootsubmit(cfgfile):
0089     command='root -b -q -l '+ cfgfile + '.C' + '>  macro.' + cfgfile +'.log'
0090     print('>> Executing \'' + command + '\'')
0091     os.system(command)
0092     print('   ... executed \'' + command + '\'');
0093 
0094 def downloadfile(url):
0095     print('   + Downloading "' + url + '"...')
0096     #os.system('wget --ca-directory $X509_CERT_DIR/ --certificate=$X509_USER_PROXY --private-key=$X509_USER_PROXY '+url)
0097     stream = os.popen('/usr/bin/curl -k -O -L --capath $X509_CERT_DIR --key $X509_USER_PROXY --cert $X509_USER_PROXY -w "%{http_code}" '+ url)
0098     output=stream.readlines()
0099     if output[0]=='200':
0100         print('   + OK!')
0101     return True
0102     else:
0103         print('   + ERROR! ' + str(output[0]))
0104     print("Skipping " + url)
0105     print("Please check the name of the file in the repository: "+GetGuiRepository(new_userparams.NewParams))
0106         sys.exit('Exiting...');
0107     return False
0108 
0109 def GetSamplePath(params, sample):
0110     return params['Release']+'/'+GetTag(params)+'/'+sample+'_'+params['Energy']
0111 
0112 def GetLocalSampleName(params, sample):
0113     return GetSamplePath(params, sample) + '/val.' + sample + '.root'
0114 
0115 def createSampleDirectory(params, sample):
0116     path = GetSamplePath(params, sample)
0117     print('>> Creating directory ' + path + '...')
0118     if(os.path.exists(path)==False):
0119         os.makedirs(path)
0120     else:
0121         print('NOTE: Directory \'' + path + '\' already exists')
0122 
0123     return path
0124     
0125 def getSampleFiles(params, sample):
0126     path = GetSamplePath(params,sample)
0127     print('>> Creating directory ' + path + '...')
0128     if(os.path.exists(path)==False):
0129         os.makedirs(path)
0130     else:
0131         print('NOTE: Directory \'' + path + '\' already exists')
0132 
0133     print('\n')
0134 
0135     # Check if pdfs have already been produced
0136     checkFile = path + '/standAloneMuons.pdf'
0137     print('>> Checking if ' + checkFile + ' already exists...')
0138     if (os.path.isfile(checkFile)==True):
0139         print('   + Files of type ' + checkFile + ' exist already.')
0140         print('     Delete them first, if you really want to overwrite them')
0141         quit()
0142       
0143     print('   + The file does not exist so we continue')
0144     localsample=GetLocalSampleName(params, sample)
0145     sampleOnWeb=new_userparams.WebRepository+'/'+ localsample
0146 
0147     if (os.path.isfile(localsample)==True):
0148         print('   + ' + params['Type'] + ' sample file found at: ' +localsample + '. Using that one')
0149 
0150     elif (new_userparams.NewParams['GetFilesFrom']=='GUI'):
0151         theGuiSample = sample
0152         
0153         guiFileName='DQM_V0001_R000000001__'+theGuiSample+'_'+params['Energy']+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0154         guiFullURL=GetGuiRepository(params)+guiFileName
0155         print(">> Downloading file from the GUI: " + guiFullURL)
0156 
0157         if (downloadfile(guiFullURL)==True):
0158             print('   + Moving ' + guiFileName + ' to ' + localsample)
0159             shutil.move(guiFileName,localsample)
0160 
0161     elif ((params['GetFilesFrom']=='WEB') & (os.path.isfile(sampleOnWeb))) :
0162         print("NOTE: New file found at: "+newSample+' -> Copy that one')
0163         os.system('cp '+sampleOnWeb+' '+path)
0164 
0165     elif ((params['GetFilesFrom']=='EOS')) :
0166         print("creating a symbolic link to a file from eos: ")
0167         eosFileName='DQM_V0001_R000000001__'+sample+'_'+params['Energy']+'__'+params['Release']+'-'+GetLabel(params)+'__'+params['Format']+'.root'
0168         eosRepository=GetEosRepository(params)
0169         sampleOnEOS=eosRepository+eosFileName
0170 
0171         if (os.path.isfile(sampleOnEOS)) :
0172             print(sampleOnEOS) 
0173             #cp_command = 'cp '+sampleOnEOS+' '+path+'/val.'+ sample+'.root'
0174             #os.system(cp_command)
0175             ln_command = 'ln -s '+sampleOnEOS+' '+path+'/val.'+ sample+'.root'
0176             os.system(ln_command)
0177         else :
0178             print("ERROR: File "+sampleOnEOS+" NOT found.")
0179             quit()
0180 
0181     else:
0182         print('*** WARNING: no signal file was found')
0183 
0184 def getReplaceMap(newparams, refparams, sample, datatype, cfgkey, cfgfile):
0185     newLocalSample=GetLocalSampleName(newparams, sample)
0186     refLocalSample=GetLocalSampleName(refparams, sample)
0187     replace_map = { 'DATATYPE': datatype,
0188                     'NEW_FILE':newLocalSample,
0189                     'REF_FILE':refLocalSample,
0190                     'REF_LABEL':sample+'_'+refparams['Energy'],
0191                     'NEW_LABEL': sample+'_'+newparams['Energy'],
0192                     'REF_RELEASE':refparams['Release'],
0193                     'NEW_RELEASE':newparams['Release'],
0194                     'REFSELECTION':GetTag(refparams),
0195                     'NEWSELECTION':GetTag(newparams),
0196                     cfgkey: cfgfile
0197                     }
0198     return replace_map
0199 
0200 ##############################################################
0201 # Main program
0202 #
0203 # Initial checks
0204 # + Copying missing parameters from New to Ref
0205 print('>> Checking input parameters...')
0206 for i in new_userparams.NewParams:
0207     if i in new_userparams.RefParams:
0208         print('   + ' + i + ' set to:')
0209         print('     -' + str(new_userparams.NewParams[i]) + ' (new)')
0210         print('     -' + str(new_userparams.RefParams[i]) + ' (ref)')
0211     else:
0212         new_userparams.RefParams[i]=new_userparams.NewParams[i]
0213         print('   + ' + i + ' set to ' + str(new_userparams.NewParams[i]) + ' (both)')
0214 
0215 # + Things needed by GUI
0216 if ((new_userparams.NewParams['GetFilesFrom']=='GUI')|(new_userparams.RefParams['GetFilesFrom']=='GUI')):
0217     if os.getenv('X509_USER_PROXY','') == '':
0218         print("ERROR: It seems you did not configure your environment to be able")
0219         print("       to download files from the GUI. Your should follow these steps:")
0220         print(" > source /cvmfs/cms.cern.ch/crab3/crab.csh")
0221         print(" > voms-proxy-init --voms cms")
0222         print(" > setenv X509_CERT_DIR $HOME/.globus")
0223         print(" > setenv X509_USER_PROXY /tmp/x509up_uVWXYZ (where VWXYZ = your unix Id on lxplus)")
0224         print(" >  or similarly for bash shell")
0225         quit()
0226 
0227 if (new_userparams.NewParams['FastSim']|new_userparams.RefParams['FastSim']):
0228     new_userparams.ValidateHLT=False
0229     new_userparams.ValidateDQM=False
0230 
0231 if (new_userparams.NewParams['HeavyIons']|new_userparams.RefParams['HeavyIons']):
0232     new_userparams.ValidateHLT=False
0233     new_userparams.ValidateRECO=False
0234     new_userparams.ValidateISO=False
0235 
0236 # Iterate over new_userparams.samples
0237 print('>> Selected samples:')
0238 for sample in new_userparams.samples:
0239     print('   + ' + sample)
0240     
0241 print('>> Processing new_userparams.samples:')
0242 for sample in new_userparams.samples :
0243     print('###################################################')
0244     print('# ' + sample)
0245     print('###################################################')
0246     print('\n')
0247 
0248     # Create the directories to store the new_userparams.samples and pdfs
0249     newpath=createSampleDirectory(new_userparams.NewParams, sample)
0250     refpath=createSampleDirectory(new_userparams.RefParams, sample)
0251     print('\n')
0252 
0253     getSampleFiles(new_userparams.NewParams, sample)
0254     getSampleFiles(new_userparams.RefParams, sample)
0255         
0256     print('   + Producing macro files...')
0257     # root macro name should not contain "-"
0258     sample_m = sample
0259     if (sample == 'RelValJpsiMuMu_Pt-8'):
0260         sample_m = 'RelValJpsiMuMu_Pt8'
0261         
0262     cfgFileName=sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0263     hltcfgFileName='HLT'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0264     seedcfgFileName='DQMSEED'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0265     recocfgFileName='DQMRECO'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0266     recomuoncfgFileName='RECO'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']
0267     isolcfgFileName='ISOL'+sample_m+'_'+new_userparams.NewParams['Release']+'_'+new_userparams.RefParams['Release']    
0268 
0269     if os.path.isfile(GetLocalSampleName(new_userparams.RefParams,sample)):
0270         replace_map = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'new_TrackValHistoPublisher', cfgFileName)
0271         if (new_userparams.ValidateHLT):
0272             replace_map_HLT = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'HLT', 'new_TrackValHistoPublisher', hltcfgFileName)
0273         if (new_userparams.ValidateDQM):
0274             replace_map_DIST = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0275             replace_map_SEED = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0276         if (new_userparams.ValidateISO):
0277             replace_map_ISOL = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0278         if (new_userparams.ValidateRECO):
0279             replace_map_RECO = getReplaceMap(new_userparams.NewParams, new_userparams.RefParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0280             replace_map_RECO['IS_FSIM']=''
0281     else:
0282         print("No reference file found at: ", refpath)
0283         replace_map = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'new_TrackValHistoPublisher', cfgFileName)
0284         if (new_userparams.ValidateHLT):
0285             replace_map_HLT = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'HLT', 'new_TrackValHistoPublisher', hltcfgFileName)
0286         if (new_userparams.ValidateDQM):
0287             replace_map_DIST = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'RecoValHistoPublisher', recocfgFileName)
0288             replace_map_SEED = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'SeedValHistoPublisher', seedcfgFileName)
0289         if (new_userparams.ValidateISO):
0290             replace_map_ISOL = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'IsoValHistoPublisher', isolcfgFileName)
0291         if (new_userparams.ValidateRECO):
0292             replace_map_RECO = getReplaceMap(new_userparams.NewParams, new_userparams.NewParams, sample, 'RECO', 'RecoMuonValHistoPublisher', recomuoncfgFileName)
0293             replace_map_RECO['IS_FSIM']=''
0294 
0295     templatemacroFile = open(macro, 'r')
0296     macroFile = open(cfgFileName+'.C' , 'w' )
0297     replace(replace_map, templatemacroFile, macroFile)
0298 
0299     if (new_userparams.ValidateHLT):
0300         templatemacroFile = open(macro, 'r')
0301         hltmacroFile = open(hltcfgFileName+'.C' , 'w' )
0302         replace(replace_map_HLT, templatemacroFile, hltmacroFile)
0303 
0304     if (new_userparams.ValidateDQM):
0305         templatemacroFile = open(macroReco, 'r')
0306         recomacroFile = open(recocfgFileName+'.C' , 'w' )
0307         replace(replace_map_DIST, templatemacroFile, recomacroFile)
0308         templatemacroFile = open(macroSeed, 'r')
0309         seedmacroFile = open(seedcfgFileName+'.C' , 'w' )
0310         replace(replace_map_SEED, templatemacroFile, seedmacroFile)
0311 
0312     if (new_userparams.ValidateISO):
0313         templatemacroFile = open(macroIsol, 'r')
0314         isolmacroFile = open(isolcfgFileName+'.C' , 'w' )
0315         replace(replace_map_ISOL, templatemacroFile, isolmacroFile)
0316 
0317     if (new_userparams.ValidateRECO):
0318         templatemacroFile = open(macroMuonReco, 'r')
0319         recomuonmacroFile = open(recomuoncfgFileName+'.C' , 'w' )
0320         replace(replace_map_RECO, templatemacroFile, recomuonmacroFile)
0321 
0322     if(new_userparams.Submit):
0323         myrootsubmit(cfgFileName)
0324         if (new_userparams.ValidateHLT):
0325             myrootsubmit(hltcfgFileName)
0326         if (new_userparams.ValidateDQM):
0327             myrootsubmit(recocfgFileName)
0328             myrootsubmit(seedcfgFileName)
0329         if (new_userparams.ValidateISO):
0330             myrootsubmit(isolcfgFileName)
0331             if (new_userparams.NewParams['FastSim']&new_userparams.RefParams['FastSim']):
0332                 shutil.move(newpath+'/MuonIsolationV_inc.pdf',newpath+'/MuonIsolationV_inc_FS.pdf')
0333         if (new_userparams.ValidateRECO):
0334             myrootsubmit(recomuoncfgFileName)
0335             if (new_userparams.NewParams['FastSim']&new_userparams.RefParams['FastSim']):
0336                 if (os.path.isfile(newpath+'/RecoMuonV.pdf') == True):
0337                     os.rename(newpath+'/RecoMuonV.pdf',newpath+'/RecoMuonV_FS.pdf')
0338                 else:
0339                     print('ERROR: Could not find "' + newpath + '/RecoMuonV.pdf')
0340 
0341     os.system('mkdir '+newpath+'/PDF')
0342     os.system('mv '+newpath+'/*.pdf '+newpath+'/PDF/.')
0343 
0344     if(new_userparams.Publish):
0345         newpath = GetSamplePath(new_userparams.NewParams,sample)
0346         newlocalsample = GetLocalSampleName(new_userparams.NewParams, sample)
0347         newdir=new_userparams.WebRepository + '/' + newpath
0348         print('>> Publishing to ' + newdir + '...')
0349         if(os.path.exists(newdir)==False):
0350             os.system('ssh '+new_userparams.User+'@lxplus.cern.ch mkdir -p ' + newdir)
0351             # os.makedirs(newdir)
0352             # os.system('rm '+ newlocalsample)  
0353             # os.system('scp -r '+newpath+'/* ' + newdir)
0354         os.system('scp -r '+newpath+'/* '+new_userparams.User+'@lxplus.cern.ch:' + newdir)
0355 
0356         if(new_userparams.Publish_rootfile):            
0357             os.system('scp -r '+newpath+'/val.*.root '+new_userparams.User+'@lxplus.cern.ch:' + newdir)
0358 
0359         print('New path is ' + newlocalsample + ' and ' + newpath)