File indexing completed on 2024-11-26 02:34:38
0001
0002
0003 import os
0004 import sys
0005 import fileinput
0006 import string
0007
0008
0009
0010
0011
0012
0013 NewRelease='CMSSW_3_8_0_pre2'
0014
0015
0016
0017 startupsamples= []
0018
0019
0020 idealsamples= ['RelValSingleMuPt100']
0021
0022
0023 Algos= ['']
0024 Qualities=['']
0025
0026
0027
0028 Tracksname=''
0029
0030
0031
0032
0033
0034
0035 Sequence='reco_and_val'
0036
0037 SearchContent="*GEN-SIM-RECO*"
0038 DriverSteps="HARVESTING:validationHarvesting"
0039 if(Sequence=="harvesting"):
0040 SearchContent="*GEN-SIM-RECO*"
0041 DriverSteps="HARVESTING:validationHarvesting+dqmHarvesting"
0042 if(Sequence=="reco_and_val"):
0043 SearchContent="*GEN-SIM*HLTDEBUG*"
0044 DriverSteps="RAW2DIGI,RECO,VALIDATION"
0045 if(Sequence=="only_val"):
0046 SearchContent="*GEN-SIM-RECO*"
0047 DriverSteps="POSTRECO,VALIDATION"
0048
0049 print(Sequence+' '+SearchContent)
0050
0051 Submit=False
0052 DBS=True
0053 OneAtATime=False
0054
0055
0056 IdealTag='MC_38Y_V1'
0057 StartupTag='STARTUP_31X_v1'
0058
0059
0060 NewSelectionLabel='_test'
0061
0062 WorkDirBase = '/tmp/'
0063
0064
0065
0066
0067 defaultNevents ='-1'
0068
0069
0070 Events={}
0071
0072
0073
0074
0075
0076
0077
0078 def replace(map, filein, fileout):
0079 replace_items = map.items()
0080 while True:
0081 line = filein.readline()
0082 if not line: break
0083 for old, new in replace_items:
0084 line = string.replace(line, old, new)
0085 fileout.write(line)
0086 fileout.close()
0087 filein.close()
0088
0089
0090
0091
0092 def do_validation(samples, GlobalTagUse, trackquality, trackalgorithm):
0093 global Sequence, NewSelection, defaultNevents, Events
0094 global Tracksname,SearchContent,DriverSteps
0095 splitResult = GlobalTagUse.split('_')
0096 GlobalTag = splitResult[0]
0097 print('Search Tag: ' + GlobalTag)
0098 print('Tag to use: ' + GlobalTagUse)
0099
0100
0101 NewSelection=GlobalTag +'_noPU'
0102 if( trackquality !=''):
0103 NewSelection+='_'+trackquality
0104 if(trackalgorithm!=''):
0105 NewSelection+='_'+trackalgorithm
0106 if(trackquality =='') and (trackalgorithm==''):
0107 if(Tracksname==''):
0108 NewSelection+='_ootb'
0109 Tracks='generalTracks'
0110 else:
0111 NewSelection+= Tracks
0112 elif(Tracksname==''):
0113 Tracks='cutsRecoTracks'
0114 else:
0115 Tracks=Tracksname
0116 NewSelection+=NewSelectionLabel
0117
0118
0119 for sample in samples :
0120 print('Get information from DBS for sample', sample)
0121
0122 WorkDir = WorkDirBase+'/'+NewRelease+'/'+NewSelection+'/'+sample
0123
0124 if(os.path.exists(WorkDir)==False):
0125 os.makedirs(WorkDir)
0126
0127
0128
0129
0130 if(True):
0131
0132 cmd='dbsql "find dataset where dataset like *'
0133 cmd+=sample+'/'+NewRelease+'_'+GlobalTag+SearchContent+' "'
0134 cmd+='|grep '+sample+'|grep -v test|sort|tail -1|cut -f2 '
0135 print(cmd)
0136 dataset= os.popen(cmd).readline().strip()
0137 print('DataSet: ', dataset, '\n')
0138
0139 if(dataset!="" or DBS==False):
0140
0141
0142 cmd2='dbsql "find file where dataset like '+ dataset +'"|grep ' + sample
0143 thisFile=0
0144 filenames='import FWCore.ParameterSet.Config as cms\n'
0145 filenames+='readFiles = cms.untracked.vstring()\n'
0146 filenames+='secFiles = cms.untracked.vstring()\n'
0147 filenames+='source = cms.Source ("PoolSource",fileNames = readFiles, secondaryFileNames = secFiles)\n'
0148 filenames+='readFiles.extend( [\n'
0149 first=True
0150 print(cmd2)
0151 for line in os.popen(cmd2).readlines():
0152 filename=line.strip()
0153 thisFile=thisFile+1
0154 if first==True:
0155 filenames+="'"
0156 filenames+=filename
0157 filenames+="'"
0158 first=False
0159 else :
0160 filenames+=",\n'"
0161 filenames+=filename
0162 filenames+="'"
0163 filenames+='\n]);\n'
0164
0165
0166 if(Sequence!="harvesting" and Sequence!="reco_and_val"):
0167 cmd3='dbsql "find dataset.parent where dataset like '+ dataset +'"|grep ' + sample
0168 parentdataset=os.popen(cmd3).readline()
0169 print('Parent DataSet: ', parentdataset, '\n')
0170
0171
0172 if parentdataset!="":
0173 cmd4='dbsql "find file where dataset like '+ parentdataset +'"|grep ' + sample
0174 filenames+='secFiles.extend( [\n'
0175 first=True
0176
0177 for line in os.popen(cmd4).readlines():
0178 secfilename=line.strip()
0179 if first==True:
0180 filenames+="'"
0181 filenames+=secfilename
0182 filenames+="'"
0183 first=False
0184 else :
0185 filenames+=",\n'"
0186 filenames+=secfilename
0187 filenames+="'"
0188 filenames+='\n]);\n'
0189 else :
0190 print("No parent dataset found skipping sample: ", sample)
0191 filenames+='secFiles.extend( ( ) )\n'
0192
0193
0194 else :
0195 filenames+='secFiles.extend( ( ) )\n'
0196
0197
0198
0199
0200 cfgFileName=('%s_%s_%d') % (sample,Sequence,thisFile)
0201 print('cfgFileName ' + cfgFileName)
0202
0203
0204 sampleFileName='sample_'+Sequence
0205 sampleFile = open(sampleFileName+'.py','w' )
0206 sampleFile.write(filenames)
0207 sampleFile.close()
0208
0209
0210 if ((sample in Events)!=True):
0211 Nevents=defaultNevents
0212 else:
0213 Nevents=Events[sample]
0214
0215 symbol_map = { 'NEVENT':Nevents, 'GLOBALTAG':GlobalTagUse, 'SEQUENCE':Sequence, 'SAMPLE': sample, 'ALGORITHM':trackalgorithm, 'QUALITY':trackquality, 'TRACKS':Tracks}
0216
0217 cmdrun='cmsRun ' + WorkDir +'/'+cfgFileName+ '.py >& ' + cfgFileName + '.log < /dev/zero '
0218
0219 print(cmdrun)
0220
0221 lancialines='#!/bin/bash \n'
0222 lancialines+='cd '+ProjectBase+'/src \n'
0223 lancialines+='eval `scramv1 run -sh` \n\n'
0224 lancialines+='export PYTHONPATH=.:$PYTHONPATH \n'
0225 lancialines+='cd '+WorkDir+'\n'
0226 lancialines+='cmsRun '+cfgFileName+'.py >& ' + cfgFileName + '.log < /dev/zero \n'
0227
0228 lancialines+='mv DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO.root' + ' ' + 'val.' +sample+'.root \n'
0229
0230 lanciaName=('lancia_%s_%s_%s_%d') % (GlobalTag,sample,Sequence,thisFile)
0231 lanciaFile = open(lanciaName,'w')
0232 lanciaFile.write(lancialines)
0233 lanciaFile.close()
0234
0235 previousFileOut=''
0236 command=''
0237 nextFileOut=''
0238 specialCommand=''
0239 processName=''
0240 if(Sequence=="reco_and_val"):
0241 nextFileOut+='step2_RECO_VALIDATION.root'
0242 command+='step2'
0243 previousFileOut+='step1.root'
0244 processName+='RecVal'
0245 if(Sequence=="only_val"):
0246 nextFileOut+='step2_VALIDATION.root'
0247 command+='step2'
0248 previousFileOut+='step2.root'
0249 processName+='VAL'
0250 if(Sequence=="harvesting"):
0251 if(previousFileOut==''):
0252 previousFileOut+='step2.root'
0253 command+='harvest'
0254 nextFileOut+='harvest_out.root'
0255 specialCommand+=' --harvesting AtJobEnd'
0256 processName+='HARV'
0257
0258 driverCmdNew='cmsDriver.py '+ command
0259 driverCmdNew+=' -s '+DriverSteps
0260 driverCmdNew+=' '+processName+' '
0261 driverCmdNew+=' --mc '
0262 driverCmdNew+=' -n'+Nevents
0263 driverCmdNew+=' --conditions FrontierConditions_GlobalTag,'+GlobalTagUse+'::All'
0264 driverCmdNew+=' --filein file:'+previousFileOut
0265 driverCmdNew+=' --fileout '+nextFileOut
0266 driverCmdNew+=' --python_filename='+cfgFileName+'.py '
0267 driverCmdNew+=' --no_exec '
0268 driverCmdNew+=' --customise=Validation/RecoMuon/customise.py '
0269 driverCmdNew+=' '+specialCommand+' '
0270
0271 print(driverCmdNew)
0272 iii=os.system(driverCmdNew)
0273
0274 print(("copying py file for sample: %s %s") % (sample,lanciaName))
0275 iii = os.system('mv '+lanciaName+' '+WorkDir+'/.')
0276 print(iii)
0277 iii = os.system('mv '+sampleFileName+'.py '+WorkDir)
0278 iii = os.system('mv '+cfgFileName+'.py '+WorkDir)
0279
0280
0281
0282
0283 retcode=0
0284
0285 if(Submit):
0286 retcode=os.system(cmdrun)
0287 if os.path.isfile(sample+'.log'):
0288 os.system('mv '+sample+'.log ' + WorkDir)
0289 else:
0290 continue
0291
0292 if (retcode!=0):
0293 print('Job for sample '+ sample + ' failed. \n')
0294 else:
0295 if Sequence=="harvesting":
0296 os.system('mv DQM_V0001_R000000001__' + GlobalTag+ '__' + sample + '__Validation.root' + ' ' + WorkDir+'/val.' +sample+'.root')
0297 if os.path.isfile('fullOutput.'+sample+'.root'):
0298 os.system('mv fullOutput.'+sample+'.root ' + WorkDir)
0299 if os.path.isfile('output.'+sample+'.root'):
0300 os.system('mv output.'+sample+'.root ' + WorkDir)
0301
0302 else:
0303 print('No dataset found skipping sample: '+ sample, '\n')
0304 else:
0305 print('Validation for sample ' + sample + ' already done. Skipping this sample. \n')
0306
0307
0308
0309
0310
0311
0312
0313
0314 try:
0315
0316 if Submit:
0317 print('Will Submit job')
0318
0319 ProjectBase = os.environ["CMSSW_BASE"]
0320
0321
0322
0323
0324
0325 except KeyError:
0326 print('Error: The environment variable CMSSW_VERSION is not available.', file=sys.stderr)
0327 print(' Please run eval `scramv1 runtime -csh` to set your environment variables', file=sys.stderr)
0328 sys.exit()
0329
0330
0331
0332
0333 if os.path.isfile( 'dbsCommandLine.py')!=True:
0334 print("Failed to find the file ($DBSCMD_HOME/dbsCommandLine.py)", file=sys.stderr)
0335
0336 NewSelection=''
0337
0338 for algo in Algos:
0339 print('algo ' + algo)
0340 for quality in Qualities:
0341 print('quality ' + quality)
0342 do_validation(idealsamples, IdealTag, quality , algo)
0343 do_validation(startupsamples, StartupTag, quality , algo)
0344