Validation/test/trackingPerformanceValidation.py

0001 #! /usr/bin/env python
0002
0003 import os
0004 import sys
0005 import fileinput
0006 import string
0007
0008 ##########################################################################
0009 ##########################################################################
0010 ######### User variables
0011
0012 #Reference release
0013 RefRelease='CMSSW_3_3_0_pre5'
0014
0015 #Relval release (set if different from $CMSSW_VERSION)
0016 NewRelease='CMSSW_3_3_0_pre6'
0017
0018 # FastSim flags:
0019 NewFastSim=True
0020 RefFastSim=False
0021
0022 # startup and ideal sample list
0023
0024 #This are the standard relvals (startup)
0025 #startupsamples= ['RelValTTbar', 'RelValMinBias', 'RelValQCD_Pt_3000_3500']
0026 #startupsamples= ['RelValTTbar']  # FASTSIM
0027
0028 #This is pileup sample
0029 #startupsamples= ['RelValTTbar_Tauola']
0030
0031 #to skip startup samples:
0032 startupsamples= []
0033
0034 #This are the standard relvals (ideal)
0035 #idealsamples= ['RelValSingleMuPt1', 'RelValSingleMuPt10', 'RelValSingleMuPt100', 'RelValSinglePiPt1', 'RelValSinglePiPt10', 'RelValSinglePiPt100', 'RelValSingleElectronPt35', 'RelValTTbar', 'RelValQCD_Pt_3000_3500','RelValMinBias']
0036 #idealsamples= ['RelValSinglePiPt1']  # FASTSIM
0037 #idealsamples= ['RelValSinglePiPt1','RelValSingleMuPt10']
0038 idealsamples= ['RelValSingleMuPt10']
0039
0040 #This is pileup sample
0041 #idealsamples= ['RelValZmumuJets_Pt_20_300_GEN']
0042
0043 #summer09 preproduction (the character '-' must be avoided)
0044 #idealsamples= ['InclusiveMu5_Pt250__Summer09', 'InclusiveMu5_Pt50__Summer09', 'MinBias_herwig__Summer09', 'TTbar__Summer09']
0045
0046 #to skip ideal samples:
0047 #idealsamples= []
0048
0049
0050
0051 # track algorithm name and quality. Can be a list.
0052 #Algos= ['ootb', 'initialStep', 'lowPtTripletStep']
0053 Algos= ['ootb', 'initialStep', 'lowPtTripletStep','pixelPairStep','detachedTripletStep','mixedTripletStep','pixelLessStep']
0054 #Qualities=['']
0055 Qualities=['', 'highPurity']
0056
0057 #Leave unchanged unless the track collection name changes
0058 Tracksname=''
0059
0060 # Sequence. Possible values:
0061 #   -only_validation
0062 #   -re_tracking
0063 #   -digi2track
0064 #   -only_validation_and_TP
0065 #   -re_tracking_and_TP
0066 #   -digi2track_and_TP
0067 #   -harvesting
0068 #   -preproduction
0069 #   -comparison_only
0070
0071 #Sequence='preproduction'
0072 Sequence='comparison_only'
0073
0074
0075 # Ideal and Statup tags
0076 IdealTag='MC_31X_V9'
0077 StartupTag='STARTUP31X_V8'
0078 if (NewFastSim):
0079     IdealTag+='_FastSim'
0080     StartupTag+='FastSim'
0081 IdealTagRef='MC_31X_V8'
0082 StartupTagRef='STARTUP31X_V7'
0083 if (RefFastSim):
0084     IdealTagRef+='_FastSim'
0085     StartupTagRef+='FastSim'
0086
0087 # PileUp: PU . No PileUp: noPU
0088 PileUp='_noPU'
0089
0090 ReferenceSelection=IdealTagRef+PileUp
0091 StartupReferenceSelection=StartupTagRef+PileUp
0092
0093 if (NewFastSim):
0094     NewFormat='GEN-SIM-DIGI-RECO'
0095 else:
0096     NewFormat='GEN-SIM-RECO'
0097
0098 # Default label is GlobalTag_noPU__Quality_Algo. Change this variable if you want to append an additional string.
0099 NewSelectionLabel=''
0100 #NewSelectionLabel='test2_logpt'
0101
0102
0103 #Reference and new repository
0104 RefRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance'
0105 NewRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance'
0106
0107 castorHarvestedFilesDirectory='/castor/cern.ch/user/n/nuno/relval/harvest/'
0108 #for preproduction samples:
0109 #RefRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance/preproduction'
0110 #NewRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance/preproduction'
0111
0112 #Default Nevents
0113 defaultNevents ='-1'
0114
0115 #Put here the number of event to be processed for specific samples (numbers must be strings) if not specified is defaultNevents:
0116 Events={}
0117 #Events={'RelValTTbar':'100'}
0118
0119 # template file names. Usually should not be changed.
0120 cfg='trackingPerformanceValidation_cfg.py'
0121 macro='macro/TrackValHistoPublisher.C'
0122
0123
0124
0125 #########################################################################
0126 #########################################################################
0127 ############ Functions
0128
0129 def replace(map, filein, fileout):
0130     replace_items = map.items()
0131     while True:
0132         line = filein.readline()
0133         if not line: break
0134         for old, new in replace_items:
0135             line = string.replace(line, old, new)
0136         fileout.write(line)
0137     fileout.close()
0138     filein.close()
0139
0140 ############################################
0141
0142
0143 def do_validation(samples, GlobalTag, trackquality, trackalgorithm):
0144     global Sequence, RefSelection, RefRepository, NewSelection, NewRepository, defaultNevents, Events, castorHarvestedFilesDirectory
0145     global cfg, macro, Tracksname
0146     print('Tag: ' + GlobalTag)
0147     tracks_map = { 'ootb':'general_AssociatorByHitsRecoDenom','initialStep':'cutsRecoZero_AssociatorByHitsRecoDenom','lowPtTripletStep':'cutsRecoFirst_AssociatorByHitsRecoDenom','pixelPairStep':'cutsRecoSecond_AssociatorByHitsRecoDenom','detachedTripletStep':'cutsRecoThird_AssociatorByHitsRecoDenom','mixedTripletStep':'cutsRecoFourth_AssociatorByHitsRecoDenom','pixelLessStep':'cutsRecoFifth_AssociatorByHitsRecoDenom'}
0148     tracks_map_hp = { 'ootb':'cutsRecoHp_AssociatorByHitsRecoDenom','initialStep':'cutsRecoZeroHp_AssociatorByHitsRecoDenom','lowPtTripletStep':'cutsRecoFirstHp_AssociatorByHitsRecoDenom','pixelPairStep':'cutsRecoSecondHp_AssociatorByHitsRecoDenom','detachedTripletStep':'cutsRecoThirdHp_AssociatorByHitsRecoDenom','mixedTripletStep':'cutsRecoFourthHp_AssociatorByHitsRecoDenom','pixelLessStep':'cutsRecoFifthHp_AssociatorByHitsRecoDenom'}
0149     if(trackalgorithm=='initialStep' or trackalgorithm=='ootb'):
0150         mineff='0.80'
0151         maxeff='1.01'
0152         maxfake='0.7'
0153     else:
0154         mineff='0.0'
0155         maxeff='1.0'
0156         maxfake='0.8'
0157     #build the New Selection name
0158     NewSelection=GlobalTag + '_' + PileUp
0159     if (NewFastSim):
0160         NewSelection+='_FastSim'
0161     if( trackquality !=''):
0162         NewSelection+='_'+trackquality
0163     if(trackalgorithm!=''and not(trackalgorithm=='ootb' and trackquality !='')):
0164         NewSelection+='_'+trackalgorithm
0165     if(trackquality =='') and (trackalgorithm==''):
0166         if(Tracksname==''):
0167             NewSelection+='_ootb'
0168             Tracks='generalTracks'
0169         else:
0170            NewSelection+= Tracks
0171     if(Tracksname==''):
0172         Tracks='cutsRecoTracks'
0173     else:
0174         Tracks=Tracksname
0175     NewSelection+=NewSelectionLabel
0176     listofdatasets = open('listofdataset.txt' , 'w' )
0177     #loop on all the requested samples
0178     for sample in samples :
0179         templatecfgFile = open(cfg, 'r')
0180         templatemacroFile = open(macro, 'r')
0181 #        newdir=NewRepository+'/'+NewRelease+'/'+NewSelection+'/'+sample
0182         newdir=NewRelease+'/'+NewSelection+'/'+sample
0183     cfgFileName=sample+GlobalTag
0184         #check if the sample is already done
0185         if(os.path.isfile(newdir+'/building.pdf' )!=True):
0186
0187             if( Sequence=="harvesting"):
0188                 harvestedfile='./DQM_V0001_R000000001__' + GlobalTag+ '__' + sample + '__Validation.root'
0189             elif( Sequence=="preproduction"):
0190                 harvestedfile='./DQM_V0001_R000000001__' + sample+ '-' + GlobalTag + '_preproduction_312-v1__'+NewFormat+'_1.root'
0191             elif( Sequence=="comparison_only"):
0192                 harvestedfile='./DQM_V0001_R000000001__' + sample+ '__' + NewRelease+ '-' +GlobalTag + '-v1__'+NewFormat+'.root'
0193                 if(os.path.exists(harvestedfile)==False):
0194                     cpcmd='rfcp '+ castorHarvestedFilesDirectory+ NewRelease +'/' + harvestedfile + ' .'
0195                     returncode=os.system(cpcmd)
0196                     if (returncode!=0):
0197                         print('copy of harvested file from castor for sample ' + sample + ' failed')
0198                         print('try backup repository...')
0199                         NewCondition='MC'
0200                         if (NewFastSim):
0201                             NewCondition=NewCondition+'_FSIM'
0202                         cpcmd='rfcp /castor/cern.ch/user/a/aperrott/ValidationRecoMuon/'+NewRelease+'_'+NewCondition+'_'+sample+'_val.'+sample+'.root ./'+harvestedfile
0203                         returncode=os.system(cpcmd)
0204                         if (returncode!=0):
0205                             continue
0206
0207             print(' Harvested file : '+harvestedfile)
0208
0209             #search the primary dataset
0210             if( Sequence!="comparison_only"):
0211                 print('Get information from DBS for sample', sample)
0212                 cmd='dbsql "find  dataset where dataset like *'
0213                 #            cmd+=sample+'/'+NewRelease+'_'+GlobalTag+'*GEN-SIM-DIGI-RAW-HLTDEBUG-RECO* "'
0214                 cmd+=sample+'/'+NewRelease+'_'+GlobalTag+'*'+NewFormat+'* order by dataset.createdate "'
0215                 cmd+='|grep '+sample+'|grep -v test|tail -1'
0216                 print(cmd)
0217                 dataset= os.popen(cmd).readline().strip()
0218                 print('DataSet:  ', dataset, '\n')
0219
0220                 #Check if a dataset is found
0221                 if dataset!="":
0222                     listofdatasets.write(dataset)
0223                     #Find and format the list of files
0224                     cmd2='dbsql "find file where dataset like '+ dataset +'"|grep ' + sample
0225                     filenames='import FWCore.ParameterSet.Config as cms\n'
0226                     filenames+='readFiles = cms.untracked.vstring()\n'
0227                     filenames+='secFiles = cms.untracked.vstring()\n'
0228                     filenames+='source = cms.Source ("PoolSource",fileNames = readFiles, secondaryFileNames = secFiles)\n'
0229                     filenames+='readFiles.extend( [\n'
0230                     first=True
0231                     print(cmd2)
0232                     for line in os.popen(cmd2).readlines():
0233                         filename=line.strip()
0234                         if first==True:
0235                             filenames+="'"
0236                             filenames+=filename
0237                             filenames+="'"
0238                             first=False
0239                         else :
0240                             filenames+=",\n'"
0241                             filenames+=filename
0242                             filenames+="'"
0243                     filenames+=']);\n'
0244
0245                     # if not harvesting find secondary file names
0246                     if(Sequence!="preproduction"):
0247                             cmd3='dbsql  "find dataset.parent where dataset like '+ dataset +'"|grep ' + sample
0248                             parentdataset=os.popen(cmd3).readline()
0249                             print('Parent DataSet:  ', parentdataset, '\n')
0250
0251                     #Check if a dataset is found
0252                             if parentdataset!="":
0253                                     cmd4='dbsql  "find file where dataset like '+ parentdataset +'"|grep ' + sample
0254                                     filenames+='secFiles.extend( [\n'
0255                                     first=True
0256
0257                                     for line in os.popen(cmd4).readlines():
0258                                         secfilename=line.strip()
0259                                         if first==True:
0260                                             filenames+="'"
0261                                             filenames+=secfilename
0262                                             filenames+="'"
0263                                             first=False
0264                                         else :
0265                                             filenames+=",\n'"
0266                                             filenames+=secfilename
0267                                             filenames+="'"
0268                                     filenames+='\n ]);\n'
0269                             else :
0270                                     print("No primary dataset found skipping sample: ", sample)
0271                                     continue
0272                     else :
0273                             filenames+='secFiles.extend( (               ) )'
0274
0275                     cfgFile = open(cfgFileName+'.py' , 'w' )
0276                     cfgFile.write(filenames)
0277
0278                     if ((sample in Events)!=True):
0279                             Nevents=defaultNevents
0280                     else:
0281                             Nevents=Events[sample]
0282                     thealgo=trackalgorithm
0283                     thequality=trackquality
0284                     if(trackalgorithm=='ootb'):
0285                         thealgo=''
0286                     if(thealgo!=''):
0287                         thealgo='\''+thealgo+'\''
0288                     if(trackquality!=''):
0289                         thequality='\''+trackquality+'\''
0290                     symbol_map = { 'NEVENT':Nevents, 'GLOBALTAG':GlobalTag, 'SEQUENCE':Sequence, 'SAMPLE': sample, 'ALGORITHM':thealgo, 'QUALITY':thequality, 'TRACKS':Tracks}
0291
0292
0293                     cfgFile = open(cfgFileName+'.py' , 'a' )
0294                     replace(symbol_map, templatecfgFile, cfgFile)
0295                     if(( (Sequence=="harvesting" or Sequence=="preproduction" or Sequence=="comparison_only") and os.path.isfile(harvestedfile) )==False):
0296                         # if the file is already harvested do not run the job again
0297                         cmdrun='cmsRun ' +cfgFileName+ '.py >&  ' + cfgFileName + '.log < /dev/zero '
0298                         retcode=os.system(cmdrun)
0299                     else:
0300                         retcode=0
0301
0302                 else:
0303                     print('No dataset found skipping sample: '+ sample, '\n')
0304                     continue
0305             else:
0306                 print(' Used sequence : '+Sequence)
0307                 retcode = 0
0308
0309             if (retcode!=0):
0310                 print('Job for sample '+ sample + ' failed. \n')
0311             else:
0312                 if (Sequence=="harvesting" or Sequence=="preproduction" or Sequence=="comparison_only"):
0313                     #copy only the needed histograms
0314                     if(trackquality==""):
0315                         rootcommand='root -b -q -l CopySubdir.C\\('+ '\\\"'+harvestedfile+'\\\",\\\"val.' +sample+'.root\\\",\\\"'+ tracks_map[trackalgorithm]+ '\\\"\\) >& /dev/null'
0316                         os.system(rootcommand)
0317                     elif(trackquality=="highPurity"):
0318                         rootcommand='root -b -q -l CopySubdir.C\\('+ '\\\"'+harvestedfile+'\\\",\\\"val.' +sample+'.root\\\",\\\"'+ tracks_map_hp[trackalgorithm]+ '\\\"\\) >& /dev/null'
0319                         os.system(rootcommand)
0320
0321
0322                 referenceSample=RefRepository+'/'+RefRelease+'/'+RefSelection+'/'+sample+'/'+'val.'+sample+'.root'
0323                 if os.path.isfile(referenceSample ):
0324                     replace_map = { 'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':RefRelease+'/'+RefSelection+'/val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL': sample, 'REF_RELEASE':RefRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':RefSelection, 'NEWSELECTION':NewSelection, 'TrackValHistoPublisher': cfgFileName, 'MINEFF':mineff, 'MAXEFF':maxeff, 'MAXFAKE':maxfake}
0325
0326                     if(os.path.exists(RefRelease+'/'+RefSelection)==False):
0327                         os.makedirs(RefRelease+'/'+RefSelection)
0328                         os.system('cp ' + referenceSample+ ' '+RefRelease+'/'+RefSelection)
0329                 else:
0330 #                    print "No reference file found at: ", RefRelease+'/'+RefSelection
0331                     print("No reference file found at: ", referenceSample)
0332                     replace_map = { 'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':'val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL': sample, 'REF_RELEASE':NewRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':NewSelection, 'NEWSELECTION':NewSelection, 'TrackValHistoPublisher': cfgFileName, 'MINEFF':mineff, 'MAXEFF':maxeff, 'MAXFAKE':maxfake}
0333
0334
0335                 macroFile = open(cfgFileName+'.C' , 'w' )
0336                 replace(replace_map, templatemacroFile, macroFile)
0337
0338
0339                 os.system('root -b -q -l '+ cfgFileName+'.C'+ '>  macro.'+cfgFileName+'.log')
0340
0341
0342                 if(os.path.exists(newdir)==False):
0343                     os.makedirs(newdir)
0344
0345                 print("moving pdf files for sample: " , sample)
0346                 os.system('mv  *.pdf ' + newdir)
0347
0348                 print("moving root file for sample: " , sample)
0349                 os.system('mv val.'+ sample+ '.root ' + newdir)
0350
0351                 print("copy py file for sample: " , sample)
0352                 os.system('cp '+cfgFileName+'.py ' + newdir)
0353
0354
0355         else:
0356             print('Validation for sample ' + sample + ' already done. Skipping this sample. \n')
0357
0358
0359
0360
0361 ##########################################################################
0362 #########################################################################
0363 ######## This is the main program
0364 if(NewRelease==''):
0365     try:
0366         #Get some environment variables to use
0367         NewRelease     = os.environ["CMSSW_VERSION"]
0368
0369     except KeyError:
0370         print('Error: The environment variable CMSSW_VERSION is not available.', file=sys.stderr)
0371         print('       Please run cmsenv', file=sys.stderr)
0372         sys.exit()
0373 else:
0374     try:
0375         #Get some environment variables to use
0376         os.environ["CMSSW_VERSION"]
0377
0378     except KeyError:
0379         print('Error: CMSSW environment variables are not available.', file=sys.stderr)
0380         print('       Please run cmsenv', file=sys.stderr)
0381         sys.exit()
0382
0383
0384
0385 NewSelection=''
0386
0387 for algo in Algos:
0388     for quality in Qualities:
0389         RefSelection=ReferenceSelection
0390         if( quality !=''):
0391             RefSelection+='_'+quality
0392         if(algo!=''and not(algo=='ootb' and quality !='')):
0393             RefSelection+='_'+algo
0394         if(quality =='') and (algo==''):
0395             RefSelection+='_ootb'
0396         do_validation(idealsamples, IdealTag, quality , algo)
0397         RefSelection=StartupReferenceSelection
0398         if( quality !=''):
0399             RefSelection+='_'+quality
0400         if(algo!=''and not(algo=='ootb' and quality !='')):
0401             RefSelection+='_'+algo
0402         if(quality =='') and (algo==''):
0403             RefSelection+='_ootb'
0404         do_validation(startupsamples, StartupTag, quality , algo)
0405