Validation/test/trackingPerformanceValidation.py

0001 #! /usr/bin/env python
0002
0003 from __future__ import print_function
0004 import os
0005 import sys
0006 import fileinput
0007 import string
0008
0009 ##########################################################################
0010 ##########################################################################
0011 ######### User variables
0012
0013 #Reference release
0014 RefRelease='CMSSW_3_3_0_pre5'
0015
0016 #Relval release (set if different from $CMSSW_VERSION)
0017 NewRelease='CMSSW_3_3_0_pre6'
0018
0019 # FastSim flags:
0020 NewFastSim=True
0021 RefFastSim=False
0022
0023 # startup and ideal sample list
0024
0025 #This are the standard relvals (startup)
0026 #startupsamples= ['RelValTTbar', 'RelValMinBias', 'RelValQCD_Pt_3000_3500']
0027 #startupsamples= ['RelValTTbar']  # FASTSIM
0028
0029 #This is pileup sample
0030 #startupsamples= ['RelValTTbar_Tauola']
0031
0032 #to skip startup samples:
0033 startupsamples= []
0034
0035 #This are the standard relvals (ideal)
0036 #idealsamples= ['RelValSingleMuPt1', 'RelValSingleMuPt10', 'RelValSingleMuPt100', 'RelValSinglePiPt1', 'RelValSinglePiPt10', 'RelValSinglePiPt100', 'RelValSingleElectronPt35', 'RelValTTbar', 'RelValQCD_Pt_3000_3500','RelValMinBias']
0037 #idealsamples= ['RelValSinglePiPt1']  # FASTSIM
0038 #idealsamples= ['RelValSinglePiPt1','RelValSingleMuPt10']
0039 idealsamples= ['RelValSingleMuPt10']
0040
0041 #This is pileup sample
0042 #idealsamples= ['RelValZmumuJets_Pt_20_300_GEN']
0043
0044 #summer09 preproduction (the character '-' must be avoided)
0045 #idealsamples= ['InclusiveMu5_Pt250__Summer09', 'InclusiveMu5_Pt50__Summer09', 'MinBias_herwig__Summer09', 'TTbar__Summer09']
0046
0047 #to skip ideal samples:
0048 #idealsamples= []
0049
0050
0051
0052 # track algorithm name and quality. Can be a list.
0053 #Algos= ['ootb', 'initialStep', 'lowPtTripletStep']
0054 Algos= ['ootb', 'initialStep', 'lowPtTripletStep','pixelPairStep','detachedTripletStep','mixedTripletStep','pixelLessStep']
0055 #Qualities=['']
0056 Qualities=['', 'highPurity']
0057
0058 #Leave unchanged unless the track collection name changes
0059 Tracksname=''
0060
0061 # Sequence. Possible values:
0062 #   -only_validation
0063 #   -re_tracking
0064 #   -digi2track
0065 #   -only_validation_and_TP
0066 #   -re_tracking_and_TP
0067 #   -digi2track_and_TP
0068 #   -harvesting
0069 #   -preproduction
0070 #   -comparison_only
0071
0072 #Sequence='preproduction'
0073 Sequence='comparison_only'
0074
0075
0076 # Ideal and Statup tags
0077 IdealTag='MC_31X_V9'
0078 StartupTag='STARTUP31X_V8'
0079 if (NewFastSim):
0080     IdealTag+='_FastSim'
0081     StartupTag+='FastSim'
0082 IdealTagRef='MC_31X_V8'
0083 StartupTagRef='STARTUP31X_V7'
0084 if (RefFastSim):
0085     IdealTagRef+='_FastSim'
0086     StartupTagRef+='FastSim'
0087
0088 # PileUp: PU . No PileUp: noPU
0089 PileUp='_noPU'
0090
0091 ReferenceSelection=IdealTagRef+PileUp
0092 StartupReferenceSelection=StartupTagRef+PileUp
0093
0094 if (NewFastSim):
0095     NewFormat='GEN-SIM-DIGI-RECO'
0096 else:
0097     NewFormat='GEN-SIM-RECO'
0098
0099 # Default label is GlobalTag_noPU__Quality_Algo. Change this variable if you want to append an additional string.
0100 NewSelectionLabel=''
0101 #NewSelectionLabel='test2_logpt'
0102
0103
0104 #Reference and new repository
0105 RefRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance'
0106 NewRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance'
0107
0108 castorHarvestedFilesDirectory='/castor/cern.ch/user/n/nuno/relval/harvest/'
0109 #for preproduction samples:
0110 #RefRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance/preproduction'
0111 #NewRepository = '/afs/cern.ch/cms/performance/tracker/activities/reconstruction/tracking_performance/preproduction'
0112
0113 #Default Nevents
0114 defaultNevents ='-1'
0115
0116 #Put here the number of event to be processed for specific samples (numbers must be strings) if not specified is defaultNevents:
0117 Events={}
0118 #Events={'RelValTTbar':'100'}
0119
0120 # template file names. Usually should not be changed.
0121 cfg='trackingPerformanceValidation_cfg.py'
0122 macro='macro/TrackValHistoPublisher.C'
0123
0124
0125
0126 #########################################################################
0127 #########################################################################
0128 ############ Functions
0129
0130 def replace(map, filein, fileout):
0131     replace_items = map.items()
0132     while True:
0133         line = filein.readline()
0134         if not line: break
0135         for old, new in replace_items:
0136             line = string.replace(line, old, new)
0137         fileout.write(line)
0138     fileout.close()
0139     filein.close()
0140
0141 ############################################
0142
0143
0144 def do_validation(samples, GlobalTag, trackquality, trackalgorithm):
0145     global Sequence, RefSelection, RefRepository, NewSelection, NewRepository, defaultNevents, Events, castorHarvestedFilesDirectory
0146     global cfg, macro, Tracksname
0147     print('Tag: ' + GlobalTag)
0148     tracks_map = { 'ootb':'general_AssociatorByHitsRecoDenom','initialStep':'cutsRecoZero_AssociatorByHitsRecoDenom','lowPtTripletStep':'cutsRecoFirst_AssociatorByHitsRecoDenom','pixelPairStep':'cutsRecoSecond_AssociatorByHitsRecoDenom','detachedTripletStep':'cutsRecoThird_AssociatorByHitsRecoDenom','mixedTripletStep':'cutsRecoFourth_AssociatorByHitsRecoDenom','pixelLessStep':'cutsRecoFifth_AssociatorByHitsRecoDenom'}
0149     tracks_map_hp = { 'ootb':'cutsRecoHp_AssociatorByHitsRecoDenom','initialStep':'cutsRecoZeroHp_AssociatorByHitsRecoDenom','lowPtTripletStep':'cutsRecoFirstHp_AssociatorByHitsRecoDenom','pixelPairStep':'cutsRecoSecondHp_AssociatorByHitsRecoDenom','detachedTripletStep':'cutsRecoThirdHp_AssociatorByHitsRecoDenom','mixedTripletStep':'cutsRecoFourthHp_AssociatorByHitsRecoDenom','pixelLessStep':'cutsRecoFifthHp_AssociatorByHitsRecoDenom'}
0150     if(trackalgorithm=='initialStep' or trackalgorithm=='ootb'):
0151         mineff='0.80'
0152         maxeff='1.01'
0153         maxfake='0.7'
0154     else:
0155         mineff='0.0'
0156         maxeff='1.0'
0157         maxfake='0.8'
0158     #build the New Selection name
0159     NewSelection=GlobalTag + '_' + PileUp
0160     if (NewFastSim):
0161         NewSelection+='_FastSim'
0162     if( trackquality !=''):
0163         NewSelection+='_'+trackquality
0164     if(trackalgorithm!=''and not(trackalgorithm=='ootb' and trackquality !='')):
0165         NewSelection+='_'+trackalgorithm
0166     if(trackquality =='') and (trackalgorithm==''):
0167         if(Tracksname==''):
0168             NewSelection+='_ootb'
0169             Tracks='generalTracks'
0170         else:
0171            NewSelection+= Tracks
0172     if(Tracksname==''):
0173         Tracks='cutsRecoTracks'
0174     else:
0175         Tracks=Tracksname
0176     NewSelection+=NewSelectionLabel
0177     listofdatasets = open('listofdataset.txt' , 'w' )
0178     #loop on all the requested samples
0179     for sample in samples :
0180         templatecfgFile = open(cfg, 'r')
0181         templatemacroFile = open(macro, 'r')
0182 #        newdir=NewRepository+'/'+NewRelease+'/'+NewSelection+'/'+sample
0183         newdir=NewRelease+'/'+NewSelection+'/'+sample
0184     cfgFileName=sample+GlobalTag
0185         #check if the sample is already done
0186         if(os.path.isfile(newdir+'/building.pdf' )!=True):
0187
0188             if( Sequence=="harvesting"):
0189                 harvestedfile='./DQM_V0001_R000000001__' + GlobalTag+ '__' + sample + '__Validation.root'
0190             elif( Sequence=="preproduction"):
0191                 harvestedfile='./DQM_V0001_R000000001__' + sample+ '-' + GlobalTag + '_preproduction_312-v1__'+NewFormat+'_1.root'
0192             elif( Sequence=="comparison_only"):
0193                 harvestedfile='./DQM_V0001_R000000001__' + sample+ '__' + NewRelease+ '-' +GlobalTag + '-v1__'+NewFormat+'.root'
0194                 if(os.path.exists(harvestedfile)==False):
0195                     cpcmd='rfcp '+ castorHarvestedFilesDirectory+ NewRelease +'/' + harvestedfile + ' .'
0196                     returncode=os.system(cpcmd)
0197                     if (returncode!=0):
0198                         print('copy of harvested file from castor for sample ' + sample + ' failed')
0199                         print('try backup repository...')
0200                         NewCondition='MC'
0201                         if (NewFastSim):
0202                             NewCondition=NewCondition+'_FSIM'
0203                         cpcmd='rfcp /castor/cern.ch/user/a/aperrott/ValidationRecoMuon/'+NewRelease+'_'+NewCondition+'_'+sample+'_val.'+sample+'.root ./'+harvestedfile
0204                         returncode=os.system(cpcmd)
0205                         if (returncode!=0):
0206                             continue
0207
0208             print(' Harvested file : '+harvestedfile)
0209
0210             #search the primary dataset
0211             if( Sequence!="comparison_only"):
0212                 print('Get information from DBS for sample', sample)
0213                 cmd='dbsql "find  dataset where dataset like *'
0214                 #            cmd+=sample+'/'+NewRelease+'_'+GlobalTag+'*GEN-SIM-DIGI-RAW-HLTDEBUG-RECO* "'
0215                 cmd+=sample+'/'+NewRelease+'_'+GlobalTag+'*'+NewFormat+'* order by dataset.createdate "'
0216                 cmd+='|grep '+sample+'|grep -v test|tail -1'
0217                 print(cmd)
0218                 dataset= os.popen(cmd).readline().strip()
0219                 print('DataSet:  ', dataset, '\n')
0220
0221                 #Check if a dataset is found
0222                 if dataset!="":
0223                     listofdatasets.write(dataset)
0224                     #Find and format the list of files
0225                     cmd2='dbsql "find file where dataset like '+ dataset +'"|grep ' + sample
0226                     filenames='import FWCore.ParameterSet.Config as cms\n'
0227                     filenames+='readFiles = cms.untracked.vstring()\n'
0228                     filenames+='secFiles = cms.untracked.vstring()\n'
0229                     filenames+='source = cms.Source ("PoolSource",fileNames = readFiles, secondaryFileNames = secFiles)\n'
0230                     filenames+='readFiles.extend( [\n'
0231                     first=True
0232                     print(cmd2)
0233                     for line in os.popen(cmd2).readlines():
0234                         filename=line.strip()
0235                         if first==True:
0236                             filenames+="'"
0237                             filenames+=filename
0238                             filenames+="'"
0239                             first=False
0240                         else :
0241                             filenames+=",\n'"
0242                             filenames+=filename
0243                             filenames+="'"
0244                     filenames+=']);\n'
0245
0246                     # if not harvesting find secondary file names
0247                     if(Sequence!="preproduction"):
0248                             cmd3='dbsql  "find dataset.parent where dataset like '+ dataset +'"|grep ' + sample
0249                             parentdataset=os.popen(cmd3).readline()
0250                             print('Parent DataSet:  ', parentdataset, '\n')
0251
0252                     #Check if a dataset is found
0253                             if parentdataset!="":
0254                                     cmd4='dbsql  "find file where dataset like '+ parentdataset +'"|grep ' + sample
0255                                     filenames+='secFiles.extend( [\n'
0256                                     first=True
0257
0258                                     for line in os.popen(cmd4).readlines():
0259                                         secfilename=line.strip()
0260                                         if first==True:
0261                                             filenames+="'"
0262                                             filenames+=secfilename
0263                                             filenames+="'"
0264                                             first=False
0265                                         else :
0266                                             filenames+=",\n'"
0267                                             filenames+=secfilename
0268                                             filenames+="'"
0269                                     filenames+='\n ]);\n'
0270                             else :
0271                                     print("No primary dataset found skipping sample: ", sample)
0272                                     continue
0273                     else :
0274                             filenames+='secFiles.extend( (               ) )'
0275
0276                     cfgFile = open(cfgFileName+'.py' , 'w' )
0277                     cfgFile.write(filenames)
0278
0279                     if ((sample in Events)!=True):
0280                             Nevents=defaultNevents
0281                     else:
0282                             Nevents=Events[sample]
0283                     thealgo=trackalgorithm
0284                     thequality=trackquality
0285                     if(trackalgorithm=='ootb'):
0286                         thealgo=''
0287                     if(thealgo!=''):
0288                         thealgo='\''+thealgo+'\''
0289                     if(trackquality!=''):
0290                         thequality='\''+trackquality+'\''
0291                     symbol_map = { 'NEVENT':Nevents, 'GLOBALTAG':GlobalTag, 'SEQUENCE':Sequence, 'SAMPLE': sample, 'ALGORITHM':thealgo, 'QUALITY':thequality, 'TRACKS':Tracks}
0292
0293
0294                     cfgFile = open(cfgFileName+'.py' , 'a' )
0295                     replace(symbol_map, templatecfgFile, cfgFile)
0296                     if(( (Sequence=="harvesting" or Sequence=="preproduction" or Sequence=="comparison_only") and os.path.isfile(harvestedfile) )==False):
0297                         # if the file is already harvested do not run the job again
0298                         cmdrun='cmsRun ' +cfgFileName+ '.py >&  ' + cfgFileName + '.log < /dev/zero '
0299                         retcode=os.system(cmdrun)
0300                     else:
0301                         retcode=0
0302
0303                 else:
0304                     print('No dataset found skipping sample: '+ sample, '\n')
0305                     continue
0306             else:
0307                 print(' Used sequence : '+Sequence)
0308                 retcode = 0
0309
0310             if (retcode!=0):
0311                 print('Job for sample '+ sample + ' failed. \n')
0312             else:
0313                 if (Sequence=="harvesting" or Sequence=="preproduction" or Sequence=="comparison_only"):
0314                     #copy only the needed histograms
0315                     if(trackquality==""):
0316                         rootcommand='root -b -q -l CopySubdir.C\\('+ '\\\"'+harvestedfile+'\\\",\\\"val.' +sample+'.root\\\",\\\"'+ tracks_map[trackalgorithm]+ '\\\"\\) >& /dev/null'
0317                         os.system(rootcommand)
0318                     elif(trackquality=="highPurity"):
0319                         rootcommand='root -b -q -l CopySubdir.C\\('+ '\\\"'+harvestedfile+'\\\",\\\"val.' +sample+'.root\\\",\\\"'+ tracks_map_hp[trackalgorithm]+ '\\\"\\) >& /dev/null'
0320                         os.system(rootcommand)
0321
0322
0323                 referenceSample=RefRepository+'/'+RefRelease+'/'+RefSelection+'/'+sample+'/'+'val.'+sample+'.root'
0324                 if os.path.isfile(referenceSample ):
0325                     replace_map = { 'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':RefRelease+'/'+RefSelection+'/val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL': sample, 'REF_RELEASE':RefRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':RefSelection, 'NEWSELECTION':NewSelection, 'TrackValHistoPublisher': cfgFileName, 'MINEFF':mineff, 'MAXEFF':maxeff, 'MAXFAKE':maxfake}
0326
0327                     if(os.path.exists(RefRelease+'/'+RefSelection)==False):
0328                         os.makedirs(RefRelease+'/'+RefSelection)
0329                         os.system('cp ' + referenceSample+ ' '+RefRelease+'/'+RefSelection)
0330                 else:
0331 #                    print "No reference file found at: ", RefRelease+'/'+RefSelection
0332                     print("No reference file found at: ", referenceSample)
0333                     replace_map = { 'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':'val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL': sample, 'REF_RELEASE':NewRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':NewSelection, 'NEWSELECTION':NewSelection, 'TrackValHistoPublisher': cfgFileName, 'MINEFF':mineff, 'MAXEFF':maxeff, 'MAXFAKE':maxfake}
0334
0335
0336                 macroFile = open(cfgFileName+'.C' , 'w' )
0337                 replace(replace_map, templatemacroFile, macroFile)
0338
0339
0340                 os.system('root -b -q -l '+ cfgFileName+'.C'+ '>  macro.'+cfgFileName+'.log')
0341
0342
0343                 if(os.path.exists(newdir)==False):
0344                     os.makedirs(newdir)
0345
0346                 print("moving pdf files for sample: " , sample)
0347                 os.system('mv  *.pdf ' + newdir)
0348
0349                 print("moving root file for sample: " , sample)
0350                 os.system('mv val.'+ sample+ '.root ' + newdir)
0351
0352                 print("copy py file for sample: " , sample)
0353                 os.system('cp '+cfgFileName+'.py ' + newdir)
0354
0355
0356         else:
0357             print('Validation for sample ' + sample + ' already done. Skipping this sample. \n')
0358
0359
0360
0361
0362 ##########################################################################
0363 #########################################################################
0364 ######## This is the main program
0365 if(NewRelease==''):
0366     try:
0367         #Get some environment variables to use
0368         NewRelease     = os.environ["CMSSW_VERSION"]
0369
0370     except KeyError:
0371         print('Error: The environment variable CMSSW_VERSION is not available.', file=sys.stderr)
0372         print('       Please run cmsenv', file=sys.stderr)
0373         sys.exit()
0374 else:
0375     try:
0376         #Get some environment variables to use
0377         os.environ["CMSSW_VERSION"]
0378
0379     except KeyError:
0380         print('Error: CMSSW environment variables are not available.', file=sys.stderr)
0381         print('       Please run cmsenv', file=sys.stderr)
0382         sys.exit()
0383
0384
0385
0386 NewSelection=''
0387
0388 for algo in Algos:
0389     for quality in Qualities:
0390         RefSelection=ReferenceSelection
0391         if( quality !=''):
0392             RefSelection+='_'+quality
0393         if(algo!=''and not(algo=='ootb' and quality !='')):
0394             RefSelection+='_'+algo
0395         if(quality =='') and (algo==''):
0396             RefSelection+='_ootb'
0397         do_validation(idealsamples, IdealTag, quality , algo)
0398         RefSelection=StartupReferenceSelection
0399         if( quality !=''):
0400             RefSelection+='_'+quality
0401         if(algo!=''and not(algo=='ootb' and quality !='')):
0402             RefSelection+='_'+algo
0403         if(quality =='') and (algo==''):
0404             RefSelection+='_ootb'
0405         do_validation(startupsamples, StartupTag, quality , algo)
0406