RecoVertex/test/V0PerformanceValidation.py

0001 #! /usr/bin/env python3
0002
0003 import os
0004 import sys
0005 import fileinput
0006 import string
0007
0008 # Reference release
0009 RefRelease='CMSSW_3_6_0_pre1'
0010
0011 # RelVal release (only set if different from $CMSSW_VERSION)
0012 NewRelease='CMSSW_3_6_0_pre1'
0013
0014 # Startup and ideal samples list
0015 #startupsamples = ['RelValTTbar', 'RelValMinBias', 'RelValQCD_Pt_3000_3500', 'RelValQCD_Pt_80_120']
0016
0017 # This is a pileup sample
0018 #startupsamples = ['RelValTTbar_Tauola']
0019 # This is the full set of STARTUP samples, to be used for timing studies
0020 #startupsamples = ['RelValQCD_Pt_3000_3500', 'RelValTTbar', 'RelValMinBias']
0021 #startupsamples = ['RelValQCD_Pt_3000_3500', 'RelValMinBias']
0022 #startupsamples = ['RelValTTbar']
0023 startupsamples = []
0024
0025 #$startupsamples = ['RelValQCD_Pt_80_120']
0026
0027 # MC relvals
0028 #idealsamples = ['RelValSingleMuPt1', 'RelValSingleMuPt10', 'RelValSingleMuPt100', RelValSinglePiPt1']
0029
0030 # Pileup MC sample
0031 #idealsamples = ['RelValZmumuJets_Pt_20_300_GEN']
0032 idealsamples = []
0033
0034 # New samples array for 3_X_X
0035 mcsamples = ['RelValTTbar']
0036 #mcsamples = []
0037
0038 Sequence = 'only_validation'
0039 #Sequence = 'harvesting'
0040
0041 # GlobalTags
0042 IdealTag = 'DESIGN_3X_V21'
0043 #StartupTag = 'START3X_V21'
0044 StartupTag = 'STARTUP3X_V8H'
0045 # This tag is the new replacement (as of 31X) for MC, IDEAL becomes DESIGN
0046 # MC_31X_V9 works for CMSSW_3_4_0_preX
0047 MCTag = 'MC_3XY_V21'
0048 #MCTag = 'MC_3XY_V12'
0049
0050 # PileUp: PU, No PileUp: noPU
0051 PileUp = 'noPU'
0052
0053 # Reference directory name
0054 ReferenceSelection = 'DESIGN_3X_V21_'+PileUp
0055 MCReferenceSelection = 'MC_3XY_V21_'+PileUp
0056 StartupReferenceSelection = 'STARTUP3X_V8H_'+PileUp
0057
0058 # This is where the reference samples are stored
0059 #RefRepository = '/afs/cern.ch/cms/performance/'
0060 RefRepository = '/nfs/data35/cms/drell/val360pre1'
0061 # NewRepository contains the files for the new release to be tested
0062 NewRepository = '/nfs/data35/cms/drell/val360pre1'
0063
0064 # Default number of events
0065 defaultNevents = '-1'
0066
0067 # Specify the number of events to be processed for specific samples (numbers must be strings)
0068 Events = {}
0069
0070 # template file names, shouldn't need to be changed
0071 cfg = 'v0PerformanceValidation_cfg.py'
0072 macro = 'macros/V0ValHistoPublisher.C'
0073
0074 #This config is only for testing V0Producer changes.
0075 #cfg = 'v0PerformanceValidation_v0prodtest_cfg.py'
0076 #cfg = 'v0PerformanceValidation_origV0Prod_cfg.py'
0077
0078 ###################################
0079 ### End configurable parameters ###
0080 ###################################
0081
0082 #----------------------------------------------------------------------------
0083
0084 #################
0085 ### Functions ###
0086 #################
0087
0088 # Does replacement of strings in files
0089 def replace(map, filein, fileout):
0090     replace_items = map.items()
0091     while True:
0092         line = filein.readline()
0093         if not line: break
0094         for old, new in replace_items:
0095             line = string.replace(line, old, new)
0096         fileout.write(line)
0097     fileout.close()
0098     filein.close()
0099
0100
0101 # This function does most of the work
0102 def do_validation(samples, GlobalTag):
0103     global Sequence, RefSelection, RefRepository, NewSelection, NewRepository, defaultNevents, Events
0104     global cfg, macro, Tracksname
0105     #print 'Tag: ' + GlobalTag
0106     NewSelection = GlobalTag + '_' + PileUp
0107
0108     for sample in samples:
0109         templateCfgFile = open(cfg, 'r')
0110         templateMacroFile = open(macro, 'r')
0111         newdir = NewRepository + '/' + NewRelease + '/' + NewSelection + '/' + sample
0112         cfgFileName = sample + GlobalTag
0113
0114         if(os.path.isfile(newdir + '/K0sEff.pdf') != True):
0115             # If the job is harvesting, check if the file is already harvested
0116             harvestedfile = './DQM_V0001_R000000001__' + GlobalTag + '__' + sample + '__Validation.root'
0117             if(( Sequence == "harvesting" and os.path.isfile(harvestedfile)) == False):
0118 #                cmd = 'dbsql "find dataset.createdate, dataset where dataset like *'
0119                 cmd = 'dbsql "find dataset where dataset like *'
0120                 cmd += sample + '/' + NewRelease + '_' + GlobalTag + '*GEN-SIM-RECO*" '
0121                 cmd += '| grep ' + sample + ' | grep -v test | sort | tail -1 | cut -f2 '
0122                 print(cmd)
0123                 print('Finding data set...\n')
0124                 dataset = os.popen(cmd).readline().strip()
0125                 print('DataSet: ', dataset, '\n')
0126                 if dataset != "":
0127                     cmd2 = 'dbsql "find file where dataset like ' + dataset + '" | grep ' + sample
0128                     filenames = 'import FWCore.ParameterSet.Config as cms\n'
0129                     filenames += 'readFiles = cms.untracked.vstring()\n'
0130                     filenames += 'secFiles = cms.untracked.vstring()\n'
0131                     filenames += 'source = cms.Source("PoolSource", fileNames = readFiles, secondaryFileNames = secFiles)\n'
0132                     filenames += 'readFiles.extend([\n'
0133                     first = True
0134                     #print cmd2
0135                     print('Getting file names for dataset ', dataset, '\n')
0136                     for line in os.popen(cmd2).readlines():
0137                         filename = line.strip()
0138                         if first == True:
0139                             filenames += "    '"
0140                             filenames += filename
0141                             filenames += "'"
0142                             first = False
0143                         else:
0144                             filenames += ",\n    '"
0145                             filenames += filename
0146                             filenames += "'"
0147                     filenames += ']);\n'
0148                     # if not harvesting, find secondary file names
0149                     if(Sequence != 'harvesting'):
0150                         cmd3 = 'dbsql "find dataset.parent where dataset like ' + dataset + '" | grep ' + sample
0151             #print cmd3
0152                         print('Getting parent dataset...\n')
0153                         parentdataset = os.popen(cmd3).readline()
0154                         print('Parent dataset: ', parentdataset, '\n')
0155                         if parentdataset != "":
0156                             cmd4 = 'dbsql "find file where dataset like ' + parentdataset + '" | grep ' + sample
0157                             #print 'cmd4', cmd4
0158                             filenames += 'secFiles.extend([\n'
0159                             first = True
0160                             for line in os.popen(cmd4).readlines():
0161                                 secfilename = line.strip()
0162                                 if first == True:
0163                                     filenames += "    '"
0164                                     filenames += secfilename
0165                                     filenames += "'"
0166                                     first = False
0167                                 else:
0168                                     filenames += ",\n    '"
0169                                     filenames += secfilename
0170                                     filenames += "'"
0171                             filenames += ']);\n\n'
0172                         else:
0173                             print("No primary dataset found, skipping sample ", sample)
0174                             continue
0175                     else:
0176                         filenames += 'secFiles.extend((     ))'
0177
0178                     cfgFile = open(cfgFileName + '.py', 'w')
0179                     cfgFile.write(filenames)
0180
0181                     if(sample in Events):
0182                         Nevents = Events[sample]
0183                     else:
0184                         Nevents = defaultNevents
0185
0186                     symbol_map = {'NEVENT':Nevents, 'GLOBALTAG':GlobalTag, 'SEQUENCE':Sequence, 'SAMPLE':sample}
0187                     cfgfile = open(cfgFileName + '.py', 'a')
0188                     replace(symbol_map, templateCfgFile, cfgFile)
0189
0190                     cmdrun = 'cmsRun ' + cfgFileName + '.py >& ' + cfgFileName + '.log < /dev/zero'
0191                     print('Running validation on sample ' + sample + '...\n')
0192                     #print cmdrun
0193                     retcode = os.system(cmdrun)
0194                 else:
0195                     print('No dataset found, skipping sample ', sample, '\n')
0196                     continue
0197             else:
0198                 retcode = 0
0199             if(retcode):
0200                 print('Job for sample ' + sample + ' failed.\n')
0201                 print('Check log file ' + sample + '.log for details.\n')
0202             else:
0203                 if Sequence == 'harvesting':
0204                     rootcommand = 'root -b -q -l CopySubdir.C\\('+'\\\"DQM_V0001_R000000001__'+GlobalTag+'__'+sample+'__Validation.root\\\", \\\"val.'+sample+'.root\\\"\\) >& /dev/null'
0205                     os.system(rootcommand)
0206                 referenceSample = RefRepository + '/' + RefRelease + '/' + RefSelection + '/' + sample + '/' + 'val.'+sample+'.root'
0207                 print('Reference sample: ' + referenceSample)
0208
0209                 if os.path.isfile(referenceSample):
0210                     print('Found reference.')
0211                     replace_map = {'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':RefRelease+'/'+RefSelection+'/'+'val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL':sample, 'REF_RELEASE':RefRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':RefSelection, 'NEWSELECTION':NewSelection, 'V0ValHistoPublisher':cfgFileName }
0212
0213                     if not os.path.exists(RefRelease + '/' + RefSelection):
0214                         os.makedirs(RefRelease + '/' + RefSelection)
0215                         os.system('cp ' + referenceSample + ' ' + RefRelease+'/'+RefSelection)
0216
0217                 else:
0218                     print("No reference file found at ", RefRepository+'/'+RefRelease+'/'+RefSelection)
0219                     replace_map = {'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':'val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL':sample, 'REF_RELEASE':NewRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':NewSelection, 'NEWSELECTION':NewSelection, 'V0ValHistoPublisher':cfgFileName }
0220
0221                 macroFile = open(cfgFileName+'.C', 'w')
0222                 replace(replace_map, templateMacroFile, macroFile)
0223
0224                 os.system('root -b -q -l ' + cfgFileName + '.C' + ' > macro.' + cfgFileName + '.log')
0225
0226                 if not os.path.exists(newdir):
0227                     os.makedirs(newdir)
0228
0229                 print("Moving PDF and PNG files for sample: ", sample)
0230                 os.system('mv *pdf *png ' + newdir)
0231
0232                 print("Moving ROOT file for sample: ", sample)
0233                 os.system('mv val.'+sample+'.root ' + newdir)
0234
0235                 print("Copying .py file for sample: ", sample)
0236                 os.system('cp '+cfgFileName+'.py ' + newdir)
0237         else:
0238             print('Validation for sample '+sample+' already done, skipping.\n')
0239
0240 ##############################################################################
0241 ### Main part of script, this runs do_validation for all specified samples ###
0242 ##############################################################################
0243 if(NewRelease == ''):
0244     try:
0245         NewRelease = os.environ["CMSSW_VERSION"]
0246     except KeyError:
0247         print('Error: The environment variable CMSSW_VERSION is not set.', file=sys.stderr)
0248         print('       Please run cmsenv in the appropriate release area.', file=sys.stderr)
0249 else:
0250     try:
0251         os.environ["CMSSW_VERSION"]
0252     except KeyError:
0253         print('Error: CMSSW environment variables are not set.', file=sys.stderr)
0254         print('       Please run cmsenv in the appropriate release area.', file=sys.stderr)
0255
0256 NewSelection = ''
0257 RefSelection = MCReferenceSelection
0258 do_validation(mcsamples, MCTag)
0259
0260 NewSelection = ''
0261 RefSelection = StartupReferenceSelection
0262 do_validation(startupsamples, StartupTag)
0263
0264 NewSelection = ''
0265 RefSelection = ReferenceSelection
0266 do_validation(idealsamples, IdealTag)
0267
0268 print('Finished running validation.')