RecoVertex/test/V0PerformanceValidation.py

0001 #! /usr/bin/env python3
0002
0003 from __future__ import print_function
0004 import os
0005 import sys
0006 import fileinput
0007 import string
0008
0009 # Reference release
0010 RefRelease='CMSSW_3_6_0_pre1'
0011
0012 # RelVal release (only set if different from $CMSSW_VERSION)
0013 NewRelease='CMSSW_3_6_0_pre1'
0014
0015 # Startup and ideal samples list
0016 #startupsamples = ['RelValTTbar', 'RelValMinBias', 'RelValQCD_Pt_3000_3500', 'RelValQCD_Pt_80_120']
0017
0018 # This is a pileup sample
0019 #startupsamples = ['RelValTTbar_Tauola']
0020 # This is the full set of STARTUP samples, to be used for timing studies
0021 #startupsamples = ['RelValQCD_Pt_3000_3500', 'RelValTTbar', 'RelValMinBias']
0022 #startupsamples = ['RelValQCD_Pt_3000_3500', 'RelValMinBias']
0023 #startupsamples = ['RelValTTbar']
0024 startupsamples = []
0025
0026 #$startupsamples = ['RelValQCD_Pt_80_120']
0027
0028 # MC relvals
0029 #idealsamples = ['RelValSingleMuPt1', 'RelValSingleMuPt10', 'RelValSingleMuPt100', RelValSinglePiPt1']
0030
0031 # Pileup MC sample
0032 #idealsamples = ['RelValZmumuJets_Pt_20_300_GEN']
0033 idealsamples = []
0034
0035 # New samples array for 3_X_X
0036 mcsamples = ['RelValTTbar']
0037 #mcsamples = []
0038
0039 Sequence = 'only_validation'
0040 #Sequence = 'harvesting'
0041
0042 # GlobalTags
0043 IdealTag = 'DESIGN_3X_V21'
0044 #StartupTag = 'START3X_V21'
0045 StartupTag = 'STARTUP3X_V8H'
0046 # This tag is the new replacement (as of 31X) for MC, IDEAL becomes DESIGN
0047 # MC_31X_V9 works for CMSSW_3_4_0_preX
0048 MCTag = 'MC_3XY_V21'
0049 #MCTag = 'MC_3XY_V12'
0050
0051 # PileUp: PU, No PileUp: noPU
0052 PileUp = 'noPU'
0053
0054 # Reference directory name
0055 ReferenceSelection = 'DESIGN_3X_V21_'+PileUp
0056 MCReferenceSelection = 'MC_3XY_V21_'+PileUp
0057 StartupReferenceSelection = 'STARTUP3X_V8H_'+PileUp
0058
0059 # This is where the reference samples are stored
0060 #RefRepository = '/afs/cern.ch/cms/performance/'
0061 RefRepository = '/nfs/data35/cms/drell/val360pre1'
0062 # NewRepository contains the files for the new release to be tested
0063 NewRepository = '/nfs/data35/cms/drell/val360pre1'
0064
0065 # Default number of events
0066 defaultNevents = '-1'
0067
0068 # Specify the number of events to be processed for specific samples (numbers must be strings)
0069 Events = {}
0070
0071 # template file names, shouldn't need to be changed
0072 cfg = 'v0PerformanceValidation_cfg.py'
0073 macro = 'macros/V0ValHistoPublisher.C'
0074
0075 #This config is only for testing V0Producer changes.
0076 #cfg = 'v0PerformanceValidation_v0prodtest_cfg.py'
0077 #cfg = 'v0PerformanceValidation_origV0Prod_cfg.py'
0078
0079 ###################################
0080 ### End configurable parameters ###
0081 ###################################
0082
0083 #----------------------------------------------------------------------------
0084
0085 #################
0086 ### Functions ###
0087 #################
0088
0089 # Does replacement of strings in files
0090 def replace(map, filein, fileout):
0091     replace_items = map.items()
0092     while True:
0093         line = filein.readline()
0094         if not line: break
0095         for old, new in replace_items:
0096             line = string.replace(line, old, new)
0097         fileout.write(line)
0098     fileout.close()
0099     filein.close()
0100
0101
0102 # This function does most of the work
0103 def do_validation(samples, GlobalTag):
0104     global Sequence, RefSelection, RefRepository, NewSelection, NewRepository, defaultNevents, Events
0105     global cfg, macro, Tracksname
0106     #print 'Tag: ' + GlobalTag
0107     NewSelection = GlobalTag + '_' + PileUp
0108
0109     for sample in samples:
0110         templateCfgFile = open(cfg, 'r')
0111         templateMacroFile = open(macro, 'r')
0112         newdir = NewRepository + '/' + NewRelease + '/' + NewSelection + '/' + sample
0113         cfgFileName = sample + GlobalTag
0114
0115         if(os.path.isfile(newdir + '/K0sEff.pdf') != True):
0116             # If the job is harvesting, check if the file is already harvested
0117             harvestedfile = './DQM_V0001_R000000001__' + GlobalTag + '__' + sample + '__Validation.root'
0118             if(( Sequence == "harvesting" and os.path.isfile(harvestedfile)) == False):
0119 #                cmd = 'dbsql "find dataset.createdate, dataset where dataset like *'
0120         cmd = 'dbsql "find dataset where dataset like *'
0121                 cmd += sample + '/' + NewRelease + '_' + GlobalTag + '*GEN-SIM-RECO*" '
0122                 cmd += '| grep ' + sample + ' | grep -v test | sort | tail -1 | cut -f2 '
0123                 print(cmd)
0124                 print('Finding data set...\n')
0125                 dataset = os.popen(cmd).readline().strip()
0126                 print('DataSet: ', dataset, '\n')
0127                 if dataset != "":
0128                     cmd2 = 'dbsql "find file where dataset like ' + dataset + '" | grep ' + sample
0129                     filenames = 'import FWCore.ParameterSet.Config as cms\n'
0130                     filenames += 'readFiles = cms.untracked.vstring()\n'
0131                     filenames += 'secFiles = cms.untracked.vstring()\n'
0132                     filenames += 'source = cms.Source("PoolSource", fileNames = readFiles, secondaryFileNames = secFiles)\n'
0133                     filenames += 'readFiles.extend([\n'
0134                     first = True
0135                     #print cmd2
0136                     print('Getting file names for dataset ', dataset, '\n')
0137                     for line in os.popen(cmd2).readlines():
0138                         filename = line.strip()
0139                         if first == True:
0140                             filenames += "    '"
0141                             filenames += filename
0142                             filenames += "'"
0143                             first = False
0144                         else:
0145                             filenames += ",\n    '"
0146                             filenames += filename
0147                             filenames += "'"
0148                     filenames += ']);\n'
0149                     # if not harvesting, find secondary file names
0150                     if(Sequence != 'harvesting'):
0151                         cmd3 = 'dbsql "find dataset.parent where dataset like ' + dataset + '" | grep ' + sample
0152             #print cmd3
0153                         print('Getting parent dataset...\n')
0154                         parentdataset = os.popen(cmd3).readline()
0155                         print('Parent dataset: ', parentdataset, '\n')
0156                         if parentdataset != "":
0157                             cmd4 = 'dbsql "find file where dataset like ' + parentdataset + '" | grep ' + sample
0158                             #print 'cmd4', cmd4
0159                             filenames += 'secFiles.extend([\n'
0160                             first = True
0161                             for line in os.popen(cmd4).readlines():
0162                                 secfilename = line.strip()
0163                                 if first == True:
0164                                     filenames += "    '"
0165                                     filenames += secfilename
0166                                     filenames += "'"
0167                                     first = False
0168                                 else:
0169                                     filenames += ",\n    '"
0170                                     filenames += secfilename
0171                                     filenames += "'"
0172                             filenames += ']);\n\n'
0173                         else:
0174                             print("No primary dataset found, skipping sample ", sample)
0175                             continue
0176                     else:
0177                         filenames += 'secFiles.extend((     ))'
0178
0179                     cfgFile = open(cfgFileName + '.py', 'w')
0180                     cfgFile.write(filenames)
0181
0182                     if(sample in Events):
0183                         Nevents = Events[sample]
0184                     else:
0185                         Nevents = defaultNevents
0186
0187                     symbol_map = {'NEVENT':Nevents, 'GLOBALTAG':GlobalTag, 'SEQUENCE':Sequence, 'SAMPLE':sample}
0188                     cfgfile = open(cfgFileName + '.py', 'a')
0189                     replace(symbol_map, templateCfgFile, cfgFile)
0190
0191                     cmdrun = 'cmsRun ' + cfgFileName + '.py >& ' + cfgFileName + '.log < /dev/zero'
0192                     print('Running validation on sample ' + sample + '...\n')
0193                     #print cmdrun
0194                     retcode = os.system(cmdrun)
0195                 else:
0196                     print('No dataset found, skipping sample ', sample, '\n')
0197                     continue
0198             else:
0199                 retcode = 0
0200             if(retcode):
0201                 print('Job for sample ' + sample + ' failed.\n')
0202                 print('Check log file ' + sample + '.log for details.\n')
0203             else:
0204                 if Sequence == 'harvesting':
0205                     rootcommand = 'root -b -q -l CopySubdir.C\\('+'\\\"DQM_V0001_R000000001__'+GlobalTag+'__'+sample+'__Validation.root\\\", \\\"val.'+sample+'.root\\\"\\) >& /dev/null'
0206                     os.system(rootcommand)
0207                 referenceSample = RefRepository + '/' + RefRelease + '/' + RefSelection + '/' + sample + '/' + 'val.'+sample+'.root'
0208                 print('Reference sample: ' + referenceSample)
0209
0210                 if os.path.isfile(referenceSample):
0211                     print('Found reference.')
0212                     replace_map = {'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':RefRelease+'/'+RefSelection+'/'+'val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL':sample, 'REF_RELEASE':RefRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':RefSelection, 'NEWSELECTION':NewSelection, 'V0ValHistoPublisher':cfgFileName }
0213
0214                     if not os.path.exists(RefRelease + '/' + RefSelection):
0215                         os.makedirs(RefRelease + '/' + RefSelection)
0216                         os.system('cp ' + referenceSample + ' ' + RefRelease+'/'+RefSelection)
0217
0218                 else:
0219                     print("No reference file found at ", RefRepository+'/'+RefRelease+'/'+RefSelection)
0220                     replace_map = {'NEW_FILE':'val.'+sample+'.root', 'REF_FILE':'val.'+sample+'.root', 'REF_LABEL':sample, 'NEW_LABEL':sample, 'REF_RELEASE':NewRelease, 'NEW_RELEASE':NewRelease, 'REFSELECTION':NewSelection, 'NEWSELECTION':NewSelection, 'V0ValHistoPublisher':cfgFileName }
0221
0222                 macroFile = open(cfgFileName+'.C', 'w')
0223                 replace(replace_map, templateMacroFile, macroFile)
0224
0225                 os.system('root -b -q -l ' + cfgFileName + '.C' + ' > macro.' + cfgFileName + '.log')
0226
0227                 if not os.path.exists(newdir):
0228                     os.makedirs(newdir)
0229
0230                 print("Moving PDF and PNG files for sample: ", sample)
0231                 os.system('mv *pdf *png ' + newdir)
0232
0233                 print("Moving ROOT file for sample: ", sample)
0234                 os.system('mv val.'+sample+'.root ' + newdir)
0235
0236                 print("Copying .py file for sample: ", sample)
0237                 os.system('cp '+cfgFileName+'.py ' + newdir)
0238         else:
0239             print('Validation for sample '+sample+' already done, skipping.\n')
0240
0241 ##############################################################################
0242 ### Main part of script, this runs do_validation for all specified samples ###
0243 ##############################################################################
0244 if(NewRelease == ''):
0245     try:
0246         NewRelease = os.environ["CMSSW_VERSION"]
0247     except KeyError:
0248         print('Error: The environment variable CMSSW_VERSION is not set.', file=sys.stderr)
0249         print('       Please run cmsenv in the appropriate release area.', file=sys.stderr)
0250 else:
0251     try:
0252         os.environ["CMSSW_VERSION"]
0253     except KeyError:
0254         print('Error: CMSSW environment variables are not set.', file=sys.stderr)
0255         print('       Please run cmsenv in the appropriate release area.', file=sys.stderr)
0256
0257 NewSelection = ''
0258 RefSelection = MCReferenceSelection
0259 do_validation(mcsamples, MCTag)
0260
0261 NewSelection = ''
0262 RefSelection = StartupReferenceSelection
0263 do_validation(startupsamples, StartupTag)
0264
0265 NewSelection = ''
0266 RefSelection = ReferenceSelection
0267 do_validation(idealsamples, IdealTag)
0268
0269 print('Finished running validation.')