File indexing completed on 2023-08-11 23:31:14
0001
0002
0003 from __future__ import absolute_import
0004 import sys
0005 import re
0006 import os
0007 from .pipe import pipe as _pipe
0008 from .options import globalTag
0009 from itertools import islice
0010
0011 def splitter(iterator, n):
0012 i = iterator.__iter__()
0013 while True:
0014 l = list(islice(i, n))
0015 if l:
0016 yield l
0017 else:
0018 break
0019
0020
0021 class HLTProcess(object):
0022
0023 def __init__(self, configuration):
0024 self.config = configuration
0025 self.data = None
0026 self.source = []
0027 self.parent = []
0028
0029 self.options = {
0030 'essources' : [],
0031 'esmodules' : [],
0032 'modules' : [],
0033 'sequences' : [],
0034 'services' : [],
0035 'paths' : [],
0036 'psets' : [],
0037 'blocks' : [],
0038 }
0039
0040 self.labels = {}
0041 if self.config.fragment:
0042 self.labels['process'] = 'fragment'
0043 self.labels['dict'] = 'fragment.__dict__'
0044 else:
0045 self.labels['process'] = 'process'
0046 self.labels['dict'] = 'process.__dict__'
0047
0048 if self.config.prescale and (self.config.prescale.lower() != 'none'):
0049 self.labels['prescale'] = self.config.prescale
0050
0051
0052 from .confdbOfflineConverter import OfflineConverter
0053 self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port, tunnel = self.config.tunnel, tunnelPort = self.config.tunnel_port)
0054 self.buildPathList()
0055 self.buildOptions()
0056 self.getSetupConfigurationFromDB()
0057 self.getRawConfigurationFromDB()
0058 self.customize()
0059
0060 def getSetupConfigurationFromDB(self):
0061 if not self.config.setup:
0062 return
0063
0064 if ".py" in self.config.setup:
0065 self.config.setupFile = self.config.setup.split(".py")[0]
0066 return
0067 args = ['--configName', self.config.setup ]
0068 args.append('--noedsources')
0069 args.append('--nopaths')
0070 for key, vals in self.options.items():
0071 if vals:
0072 args.extend(('--'+key, ','.join(vals)))
0073 args.append('--cff')
0074 data, err = self.converter.query( *args )
0075 if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0076 sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
0077 sys.stderr.write(err + "\n\n")
0078 sys.exit(1)
0079 self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
0080 outfile = open(self.config.setupFile+".py","w+")
0081 outfile.write("# This file is automatically generated by hltGetConfiguration.\n" + data)
0082
0083 def getRawConfigurationFromDB(self):
0084 if self.config.menu.run:
0085 args = ['--runNumber', self.config.menu.run]
0086 else:
0087 args = ['--configName', self.config.menu.name ]
0088 if not self.config.hilton:
0089
0090 args.append('--noedsources')
0091 for key, vals in self.options.items():
0092 if vals:
0093 args.extend(('--'+key, ','.join(vals)))
0094 data, err = self.converter.query( *args )
0095 if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0096 sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0097 sys.stderr.write(err + "\n\n")
0098 sys.exit(1)
0099 self.data = data
0100
0101 def getPathList(self):
0102 if self.config.menu.run:
0103 args = ['--runNumber', self.config.menu.run]
0104 else:
0105 args = ['--configName', self.config.menu.name]
0106 args.extend( (
0107 '--cff',
0108 '--noedsources',
0109 '--noes',
0110 '--noservices',
0111 '--nosequences',
0112 '--nomodules'
0113 ) )
0114
0115 data, err = self.converter.query( *args )
0116 if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0117 sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0118 sys.stderr.write(err + "\n\n")
0119 sys.exit(1)
0120 filter = re.compile(r' *= *cms.(End|Final)?Path.*')
0121 paths = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
0122 return paths
0123
0124
0125 @staticmethod
0126 def expandWildcards(globs, collection):
0127
0128
0129 matches = []
0130 for glob in globs:
0131 negate = ''
0132 if glob[0] == '-':
0133 negate = '-'
0134 glob = glob[1:]
0135
0136 filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
0137 matches.extend( negate + element for element in collection if filter.match(element) )
0138 return matches
0139
0140
0141 @staticmethod
0142 def consolidateNegativeList(elements):
0143
0144
0145 result = set()
0146 for element in elements:
0147 if element[0] == '-':
0148 result.add( element )
0149 else:
0150 result.discard( '-' + element )
0151 return sorted( element for element in result )
0152
0153 @staticmethod
0154 def consolidatePositiveList(elements):
0155
0156
0157 result = set()
0158 for element in elements:
0159 if element[0] == '-':
0160 result.discard( element[1:] )
0161 else:
0162 result.add( element )
0163 return sorted( element for element in result )
0164
0165
0166
0167 def dump(self):
0168 self.data = self.data % self.labels
0169 if self.config.fragment:
0170 self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
0171 self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
0172 return self.data
0173
0174
0175
0176 def specificCustomize(self):
0177
0178 if self.config.fragment:
0179 self.data += """
0180 # add specific customizations
0181 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0182 fragment = customizeHLTforAll(fragment,"%s")
0183 """ % (self.config.type)
0184 elif self.config.hilton:
0185
0186 pass
0187 else:
0188 if self.config.type=="Fake":
0189 prefix = "run1"
0190 elif self.config.type in ("Fake1","Fake2","2018"):
0191 prefix = "run2"
0192 else:
0193 prefix = "run3"
0194 _gtData = "auto:"+prefix+"_hlt_"+self.config.type
0195 _gtMc = "auto:"+prefix+"_mc_" +self.config.type
0196 self.data += """
0197 # add specific customizations
0198 _customInfo = {}
0199 _customInfo['menuType' ]= "%s"
0200 _customInfo['globalTags']= {}
0201 _customInfo['globalTags'][True ] = "%s"
0202 _customInfo['globalTags'][False] = "%s"
0203 _customInfo['inputFiles']={}
0204 _customInfo['inputFiles'][True] = "file:RelVal_Raw_%s_DATA.root"
0205 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
0206 _customInfo['maxEvents' ]= %s
0207 _customInfo['globalTag' ]= "%s"
0208 _customInfo['inputFile' ]= %s
0209 _customInfo['realData' ]= %s
0210
0211 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0212 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
0213 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
0214
0215 self.data += """
0216 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
0217 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
0218 """ % (self.config.type)
0219
0220
0221 self.data += """
0222 # Eras-based customisations
0223 from HLTrigger.Configuration.Eras import modifyHLTforEras
0224 modifyHLTforEras(%(process)s)
0225 """
0226
0227 if self.config.customise:
0228 self.data += "\n"
0229 self.data += "#User-defined customization functions\n"
0230 for customise in self.config.customise.split(","):
0231 customiseValues = customise.split(".")
0232 if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
0233 if len(customiseValues)==1:
0234 customiseValues.append("customise")
0235 customiseValues[0] = customiseValues[0].replace("/",".")
0236 self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
0237 self.data += "process = "+customiseValues[1]+"(process)\n"
0238
0239
0240
0241 def customize(self):
0242
0243
0244 if not self.config.fragment:
0245 self.build_source()
0246
0247
0248 self.fixPrescales()
0249
0250
0251 self.instrumentOpenMode()
0252
0253
0254 self.instrumentErrorEventType()
0255
0256
0257 self.instrumentTiming()
0258
0259
0260 self.overrideL1MenuXml()
0261
0262
0263 self.runL1Emulator()
0264
0265
0266 self.loadSetupCff()
0267
0268 if self.config.fragment:
0269 self.data += """
0270 # dummify hltGetConditions in cff's
0271 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
0272 %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
0273 result = cms.bool( True )
0274 )
0275 %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
0276 """
0277
0278
0279
0280
0281
0282
0283
0284
0285 for path in self.all_paths:
0286 match = re.match(r'(Scouting\w+)Output$', path)
0287 if match:
0288 module = 'hltOutput' + match.group(1)
0289 self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
0290 self.data = self.data.replace(' + process.'+module, '')
0291 self.data = self.data.replace(' process.'+module, '')
0292 else:
0293
0294
0295 self.overrideProcessName()
0296
0297
0298 self.addEras()
0299
0300
0301 self.overrideOutput()
0302
0303
0304 self.addGlobalOptions()
0305
0306
0307 self.overrideGlobalTag()
0308
0309
0310 self.updateMessageLogger()
0311
0312
0313 self.instrumentDQM()
0314
0315
0316 self.specificCustomize()
0317
0318
0319 def addGlobalOptions(self):
0320
0321 self.data += """
0322 # limit the number of events to be processed
0323 %%(process)s.maxEvents = cms.untracked.PSet(
0324 input = cms.untracked.int32( %d )
0325 )
0326 """ % self.config.events
0327
0328 self.data += """
0329 # enable TrigReport, TimeReport and MultiThreading
0330 %(process)s.options.wantSummary = True
0331 %(process)s.options.numberOfThreads = 4
0332 %(process)s.options.numberOfStreams = 0
0333 """
0334
0335 def _fix_parameter(self, **args):
0336 """arguments:
0337 name: parameter name (optional)
0338 type: parameter type (look for tracked and untracked variants)
0339 value: original value
0340 replace: replacement value
0341 """
0342 if 'name' in args:
0343 self.data = re.sub(
0344 r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0345 r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0346 self.data)
0347 else:
0348 self.data = re.sub(
0349 r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0350 r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0351 self.data)
0352
0353
0354 def fixPrescales(self):
0355
0356 if self.options['paths']:
0357 if self.options['paths'][0][0] == '-':
0358
0359 for minuspath in self.options['paths']:
0360 path = minuspath[1:]
0361 self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
0362 else:
0363
0364 for path in self.all_paths:
0365 if path not in self.options['paths']:
0366 self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
0367
0368 if self.config.prescale and (self.config.prescale.lower() != 'none'):
0369
0370 self.data += """
0371 # force the use of a specific HLT prescale column
0372 if 'PrescaleService' in %(dict)s:
0373 %(process)s.PrescaleService.forceDefault = True
0374 %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
0375 """
0376
0377
0378 def instrumentOpenMode(self):
0379 if self.config.open:
0380
0381 filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
0382 re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
0383
0384 self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
0385 self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
0386
0387 for some in splitter(filters, 1000):
0388 re_filters = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
0389 self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
0390
0391
0392 def instrumentErrorEventType(self):
0393 if self.config.errortype:
0394
0395 self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
0396 self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
0397 self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
0398
0399
0400 def overrideGlobalTag(self):
0401
0402
0403
0404
0405
0406
0407
0408
0409
0410
0411
0412
0413 if not self.config.data and not self.config.globaltag:
0414 if self.config.type in globalTag:
0415 self.config.globaltag = globalTag[self.config.type]
0416 else:
0417 self.config.globaltag = globalTag['GRun']
0418
0419
0420 if self.config.l1.override:
0421 self.config.l1.tag = self.config.l1.override
0422 self.config.l1.record = 'L1TUtmTriggerMenuRcd'
0423 self.config.l1.connect = ''
0424 self.config.l1.label = ''
0425 if not self.config.l1.snapshotTime:
0426 self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
0427 self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
0428 else:
0429 self.config.l1cond = None
0430
0431 if self.config.globaltag or self.config.l1cond:
0432 text = """
0433 # override the GlobalTag, connection string and pfnPrefix
0434 if 'GlobalTag' in %(dict)s:
0435 from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
0436 %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
0437 if self.config.globaltag:
0438 text += ", globaltag = %s" % repr(self.config.globaltag)
0439 if self.config.l1cond:
0440 text += ", conditions = %s" % repr(self.config.l1cond)
0441 text += ")\n"
0442 self.data += text
0443
0444 def overrideL1MenuXml(self):
0445
0446 if self.config.l1Xml.XmlFile:
0447 text = """
0448 # override the GlobalTag's L1T menu from an Xml file
0449 from HLTrigger.Configuration.CustomConfigs import L1XML
0450 %%(process)s = L1XML(%%(process)s,"%s")
0451 """ % (self.config.l1Xml.XmlFile)
0452 self.data += text
0453
0454 def runL1Emulator(self):
0455
0456 if self.config.emulator:
0457 text = """
0458 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0459 from HLTrigger.Configuration.CustomConfigs import L1REPACK
0460 %%(process)s = L1REPACK(%%(process)s,"%s")
0461 """ % (self.config.emulator)
0462 self.data += text
0463
0464 def overrideOutput(self):
0465
0466
0467
0468 if not self.config.hilton:
0469 self.data = re.sub(
0470 r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"(ShmStreamConsumer)" *,',
0471 r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),',
0472 self.data
0473 )
0474
0475 self.data = re.sub("""\
0476 \\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *['"](EvFOutputModule|GlobalEvFOutputModule)['"] *,
0477 use_compression = cms.untracked.bool\( (True|False) \),
0478 compression_algorithm = cms.untracked.string\( ['"](.+?)['"] \),
0479 compression_level = cms.untracked.int32\( (-?\d+) \),
0480 lumiSection_interval = cms.untracked.int32\( (-?\d+) \),
0481 (.+?),
0482 psetMap = cms.untracked.InputTag\( ['"]hltPSetMap['"] \)
0483 ""","""\
0484 %(process)s.hltOutput\g<2> = cms.OutputModule( "PoolOutputModule",
0485 fileName = cms.untracked.string( "output\g<2>.root" ),
0486 compressionAlgorithm = cms.untracked.string( "\g<5>" ),
0487 compressionLevel = cms.untracked.int32( \g<6> ),
0488 fastCloning = cms.untracked.bool( False ),
0489 dataset = cms.untracked.PSet(
0490 filterName = cms.untracked.string( "" ),
0491 dataTier = cms.untracked.string( "RAW" )
0492 ),
0493 \g<8>
0494 """, self.data, 0, re.DOTALL)
0495
0496 if not self.config.fragment and self.config.output == 'minimal':
0497
0498 self.data += """
0499 # add a single "keep *" output
0500 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
0501 fileName = cms.untracked.string( "output.root" ),
0502 fastCloning = cms.untracked.bool( False ),
0503 dataset = cms.untracked.PSet(
0504 dataTier = cms.untracked.string( 'AOD' ),
0505 filterName = cms.untracked.string( '' )
0506 ),
0507 outputCommands = cms.untracked.vstring( 'drop *',
0508 'keep edmTriggerResults_*_*_*',
0509 'keep triggerTriggerEvent_*_*_*',
0510 'keep GlobalAlgBlkBXVector_*_*_*',
0511 'keep GlobalExtBlkBXVector_*_*_*',
0512 'keep l1tEGammaBXVector_*_EGamma_*',
0513 'keep l1tEtSumBXVector_*_EtSum_*',
0514 'keep l1tJetBXVector_*_Jet_*',
0515 'keep l1tMuonBXVector_*_Muon_*',
0516 'keep l1tTauBXVector_*_Tau_*',
0517 )
0518 )
0519 %(process)s.MinimalOutput = cms.FinalPath( %(process)s.hltOutputMinimal )
0520 %(process)s.schedule.append( %(process)s.MinimalOutput )
0521 """
0522 elif not self.config.fragment and self.config.output == 'full':
0523
0524 self.data += """
0525 # add a single "keep *" output
0526 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
0527 fileName = cms.untracked.string( "output.root" ),
0528 fastCloning = cms.untracked.bool( False ),
0529 dataset = cms.untracked.PSet(
0530 dataTier = cms.untracked.string( 'RECO' ),
0531 filterName = cms.untracked.string( '' )
0532 ),
0533 outputCommands = cms.untracked.vstring( 'keep *' )
0534 )
0535 %(process)s.FullOutput = cms.FinalPath( %(process)s.hltOutputFull )
0536 %(process)s.schedule.append( %(process)s.FullOutput )
0537 """
0538
0539
0540 def addEras(self):
0541 if self.config.eras is None:
0542 return
0543 from Configuration.StandardSequences.Eras import eras
0544 erasSplit = self.config.eras.split(',')
0545 self.data = re.sub(r'process = cms.Process\( *"\w+"', '\n'.join(eras.pythonCfgLines[era] for era in erasSplit)+'\n\g<0>, '+', '.join(era for era in erasSplit), self.data)
0546
0547
0548 def loadSetupCff(self):
0549 if self.config.setup is None:
0550 return
0551 processLine = self.data.find("\n",self.data.find("cms.Process"))
0552 self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
0553
0554
0555 def overrideProcessName(self):
0556 if self.config.name is None:
0557 return
0558
0559
0560 self.config.name = self.config.name.replace("_","")
0561
0562 quote = '[\'\"]'
0563 self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
0564
0565
0566 if self.config.setup: self.data += """
0567 # avoid PrescaleService error due to missing HLT paths
0568 if 'PrescaleService' in process.__dict__:
0569 for pset in reversed(process.PrescaleService.prescaleTable):
0570 if not hasattr(process,pset.pathName.value()):
0571 process.PrescaleService.prescaleTable.remove(pset)
0572 """
0573
0574
0575 def updateMessageLogger(self):
0576
0577 self.data += """
0578 # show summaries from trigger analysers used at HLT
0579 if 'MessageLogger' in %(dict)s:
0580 %(process)s.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet()
0581 %(process)s.MessageLogger.L1GtTrigReport = cms.untracked.PSet()
0582 %(process)s.MessageLogger.L1TGlobalSummary = cms.untracked.PSet()
0583 %(process)s.MessageLogger.HLTrigReport = cms.untracked.PSet()
0584 %(process)s.MessageLogger.FastReport = cms.untracked.PSet()
0585 %(process)s.MessageLogger.ThroughputService = cms.untracked.PSet()
0586 """
0587
0588
0589 def loadAdditionalConditions(self, comment, *conditions):
0590
0591 self.data += """
0592 # %s
0593 if 'GlobalTag' in %%(dict)s:
0594 """ % comment
0595 for condition in conditions:
0596 self.data += """ %%(process)s.GlobalTag.toGet.append(
0597 cms.PSet(
0598 record = cms.string( '%(record)s' ),
0599 tag = cms.string( '%(tag)s' ),
0600 label = cms.untracked.string( '%(label)s' ),
0601 )
0602 )
0603 """ % condition
0604
0605
0606 def loadCffCommand(self, module):
0607
0608 if self.config.fragment:
0609 return 'from %s import *\n' % module
0610 else:
0611 return 'process.load( "%s" )\n' % module
0612
0613 def loadCff(self, module):
0614 self.data += self.loadCffCommand(module)
0615
0616
0617 def overrideParameters(self, module, parameters):
0618
0619 self.data += "if '%s' in %%(dict)s:\n" % module
0620 for (parameter, value) in parameters:
0621 self.data += " %%(process)s.%s.%s = %s\n" % (module, parameter, value)
0622 self.data += "\n"
0623
0624
0625 def removeElementFromSequencesTasksAndPaths(self, label):
0626 if label in self.data:
0627 label_re = r'\b(process\.)?' + label
0628 self.data = re.sub(r' *(\+|,) *' + label_re, '', self.data)
0629 self.data = re.sub(label_re + r' *(\+|,) *', '', self.data)
0630 self.data = re.sub(label_re, '', self.data)
0631
0632
0633 def instrumentTiming(self):
0634
0635 if self.config.timing:
0636 self.data += """
0637 # instrument the menu with the modules and EndPath needed for timing studies
0638 """
0639
0640 self.data += '\n# configure the FastTimerService\n'
0641 self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
0642
0643 self.data += """# print a text summary at the end of the job
0644 %(process)s.FastTimerService.printEventSummary = False
0645 %(process)s.FastTimerService.printRunSummary = False
0646 %(process)s.FastTimerService.printJobSummary = True
0647
0648 # enable DQM plots
0649 %(process)s.FastTimerService.enableDQM = True
0650
0651 # enable per-path DQM plots
0652 %(process)s.FastTimerService.enableDQMbyPath = True
0653
0654 # enable per-module DQM plots
0655 %(process)s.FastTimerService.enableDQMbyModule = True
0656
0657 # enable per-event DQM plots vs lumisection
0658 %(process)s.FastTimerService.enableDQMbyLumiSection = True
0659 %(process)s.FastTimerService.dqmLumiSectionsRange = 2500
0660
0661 # set the time resolution of the DQM plots
0662 %(process)s.FastTimerService.dqmTimeRange = 2000.
0663 %(process)s.FastTimerService.dqmTimeResolution = 10.
0664 %(process)s.FastTimerService.dqmPathTimeRange = 1000.
0665 %(process)s.FastTimerService.dqmPathTimeResolution = 5.
0666 %(process)s.FastTimerService.dqmModuleTimeRange = 200.
0667 %(process)s.FastTimerService.dqmModuleTimeResolution = 1.
0668
0669 # set the base DQM folder for the DQM plots
0670 %(process)s.FastTimerService.dqmPath = 'HLT/TimerService'
0671 %(process)s.FastTimerService.enableDQMbyProcesses = False
0672
0673 # write a JSON file with the information to be displayed in a pie chart
0674 %(process)s.FastTimerService.writeJSONSummary = True
0675 %(process)s.FastTimerService.jsonFileName = 'resources.json'
0676 """
0677
0678 self.data += '\n# configure the ThroughputService\n'
0679 self.loadCff('HLTrigger.Timer.ThroughputService_cfi')
0680
0681 self.data += """# enable DQM plots
0682 %(process)s.ThroughputService.enableDQM = True
0683
0684 # set the resolution of the DQM plots
0685 %(process)s.ThroughputService.eventRange = 10000
0686 %(process)s.ThroughputService.eventResolution = 1
0687 %(process)s.ThroughputService.timeRange = 60000
0688 %(process)s.ThroughputService.timeResolution = 10
0689
0690 # set the base DQM folder for the DQM plots
0691 %(process)s.ThroughputService.dqmPath = 'HLT/Throughput'
0692 %(process)s.ThroughputService.dqmPathByProcesses = False
0693 """
0694
0695
0696 def instrumentDQM(self):
0697 if not self.config.hilton:
0698
0699
0700
0701 self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaverPB')
0702 self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaver')
0703
0704
0705 dqmstore = "\n# load the DQMStore and DQMRootOutputModule\n"
0706 dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
0707 dqmstore += """
0708 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
0709 fileName = cms.untracked.string("DQMIO.root")
0710 )
0711 """
0712 empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\( *\).*')
0713 other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\()(.*)')
0714 if empty_path.search(self.data):
0715
0716 self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n', self.data)
0717 elif other_path.search(self.data):
0718
0719 self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<4>', self.data)
0720 else:
0721
0722 self.data += dqmstore
0723 self.data += '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n'
0724 self.data += '%(process)s.schedule.append( %(process)s.DQMOutput )\n'
0725
0726
0727 @staticmethod
0728 def dumppaths(paths):
0729 sys.stderr.write('Path selection:\n')
0730 for path in paths:
0731 sys.stderr.write('\t%s\n' % path)
0732 sys.stderr.write('\n\n')
0733
0734 def buildPathList(self):
0735 self.all_paths = self.getPathList()
0736
0737 if self.config.paths:
0738
0739 paths = self.config.paths.split(',')
0740 else:
0741
0742 paths = []
0743
0744
0745
0746
0747
0748
0749 if self.config.fragment or self.config.output in ('none', ):
0750 if self.config.paths:
0751
0752 pass
0753 else:
0754
0755 paths.append( "-*Output" )
0756 paths.append( "-RatesMonitoring")
0757 paths.append( "-DQMHistograms")
0758 if self.config.fragment: paths.append( "Scouting*Output" )
0759
0760 elif self.config.output in ('dqm', 'minimal', 'full'):
0761 if self.config.paths:
0762
0763 paths.append( "DQMHistograms" )
0764 else:
0765
0766 paths.append( "-*Output" )
0767 paths.append( "-RatesMonitoring")
0768 if self.config.fragment: paths.append( "Scouting*Output" )
0769
0770 else:
0771 if self.config.paths:
0772
0773 paths.append( "*Output" )
0774 paths.append( "DQMHistograms" )
0775 else:
0776
0777 pass
0778
0779
0780 if self.config.profiling:
0781 paths.append( "-HLTAnalyzerEndpath" )
0782
0783
0784 paths.append( "-OfflineOutput" )
0785
0786
0787 paths = self.expandWildcards(paths, self.all_paths)
0788
0789 if self.config.paths:
0790
0791 paths = self.consolidatePositiveList(paths)
0792 if not paths:
0793 raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
0794 else:
0795
0796 paths = self.consolidateNegativeList(paths)
0797 self.options['paths'] = paths
0798
0799 def buildOptions(self):
0800
0801 self.options['services'].append( "-DQM" )
0802 self.options['services'].append( "-FUShmDQMOutputService" )
0803 self.options['services'].append( "-MicroStateService" )
0804 self.options['services'].append( "-ModuleWebRegistry" )
0805 self.options['services'].append( "-TimeProfilerService" )
0806
0807
0808
0809 if not self.config.hilton:
0810 self.options['services'].append( "-EvFDaqDirector" )
0811 self.options['services'].append( "-FastMonitoringService" )
0812 self.options['services'].append( "-DQMStore" )
0813 self.options['modules'].append( "-hltDQMFileSaver" )
0814 self.options['modules'].append( "-hltDQMFileSaverPB" )
0815
0816 if self.config.fragment:
0817
0818 self.options['essources'].append( "-GlobalTag" )
0819 self.options['essources'].append( "-HepPDTESSource" )
0820 self.options['essources'].append( "-XMLIdealGeometryESSource" )
0821 self.options['essources'].append( "-eegeom" )
0822 self.options['essources'].append( "-es_hardcode" )
0823 self.options['essources'].append( "-magfield" )
0824
0825 self.options['esmodules'].append( "-SlaveField0" )
0826 self.options['esmodules'].append( "-SlaveField20" )
0827 self.options['esmodules'].append( "-SlaveField30" )
0828 self.options['esmodules'].append( "-SlaveField35" )
0829 self.options['esmodules'].append( "-SlaveField38" )
0830 self.options['esmodules'].append( "-SlaveField40" )
0831 self.options['esmodules'].append( "-VBF0" )
0832 self.options['esmodules'].append( "-VBF20" )
0833 self.options['esmodules'].append( "-VBF30" )
0834 self.options['esmodules'].append( "-VBF35" )
0835 self.options['esmodules'].append( "-VBF38" )
0836 self.options['esmodules'].append( "-VBF40" )
0837 self.options['esmodules'].append( "-CSCGeometryESModule" )
0838 self.options['esmodules'].append( "-CaloGeometryBuilder" )
0839 self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
0840 self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
0841 self.options['esmodules'].append( "-DTGeometryESModule" )
0842 self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
0843 self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
0844 self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
0845 self.options['esmodules'].append( "-EcalLaserCorrectionService" )
0846 self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
0847 self.options['esmodules'].append( "-GEMGeometryESModule" )
0848 self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
0849 self.options['esmodules'].append( "-HcalTopologyIdealEP" )
0850 self.options['esmodules'].append( "-MuonNumberingInitialization" )
0851 self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
0852 self.options['esmodules'].append( "-RPCGeometryESModule" )
0853 self.options['esmodules'].append( "-SiStripGainESProducer" )
0854 self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
0855 self.options['esmodules'].append( "-SiStripQualityESProducer" )
0856 self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
0857 self.options['esmodules'].append( "-TrackerAdditionalParametersPerDetESModule" )
0858 self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
0859 self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
0860 self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
0861 self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
0862 self.options['esmodules'].append( "-hcal_db_producer" )
0863 self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
0864 self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
0865 self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
0866 self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
0867 self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
0868 self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
0869 self.options['esmodules'].append( "-trackerTopology" )
0870
0871 self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
0872 self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
0873 self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
0874 self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
0875 self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
0876 self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
0877 self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
0878 self.options['esmodules'].append( "-XMLFromDBSource" )
0879 self.options['esmodules'].append( "-sistripconn" )
0880 self.options['esmodules'].append( "-siPixelQualityESProducer" )
0881
0882 self.options['services'].append( "-MessageLogger" )
0883
0884 self.options['psets'].append( "-maxEvents" )
0885 self.options['psets'].append( "-options" )
0886
0887
0888 self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
0889 self.options['modules'].append( "-hltOutputScoutingPF" )
0890
0891 if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
0892 self.options['services'].append( "-PrescaleService" )
0893
0894 if self.config.fragment or self.config.timing:
0895 self.options['services'].append( "-FastTimerService" )
0896 self.options['services'].append( "-ThroughputService" )
0897
0898
0899 def append_filenames(self, name, filenames):
0900 if len(filenames) > 255:
0901 token_open = "( *("
0902 token_close = ") )"
0903 else:
0904 token_open = "("
0905 token_close = ")"
0906
0907 self.data += " %s = cms.untracked.vstring%s\n" % (name, token_open)
0908 for line in filenames:
0909 self.data += " '%s',\n" % line
0910 self.data += " %s,\n" % (token_close)
0911
0912
0913 def expand_filenames(self, input):
0914
0915 if input[0:8] == 'dataset:':
0916 from .dasFileQuery import dasFileQuery
0917
0918 dataset = input[8:]
0919 files = dasFileQuery(dataset)
0920 else:
0921
0922 files = input.split(',')
0923 return files
0924
0925 def build_source(self):
0926 if self.config.hilton:
0927
0928 return
0929
0930 if self.config.input:
0931
0932 self.source = self.expand_filenames(self.config.input)
0933 elif self.config.data:
0934
0935 self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
0936 else:
0937
0938 self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
0939
0940 if self.config.parent:
0941
0942 self.parent = self.expand_filenames(self.config.parent)
0943
0944 self.data += """
0945 # source module (EDM inputs)
0946 %(process)s.source = cms.Source( "PoolSource",
0947 """
0948 self.append_filenames("fileNames", self.source)
0949 if (self.parent):
0950 self.append_filenames("secondaryFileNames", self.parent)
0951 self.data += """\
0952 inputCommands = cms.untracked.vstring(
0953 'keep *'
0954 )
0955 )
0956 """