File indexing completed on 2022-06-06 07:38:40
0001
0002
0003 from __future__ import absolute_import
0004 import sys
0005 import re
0006 import os
0007 from .pipe import pipe as _pipe
0008 from .options import globalTag
0009 from itertools import islice
0010
0011 def splitter(iterator, n):
0012 i = iterator.__iter__()
0013 while True:
0014 l = list(islice(i, n))
0015 if l:
0016 yield l
0017 else:
0018 break
0019
0020
0021 class HLTProcess(object):
0022
0023 def __init__(self, configuration):
0024 self.config = configuration
0025 self.data = None
0026 self.source = []
0027 self.parent = []
0028
0029 self.options = {
0030 'essources' : [],
0031 'esmodules' : [],
0032 'modules' : [],
0033 'sequences' : [],
0034 'services' : [],
0035 'paths' : [],
0036 'psets' : [],
0037 'blocks' : [],
0038 }
0039
0040 self.labels = {}
0041 if self.config.fragment:
0042 self.labels['process'] = 'fragment'
0043 self.labels['dict'] = 'fragment.__dict__'
0044 else:
0045 self.labels['process'] = 'process'
0046 self.labels['dict'] = 'process.__dict__'
0047
0048 if self.config.prescale and (self.config.prescale.lower() != 'none'):
0049 self.labels['prescale'] = self.config.prescale
0050
0051
0052 from .confdbOfflineConverter import OfflineConverter
0053 self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port)
0054 self.buildPathList()
0055 self.buildOptions()
0056 self.getSetupConfigurationFromDB()
0057 self.getRawConfigurationFromDB()
0058 self.customize()
0059
0060 def getSetupConfigurationFromDB(self):
0061 if not self.config.setup:
0062 return
0063
0064 if ".py" in self.config.setup:
0065 self.config.setupFile = self.config.setup.split(".py")[0]
0066 return
0067 args = ['--configName', self.config.setup ]
0068 args.append('--noedsources')
0069 args.append('--nopaths')
0070 for key, vals in self.options.items():
0071 if vals:
0072 args.extend(('--'+key, ','.join(vals)))
0073 args.append('--cff')
0074 data, err = self.converter.query( *args )
0075 if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0076 sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
0077 sys.stderr.write(err + "\n\n")
0078 sys.exit(1)
0079 self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
0080 outfile = open(self.config.setupFile+".py","w+")
0081 outfile.write("# This file is automatically generated by hltGetConfiguration.\n" + data)
0082
0083 def getRawConfigurationFromDB(self):
0084 if self.config.menu.run:
0085 args = ['--runNumber', self.config.menu.run]
0086 else:
0087 args = ['--configName', self.config.menu.name ]
0088 if not self.config.hilton:
0089
0090 args.append('--noedsources')
0091 for key, vals in self.options.items():
0092 if vals:
0093 args.extend(('--'+key, ','.join(vals)))
0094 data, err = self.converter.query( *args )
0095 if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0096 sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0097 sys.stderr.write(err + "\n\n")
0098 sys.exit(1)
0099 self.data = data
0100
0101 def getPathList(self):
0102 if self.config.menu.run:
0103 args = ['--runNumber', self.config.menu.run]
0104 else:
0105 args = ['--configName', self.config.menu.name]
0106 args.extend( (
0107 '--cff',
0108 '--noedsources',
0109 '--noes',
0110 '--noservices',
0111 '--nosequences',
0112 '--nomodules'
0113 ) )
0114
0115 data, err = self.converter.query( *args )
0116 if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0117 sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0118 sys.stderr.write(err + "\n\n")
0119 sys.exit(1)
0120 filter = re.compile(r' *= *cms.(End|Final)?Path.*')
0121 paths = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
0122 return paths
0123
0124
0125 @staticmethod
0126 def expandWildcards(globs, collection):
0127
0128
0129 matches = []
0130 for glob in globs:
0131 negate = ''
0132 if glob[0] == '-':
0133 negate = '-'
0134 glob = glob[1:]
0135
0136 filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
0137 matches.extend( negate + element for element in collection if filter.match(element) )
0138 return matches
0139
0140
0141 @staticmethod
0142 def consolidateNegativeList(elements):
0143
0144
0145 result = set()
0146 for element in elements:
0147 if element[0] == '-':
0148 result.add( element )
0149 else:
0150 result.discard( '-' + element )
0151 return sorted( element for element in result )
0152
0153 @staticmethod
0154 def consolidatePositiveList(elements):
0155
0156
0157 result = set()
0158 for element in elements:
0159 if element[0] == '-':
0160 result.discard( element[1:] )
0161 else:
0162 result.add( element )
0163 return sorted( element for element in result )
0164
0165
0166
0167 def dump(self):
0168 self.data = self.data % self.labels
0169 if self.config.fragment:
0170 self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
0171 self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
0172 return self.data
0173
0174
0175
0176 def specificCustomize(self):
0177
0178 if self.config.fragment:
0179 self.data += """
0180 # add specific customizations
0181 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0182 fragment = customizeHLTforAll(fragment,"%s")
0183 """ % (self.config.type)
0184 elif self.config.hilton:
0185
0186 pass
0187 else:
0188 if self.config.type=="Fake":
0189 prefix = "run1"
0190 elif self.config.type in ("Fake1","Fake2","2018"):
0191 prefix = "run2"
0192 else:
0193 prefix = "run3"
0194 _gtData = "auto:"+prefix+"_hlt_"+self.config.type
0195 _gtMc = "auto:"+prefix+"_mc_" +self.config.type
0196 self.data += """
0197 # add specific customizations
0198 _customInfo = {}
0199 _customInfo['menuType' ]= "%s"
0200 _customInfo['globalTags']= {}
0201 _customInfo['globalTags'][True ] = "%s"
0202 _customInfo['globalTags'][False] = "%s"
0203 _customInfo['inputFiles']={}
0204 _customInfo['inputFiles'][True] = "file:RelVal_Raw_%s_DATA.root"
0205 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
0206 _customInfo['maxEvents' ]= %s
0207 _customInfo['globalTag' ]= "%s"
0208 _customInfo['inputFile' ]= %s
0209 _customInfo['realData' ]= %s
0210
0211 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0212 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
0213 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
0214
0215 self.data += """
0216 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
0217 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
0218 """ % (self.config.type)
0219
0220
0221 self.data += """
0222 # Eras-based customisations
0223 from HLTrigger.Configuration.Eras import modifyHLTforEras
0224 modifyHLTforEras(%(process)s)
0225 """
0226
0227 if self.config.customise:
0228 self.data += "\n"
0229 self.data += "#User-defined customization functions\n"
0230 for customise in self.config.customise.split(","):
0231 customiseValues = customise.split(".")
0232 if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
0233 if len(customiseValues)==1:
0234 customiseValues.append("customise")
0235 customiseValues[0] = customiseValues[0].replace("/",".")
0236 self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
0237 self.data += "process = "+customiseValues[1]+"(process)\n"
0238
0239
0240
0241 def customize(self):
0242
0243
0244 if not self.config.fragment:
0245 self.build_source()
0246
0247
0248 if self.config.type in ('HIon', ):
0249 if self.config.data:
0250 if not self.config.fragment:
0251 self._fix_parameter( type = 'InputTag', value = 'rawDataCollector', replace = 'rawDataRepacker')
0252
0253
0254 self.fixPrescales()
0255
0256
0257 self.instrumentOpenMode()
0258
0259
0260 self.instrumentErrorEventType()
0261
0262
0263 self.instrumentTiming()
0264
0265
0266 self.overrideL1MenuXml()
0267
0268
0269 self.runL1Emulator()
0270
0271
0272 self.loadSetupCff()
0273
0274 if self.config.fragment:
0275 self.data += """
0276 # dummify hltGetConditions in cff's
0277 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
0278 %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
0279 result = cms.bool( True )
0280 )
0281 %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
0282 """
0283
0284
0285
0286
0287
0288
0289
0290
0291 for path in self.all_paths:
0292 match = re.match(r'(Scouting\w+)Output$', path)
0293 if match:
0294 module = 'hltOutput' + match.group(1)
0295 self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
0296 self.data = self.data.replace(' + process.'+module, '')
0297 self.data = self.data.replace(' process.'+module, '')
0298 else:
0299
0300
0301 self.overrideProcessName()
0302
0303
0304 self.addEras()
0305
0306
0307 self.overrideOutput()
0308
0309
0310 self.addGlobalOptions()
0311
0312
0313 self.overrideGlobalTag()
0314
0315
0316 self.updateMessageLogger()
0317
0318
0319 self.instrumentDQM()
0320
0321
0322 self.specificCustomize()
0323
0324
0325 def addGlobalOptions(self):
0326
0327 self.data += """
0328 # limit the number of events to be processed
0329 %%(process)s.maxEvents = cms.untracked.PSet(
0330 input = cms.untracked.int32( %d )
0331 )
0332 """ % self.config.events
0333
0334 self.data += """
0335 # enable TrigReport, TimeReport and MultiThreading
0336 %(process)s.options.wantSummary = True
0337 %(process)s.options.numberOfThreads = 4
0338 %(process)s.options.numberOfStreams = 0
0339 """
0340
0341 def _fix_parameter(self, **args):
0342 """arguments:
0343 name: parameter name (optional)
0344 type: parameter type (look for tracked and untracked variants)
0345 value: original value
0346 replace: replacement value
0347 """
0348 if 'name' in args:
0349 self.data = re.sub(
0350 r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0351 r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0352 self.data)
0353 else:
0354 self.data = re.sub(
0355 r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0356 r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0357 self.data)
0358
0359
0360 def fixPrescales(self):
0361
0362 if self.options['paths']:
0363 if self.options['paths'][0][0] == '-':
0364
0365 for minuspath in self.options['paths']:
0366 path = minuspath[1:]
0367 self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
0368 else:
0369
0370 for path in self.all_paths:
0371 if path not in self.options['paths']:
0372 self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
0373
0374 if self.config.prescale and (self.config.prescale.lower() != 'none'):
0375
0376 self.data += """
0377 # force the use of a specific HLT prescale column
0378 if 'PrescaleService' in %(dict)s:
0379 %(process)s.PrescaleService.forceDefault = True
0380 %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
0381 """
0382
0383
0384 def instrumentOpenMode(self):
0385 if self.config.open:
0386
0387 filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
0388 re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
0389
0390 self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
0391 self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
0392
0393 for some in splitter(filters, 1000):
0394 re_filters = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
0395 self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
0396
0397
0398 def instrumentErrorEventType(self):
0399 if self.config.errortype:
0400
0401 self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
0402 self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
0403 self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
0404
0405
0406 def overrideGlobalTag(self):
0407
0408
0409
0410
0411
0412
0413
0414
0415
0416
0417
0418
0419 if not self.config.data and not self.config.globaltag:
0420 if self.config.type in globalTag:
0421 self.config.globaltag = globalTag[self.config.type]
0422 else:
0423 self.config.globaltag = globalTag['GRun']
0424
0425
0426 if self.config.l1.override:
0427 self.config.l1.tag = self.config.l1.override
0428 self.config.l1.record = 'L1TUtmTriggerMenuRcd'
0429 self.config.l1.connect = ''
0430 self.config.l1.label = ''
0431 if not self.config.l1.snapshotTime:
0432 self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
0433 self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
0434 else:
0435 self.config.l1cond = None
0436
0437 if self.config.globaltag or self.config.l1cond:
0438 text = """
0439 # override the GlobalTag, connection string and pfnPrefix
0440 if 'GlobalTag' in %(dict)s:
0441 from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
0442 %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
0443 if self.config.globaltag:
0444 text += ", globaltag = %s" % repr(self.config.globaltag)
0445 if self.config.l1cond:
0446 text += ", conditions = %s" % repr(self.config.l1cond)
0447 text += ")\n"
0448 self.data += text
0449
0450 def overrideL1MenuXml(self):
0451
0452 if self.config.l1Xml.XmlFile:
0453 text = """
0454 # override the GlobalTag's L1T menu from an Xml file
0455 from HLTrigger.Configuration.CustomConfigs import L1XML
0456 %%(process)s = L1XML(%%(process)s,"%s")
0457 """ % (self.config.l1Xml.XmlFile)
0458 self.data += text
0459
0460 def runL1Emulator(self):
0461
0462 if self.config.emulator:
0463 text = """
0464 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0465 from HLTrigger.Configuration.CustomConfigs import L1REPACK
0466 %%(process)s = L1REPACK(%%(process)s,"%s")
0467 """ % (self.config.emulator)
0468 self.data += text
0469
0470 def overrideOutput(self):
0471
0472
0473
0474 if not self.config.hilton:
0475 self.data = re.sub(
0476 r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"(ShmStreamConsumer)" *,',
0477 r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),',
0478 self.data
0479 )
0480 self.data = re.sub(
0481 r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"EvFOutputModule" *,\n use_compression = cms.untracked.bool\( True \),\n compression_algorithm = cms.untracked.string\( "ZLIB" \),\n compression_level = cms.untracked.int32\( 1 \),\n lumiSection_interval = cms.untracked.int32\( 0 \),\n(.+?),\n psetMap = cms.untracked.InputTag\( "hltPSetMap" \)\n',
0482 r'\1hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),\n\3\n',
0483 self.data,0,re.DOTALL
0484 )
0485 self.data = re.sub(
0486 r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"GlobalEvFOutputModule" *,\n use_compression = cms.untracked.bool\( True \),\n compression_algorithm = cms.untracked.string\( "ZLIB" \),\n compression_level = cms.untracked.int32\( 1 \),\n lumiSection_interval = cms.untracked.int32\( 0 \),\n(.+?),\n psetMap = cms.untracked.InputTag\( "hltPSetMap" \)\n',
0487 r'\1hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),\n\3\n',
0488 self.data,0,re.DOTALL
0489 )
0490 if not self.config.fragment and self.config.output == 'minimal':
0491
0492 self.data += """
0493 # add a single "keep *" output
0494 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
0495 fileName = cms.untracked.string( "output.root" ),
0496 fastCloning = cms.untracked.bool( False ),
0497 dataset = cms.untracked.PSet(
0498 dataTier = cms.untracked.string( 'AOD' ),
0499 filterName = cms.untracked.string( '' )
0500 ),
0501 outputCommands = cms.untracked.vstring( 'drop *',
0502 'keep edmTriggerResults_*_*_*',
0503 'keep triggerTriggerEvent_*_*_*',
0504 'keep GlobalAlgBlkBXVector_*_*_*',
0505 'keep GlobalExtBlkBXVector_*_*_*',
0506 'keep l1tEGammaBXVector_*_EGamma_*',
0507 'keep l1tEtSumBXVector_*_EtSum_*',
0508 'keep l1tJetBXVector_*_Jet_*',
0509 'keep l1tMuonBXVector_*_Muon_*',
0510 'keep l1tTauBXVector_*_Tau_*',
0511 )
0512 )
0513 %(process)s.MinimalOutput = cms.FinalPath( %(process)s.hltOutputMinimal )
0514 %(process)s.schedule.append( %(process)s.MinimalOutput )
0515 """
0516 elif not self.config.fragment and self.config.output == 'full':
0517
0518 self.data += """
0519 # add a single "keep *" output
0520 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
0521 fileName = cms.untracked.string( "output.root" ),
0522 fastCloning = cms.untracked.bool( False ),
0523 dataset = cms.untracked.PSet(
0524 dataTier = cms.untracked.string( 'RECO' ),
0525 filterName = cms.untracked.string( '' )
0526 ),
0527 outputCommands = cms.untracked.vstring( 'keep *' )
0528 )
0529 %(process)s.FullOutput = cms.FinalPath( %(process)s.hltOutputFull )
0530 %(process)s.schedule.append( %(process)s.FullOutput )
0531 """
0532
0533
0534 def addEras(self):
0535 if self.config.eras is None:
0536 return
0537 from Configuration.StandardSequences.Eras import eras
0538 erasSplit = self.config.eras.split(',')
0539 self.data = re.sub(r'process = cms.Process\( *"\w+"', '\n'.join(eras.pythonCfgLines[era] for era in erasSplit)+'\n\g<0>, '+', '.join(era for era in erasSplit), self.data)
0540
0541
0542 def loadSetupCff(self):
0543 if self.config.setup is None:
0544 return
0545 processLine = self.data.find("\n",self.data.find("cms.Process"))
0546 self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
0547
0548
0549 def overrideProcessName(self):
0550 if self.config.name is None:
0551 return
0552
0553
0554 self.config.name = self.config.name.replace("_","")
0555
0556 quote = '[\'\"]'
0557 self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
0558
0559
0560 if self.config.setup: self.data += """
0561 # avoid PrescaleService error due to missing HLT paths
0562 if 'PrescaleService' in process.__dict__:
0563 for pset in reversed(process.PrescaleService.prescaleTable):
0564 if not hasattr(process,pset.pathName.value()):
0565 process.PrescaleService.prescaleTable.remove(pset)
0566 """
0567
0568
0569 def updateMessageLogger(self):
0570
0571 self.data += """
0572 # show summaries from trigger analysers used at HLT
0573 if 'MessageLogger' in %(dict)s:
0574 %(process)s.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet()
0575 %(process)s.MessageLogger.L1GtTrigReport = cms.untracked.PSet()
0576 %(process)s.MessageLogger.L1TGlobalSummary = cms.untracked.PSet()
0577 %(process)s.MessageLogger.HLTrigReport = cms.untracked.PSet()
0578 %(process)s.MessageLogger.FastReport = cms.untracked.PSet()
0579 %(process)s.MessageLogger.ThroughputService = cms.untracked.PSet()
0580 """
0581
0582
0583 def loadAdditionalConditions(self, comment, *conditions):
0584
0585 self.data += """
0586 # %s
0587 if 'GlobalTag' in %%(dict)s:
0588 """ % comment
0589 for condition in conditions:
0590 self.data += """ %%(process)s.GlobalTag.toGet.append(
0591 cms.PSet(
0592 record = cms.string( '%(record)s' ),
0593 tag = cms.string( '%(tag)s' ),
0594 label = cms.untracked.string( '%(label)s' ),
0595 )
0596 )
0597 """ % condition
0598
0599
0600 def loadCffCommand(self, module):
0601
0602 if self.config.fragment:
0603 return 'from %s import *\n' % module
0604 else:
0605 return 'process.load( "%s" )\n' % module
0606
0607 def loadCff(self, module):
0608 self.data += self.loadCffCommand(module)
0609
0610
0611 def overrideParameters(self, module, parameters):
0612
0613 self.data += "if '%s' in %%(dict)s:\n" % module
0614 for (parameter, value) in parameters:
0615 self.data += " %%(process)s.%s.%s = %s\n" % (module, parameter, value)
0616 self.data += "\n"
0617
0618
0619 def removeElementFromSequencesTasksAndPaths(self, label):
0620 if label in self.data:
0621 label_re = r'\b(process\.)?' + label
0622 self.data = re.sub(r' *(\+|,) *' + label_re, '', self.data)
0623 self.data = re.sub(label_re + r' *(\+|,) *', '', self.data)
0624 self.data = re.sub(label_re, '', self.data)
0625
0626
0627 def instrumentTiming(self):
0628
0629 if self.config.timing:
0630 self.data += """
0631 # instrument the menu with the modules and EndPath needed for timing studies
0632 """
0633
0634 self.data += '\n# configure the FastTimerService\n'
0635 self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
0636
0637 self.data += """# print a text summary at the end of the job
0638 %(process)s.FastTimerService.printEventSummary = False
0639 %(process)s.FastTimerService.printRunSummary = False
0640 %(process)s.FastTimerService.printJobSummary = True
0641
0642 # enable DQM plots
0643 %(process)s.FastTimerService.enableDQM = True
0644
0645 # enable per-path DQM plots (starting with CMSSW 9.2.3-patch2)
0646 %(process)s.FastTimerService.enableDQMbyPath = True
0647
0648 # enable per-module DQM plots
0649 %(process)s.FastTimerService.enableDQMbyModule = True
0650
0651 # enable per-event DQM plots vs lumisection
0652 %(process)s.FastTimerService.enableDQMbyLumiSection = True
0653 %(process)s.FastTimerService.dqmLumiSectionsRange = 2500
0654
0655 # set the time resolution of the DQM plots
0656 %(process)s.FastTimerService.dqmTimeRange = 2000.
0657 %(process)s.FastTimerService.dqmTimeResolution = 10.
0658 %(process)s.FastTimerService.dqmPathTimeRange = 1000.
0659 %(process)s.FastTimerService.dqmPathTimeResolution = 5.
0660 %(process)s.FastTimerService.dqmModuleTimeRange = 200.
0661 %(process)s.FastTimerService.dqmModuleTimeResolution = 1.
0662
0663 # set the base DQM folder for the plots
0664 %(process)s.FastTimerService.dqmPath = 'HLT/TimerService'
0665 %(process)s.FastTimerService.enableDQMbyProcesses = False
0666 """
0667
0668
0669 def instrumentDQM(self):
0670 if not self.config.hilton:
0671
0672
0673
0674 self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaverPB')
0675 self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaver')
0676
0677
0678 dqmstore = "\n# load the DQMStore and DQMRootOutputModule\n"
0679 dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
0680 dqmstore += """
0681 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
0682 fileName = cms.untracked.string("DQMIO.root")
0683 )
0684 """
0685 empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\( *\).*')
0686 other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\()(.*)')
0687 if empty_path.search(self.data):
0688
0689 self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n', self.data)
0690 elif other_path.search(self.data):
0691
0692 self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<4>', self.data)
0693 else:
0694
0695 self.data += dqmstore
0696 self.data += '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n'
0697 self.data += '%(process)s.schedule.append( %(process)s.DQMOutput )\n'
0698
0699
0700 @staticmethod
0701 def dumppaths(paths):
0702 sys.stderr.write('Path selection:\n')
0703 for path in paths:
0704 sys.stderr.write('\t%s\n' % path)
0705 sys.stderr.write('\n\n')
0706
0707 def buildPathList(self):
0708 self.all_paths = self.getPathList()
0709
0710 if self.config.paths:
0711
0712 paths = self.config.paths.split(',')
0713 else:
0714
0715 paths = []
0716
0717
0718
0719
0720
0721
0722 if self.config.fragment or self.config.output in ('none', ):
0723 if self.config.paths:
0724
0725 pass
0726 else:
0727
0728 paths.append( "-*Output" )
0729 paths.append( "-RatesMonitoring")
0730 paths.append( "-DQMHistograms")
0731 if self.config.fragment: paths.append( "Scouting*Output" )
0732
0733 elif self.config.output in ('dqm', 'minimal', 'full'):
0734 if self.config.paths:
0735
0736 paths.append( "DQMHistograms" )
0737 else:
0738
0739 paths.append( "-*Output" )
0740 paths.append( "-RatesMonitoring")
0741 if self.config.fragment: paths.append( "Scouting*Output" )
0742
0743 else:
0744 if self.config.paths:
0745
0746 paths.append( "*Output" )
0747 paths.append( "DQMHistograms" )
0748 else:
0749
0750 pass
0751
0752
0753 if self.config.profiling:
0754 paths.append( "-HLTAnalyzerEndpath" )
0755
0756
0757 paths.append( "-OfflineOutput" )
0758
0759
0760 paths = self.expandWildcards(paths, self.all_paths)
0761
0762 if self.config.paths:
0763
0764 paths = self.consolidatePositiveList(paths)
0765 if not paths:
0766 raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
0767 else:
0768
0769 paths = self.consolidateNegativeList(paths)
0770 self.options['paths'] = paths
0771
0772 def buildOptions(self):
0773
0774 self.options['services'].append( "-DQM" )
0775 self.options['services'].append( "-FUShmDQMOutputService" )
0776 self.options['services'].append( "-MicroStateService" )
0777 self.options['services'].append( "-ModuleWebRegistry" )
0778 self.options['services'].append( "-TimeProfilerService" )
0779
0780
0781
0782 if not self.config.hilton:
0783 self.options['services'].append( "-EvFDaqDirector" )
0784 self.options['services'].append( "-FastMonitoringService" )
0785 self.options['services'].append( "-DQMStore" )
0786 self.options['modules'].append( "-hltDQMFileSaver" )
0787 self.options['modules'].append( "-hltDQMFileSaverPB" )
0788
0789 if self.config.fragment:
0790
0791 self.options['essources'].append( "-GlobalTag" )
0792 self.options['essources'].append( "-HepPDTESSource" )
0793 self.options['essources'].append( "-XMLIdealGeometryESSource" )
0794 self.options['essources'].append( "-eegeom" )
0795 self.options['essources'].append( "-es_hardcode" )
0796 self.options['essources'].append( "-magfield" )
0797
0798 self.options['esmodules'].append( "-SlaveField0" )
0799 self.options['esmodules'].append( "-SlaveField20" )
0800 self.options['esmodules'].append( "-SlaveField30" )
0801 self.options['esmodules'].append( "-SlaveField35" )
0802 self.options['esmodules'].append( "-SlaveField38" )
0803 self.options['esmodules'].append( "-SlaveField40" )
0804 self.options['esmodules'].append( "-VBF0" )
0805 self.options['esmodules'].append( "-VBF20" )
0806 self.options['esmodules'].append( "-VBF30" )
0807 self.options['esmodules'].append( "-VBF35" )
0808 self.options['esmodules'].append( "-VBF38" )
0809 self.options['esmodules'].append( "-VBF40" )
0810 self.options['esmodules'].append( "-CSCGeometryESModule" )
0811 self.options['esmodules'].append( "-CaloGeometryBuilder" )
0812 self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
0813 self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
0814 self.options['esmodules'].append( "-DTGeometryESModule" )
0815 self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
0816 self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
0817 self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
0818 self.options['esmodules'].append( "-EcalLaserCorrectionService" )
0819 self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
0820 self.options['esmodules'].append( "-GEMGeometryESModule" )
0821 self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
0822 self.options['esmodules'].append( "-HcalTopologyIdealEP" )
0823 self.options['esmodules'].append( "-MuonNumberingInitialization" )
0824 self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
0825 self.options['esmodules'].append( "-RPCGeometryESModule" )
0826 self.options['esmodules'].append( "-SiStripGainESProducer" )
0827 self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
0828 self.options['esmodules'].append( "-SiStripQualityESProducer" )
0829 self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
0830 self.options['esmodules'].append( "-TrackerAdditionalParametersPerDetESModule" )
0831 self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
0832 self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
0833 self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
0834 self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
0835 self.options['esmodules'].append( "-hcal_db_producer" )
0836 self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
0837 self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
0838 self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
0839 self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
0840 self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
0841 self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
0842 self.options['esmodules'].append( "-trackerTopology" )
0843
0844 self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
0845 self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
0846 self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
0847 self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
0848 self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
0849 self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
0850 self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
0851 self.options['esmodules'].append( "-XMLFromDBSource" )
0852 self.options['esmodules'].append( "-sistripconn" )
0853
0854 self.options['services'].append( "-MessageLogger" )
0855
0856 self.options['psets'].append( "-maxEvents" )
0857 self.options['psets'].append( "-options" )
0858
0859
0860 self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
0861 self.options['modules'].append( "-hltOutputScoutingPF" )
0862
0863 if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
0864 self.options['services'].append( "-PrescaleService" )
0865
0866 if self.config.fragment or self.config.timing:
0867 self.options['services'].append( "-FastTimerService" )
0868
0869
0870 def append_filenames(self, name, filenames):
0871 if len(filenames) > 255:
0872 token_open = "( *("
0873 token_close = ") )"
0874 else:
0875 token_open = "("
0876 token_close = ")"
0877
0878 self.data += " %s = cms.untracked.vstring%s\n" % (name, token_open)
0879 for line in filenames:
0880 self.data += " '%s',\n" % line
0881 self.data += " %s,\n" % (token_close)
0882
0883
0884 def expand_filenames(self, input):
0885
0886 if input[0:8] == 'dataset:':
0887 from .dasFileQuery import dasFileQuery
0888
0889 dataset = input[8:]
0890 files = dasFileQuery(dataset)
0891 else:
0892
0893 files = input.split(',')
0894 return files
0895
0896 def build_source(self):
0897 if self.config.hilton:
0898
0899 return
0900
0901 if self.config.input:
0902
0903 self.source = self.expand_filenames(self.config.input)
0904 elif self.config.data:
0905
0906 self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
0907 else:
0908
0909 self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
0910
0911 if self.config.parent:
0912
0913 self.parent = self.expand_filenames(self.config.parent)
0914
0915 self.data += """
0916 # source module (EDM inputs)
0917 %(process)s.source = cms.Source( "PoolSource",
0918 """
0919 self.append_filenames("fileNames", self.source)
0920 if (self.parent):
0921 self.append_filenames("secondaryFileNames", self.parent)
0922 self.data += """\
0923 inputCommands = cms.untracked.vstring(
0924 'keep *'
0925 )
0926 )
0927 """