python/Tools/confdb.py

0001 #!/usr/bin/env python
0002
0003 from __future__ import absolute_import
0004 import sys
0005 import re
0006 import os
0007 from .pipe import pipe as _pipe
0008 from .options import globalTag
0009 from itertools import islice
0010
0011 def splitter(iterator, n):
0012   i = iterator.__iter__()
0013   while True:
0014     l = list(islice(i, n))
0015     if l:
0016       yield l
0017     else:
0018       break
0019
0020
0021 class HLTProcess(object):
0022
0023   def __init__(self, configuration):
0024     self.config = configuration
0025     self.data   = None
0026     self.source = []
0027     self.parent = []
0028
0029     self.options = {
0030       'essources' : [],
0031       'esmodules' : [],
0032       'modules'   : [],
0033       'sequences' : [],
0034       'services'  : [],
0035       'paths'     : [],
0036       'psets'     : [],
0037       'blocks'    : [],
0038     }
0039
0040     self.labels = {}
0041     if self.config.fragment:
0042       self.labels['process'] = 'fragment'
0043       self.labels['dict']    = 'fragment.__dict__'
0044     else:
0045       self.labels['process'] = 'process'
0046       self.labels['dict']    = 'process.__dict__'
0047
0048     if self.config.prescale and (self.config.prescale.lower() != 'none'):
0049       self.labels['prescale'] = self.config.prescale
0050
0051     # get the configuration from ConfdB
0052     from .confdbOfflineConverter import OfflineConverter
0053     self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port)
0054     self.buildPathList()
0055     self.buildOptions()
0056     self.getSetupConfigurationFromDB()
0057     self.getRawConfigurationFromDB()
0058     self.customize()
0059
0060   def getSetupConfigurationFromDB(self):
0061     if not self.config.setup:
0062         return
0063     ## if --setup is a python file, use directly that file as setup_cff.py
0064     if ".py" in self.config.setup:
0065         self.config.setupFile = self.config.setup.split(".py")[0]
0066         return
0067     args = ['--configName', self.config.setup ]
0068     args.append('--noedsources')
0069     args.append('--nopaths')
0070     for key, vals in self.options.items():
0071       if vals:
0072         args.extend(('--'+key, ','.join(vals)))
0073     args.append('--cff')
0074     data, err = self.converter.query( *args )
0075     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0076         sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
0077         sys.stderr.write(err + "\n\n")
0078         sys.exit(1)
0079     self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
0080     outfile = open(self.config.setupFile+".py","w+")
0081     outfile.write("# This file is automatically generated by hltGetConfiguration.\n" + data)
0082
0083   def getRawConfigurationFromDB(self):
0084     if self.config.menu.run:
0085       args = ['--runNumber', self.config.menu.run]
0086     else:
0087       args = ['--configName', self.config.menu.name ]
0088     if not self.config.hilton:
0089         # keep the original Source when running on Hilton
0090         args.append('--noedsources')
0091     for key, vals in self.options.items():
0092       if vals:
0093         args.extend(('--'+key, ','.join(vals)))
0094     data, err = self.converter.query( *args )
0095     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0096         sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0097         sys.stderr.write(err + "\n\n")
0098         sys.exit(1)
0099     self.data = data
0100
0101   def getPathList(self):
0102     if self.config.menu.run:
0103       args = ['--runNumber', self.config.menu.run]
0104     else:
0105       args = ['--configName', self.config.menu.name]
0106     args.extend( (
0107       '--cff',
0108       '--noedsources',
0109       '--noes',
0110       '--noservices',
0111       '--nosequences',
0112       '--nomodules'
0113     ) )
0114
0115     data, err = self.converter.query( *args )
0116     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0117         sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0118         sys.stderr.write(err + "\n\n")
0119         sys.exit(1)
0120     filter = re.compile(r' *= *cms.(End|Final)?Path.*')
0121     paths  = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
0122     return paths
0123
0124
0125   @staticmethod
0126   def expandWildcards(globs, collection):
0127     # expand a list of unix-style wildcards matching a given collection
0128     # wildcards with no matches are silently discarded
0129     matches = []
0130     for glob in globs:
0131       negate = ''
0132       if glob[0] == '-':
0133         negate = '-'
0134         glob   = glob[1:]
0135       # translate a unix-style glob expression into a regular expression
0136       filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
0137       matches.extend( negate + element for element in collection if filter.match(element) )
0138     return matches
0139
0140
0141   @staticmethod
0142   def consolidateNegativeList(elements):
0143     # consolidate a list of path exclusions and re-inclusions
0144     # the result is the list of paths to be removed from the dump
0145     result = set()
0146     for element in elements:
0147       if element[0] == '-':
0148         result.add( element )
0149       else:
0150         result.discard( '-' + element )
0151     return sorted( element for element in result )
0152
0153   @staticmethod
0154   def consolidatePositiveList(elements):
0155     # consolidate a list of path selection and re-exclusions
0156     # the result is the list of paths to be included in the dump
0157     result = set()
0158     for element in elements:
0159       if element[0] == '-':
0160         result.discard( element[1:] )
0161       else:
0162         result.add( element )
0163     return sorted( element for element in result )
0164
0165
0166   # dump the final configuration
0167   def dump(self):
0168     self.data = self.data % self.labels
0169     if self.config.fragment:
0170       self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
0171       self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
0172     return self.data
0173
0174
0175   # add specific customizations
0176   def specificCustomize(self):
0177     # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
0178     if self.config.fragment:
0179       self.data += """
0180 # add specific customizations
0181 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0182 fragment = customizeHLTforAll(fragment,"%s")
0183 """ % (self.config.type)
0184     elif self.config.hilton:
0185       # do not apply the STORM-specific customisation
0186       pass
0187     else:
0188       if self.config.type=="Fake":
0189         prefix = "run1"
0190       elif self.config.type in ("Fake1","Fake2","2018"):
0191         prefix = "run2"
0192       else:
0193         prefix = "run3"
0194       _gtData = "auto:"+prefix+"_hlt_"+self.config.type
0195       _gtMc   = "auto:"+prefix+"_mc_" +self.config.type
0196       self.data += """
0197 # add specific customizations
0198 _customInfo = {}
0199 _customInfo['menuType'  ]= "%s"
0200 _customInfo['globalTags']= {}
0201 _customInfo['globalTags'][True ] = "%s"
0202 _customInfo['globalTags'][False] = "%s"
0203 _customInfo['inputFiles']={}
0204 _customInfo['inputFiles'][True]  = "file:RelVal_Raw_%s_DATA.root"
0205 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
0206 _customInfo['maxEvents' ]=  %s
0207 _customInfo['globalTag' ]= "%s"
0208 _customInfo['inputFile' ]=  %s
0209 _customInfo['realData'  ]=  %s
0210
0211 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0212 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
0213 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
0214
0215     self.data += """
0216 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
0217 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
0218 """ % (self.config.type)
0219
0220     # Eras-based customisations
0221     self.data += """
0222 # Eras-based customisations
0223 from HLTrigger.Configuration.Eras import modifyHLTforEras
0224 modifyHLTforEras(%(process)s)
0225 """
0226     # add the user-defined customization functions, if any
0227     if self.config.customise:
0228         self.data += "\n"
0229         self.data += "#User-defined customization functions\n"
0230         for customise in self.config.customise.split(","):
0231             customiseValues = customise.split(".")
0232             if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
0233             if len(customiseValues)==1:
0234                  customiseValues.append("customise")
0235             customiseValues[0] = customiseValues[0].replace("/",".")
0236             self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
0237             self.data += "process = "+customiseValues[1]+"(process)\n"
0238
0239
0240   # customize the configuration according to the options
0241   def customize(self):
0242
0243     # adapt the source to the current scenario
0244     if not self.config.fragment:
0245       self.build_source()
0246
0247     # manual override some parameters
0248     if self.config.type in ('HIon', ):
0249       if self.config.data:
0250         if not self.config.fragment:
0251           self._fix_parameter( type = 'InputTag', value = 'rawDataCollector',  replace = 'rawDataRepacker')
0252
0253     # if requested, remove the HLT prescales
0254     self.fixPrescales()
0255
0256     # if requested, override all ED/HLTfilters to always pass ("open" mode)
0257     self.instrumentOpenMode()
0258
0259     # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
0260     self.instrumentErrorEventType()
0261
0262     # if requested, instrument the self with the modules and EndPath needed for timing studies
0263     self.instrumentTiming()
0264
0265     # if requested, override the L1 self from the GlobalTag (Xml)
0266     self.overrideL1MenuXml()
0267
0268     # if requested, run the L1 emulator
0269     self.runL1Emulator()
0270
0271     # add process.load("setup_cff")
0272     self.loadSetupCff()
0273
0274     if self.config.fragment:
0275       self.data += """
0276 # dummify hltGetConditions in cff's
0277 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
0278     %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
0279         result = cms.bool( True )
0280     )
0281     %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
0282 """
0283
0284       # the scouting path issue:
0285       # 1) for config fragments, we remove all output modules
0286       # 2) however in old style datasets, the scouting output paths also run the unpackers which are needed
0287       # 3) therefore they have to keep the scouting path but remove the scouting output module
0288       # 4) in new style datasets, aka datasetpaths & finalpaths, the scouting unpackers are on another path and all of this is unnecessary
0289       # 5) however its hard to detect whether we have new style or old style so we run this for both
0290       # 6) therefore we end up with a superfluous Scouting*OutputPaths which are empty
0291       for path in self.all_paths:
0292         match = re.match(r'(Scouting\w+)Output$', path)
0293         if match:
0294           module = 'hltOutput' + match.group(1)
0295           self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
0296           self.data = self.data.replace(' + process.'+module, '')
0297           self.data = self.data.replace(' process.'+module, '')
0298     else:
0299
0300       # override the process name and adapt the relevant filters
0301       self.overrideProcessName()
0302
0303       # select specific Eras
0304       self.addEras()
0305
0306       # override the output modules to output root files
0307       self.overrideOutput()
0308
0309       # add global options
0310       self.addGlobalOptions()
0311
0312       # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
0313       self.overrideGlobalTag()
0314
0315       # request summary informations from the MessageLogger
0316       self.updateMessageLogger()
0317
0318       # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
0319       self.instrumentDQM()
0320
0321     # add specific customisations
0322     self.specificCustomize()
0323
0324
0325   def addGlobalOptions(self):
0326     # add global options
0327     self.data += """
0328 # limit the number of events to be processed
0329 %%(process)s.maxEvents = cms.untracked.PSet(
0330     input = cms.untracked.int32( %d )
0331 )
0332 """ % self.config.events
0333
0334     self.data += """
0335 # enable TrigReport, TimeReport and MultiThreading
0336 %(process)s.options.wantSummary = True
0337 %(process)s.options.numberOfThreads = 4
0338 %(process)s.options.numberOfStreams = 0
0339 """
0340
0341   def _fix_parameter(self, **args):
0342     """arguments:
0343         name:     parameter name (optional)
0344         type:     parameter type (look for tracked and untracked variants)
0345         value:    original value
0346         replace:  replacement value
0347     """
0348     if 'name' in args:
0349       self.data = re.sub(
0350           r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0351           r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0352           self.data)
0353     else:
0354       self.data = re.sub(
0355           r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0356           r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0357           self.data)
0358
0359
0360   def fixPrescales(self):
0361     # update the PrescaleService to match the new list of paths
0362     if self.options['paths']:
0363       if self.options['paths'][0][0] == '-':
0364         # drop requested paths
0365         for minuspath in self.options['paths']:
0366           path = minuspath[1:]
0367           self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
0368       else:
0369         # keep requested paths
0370         for path in self.all_paths:
0371           if path not in self.options['paths']:
0372             self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
0373
0374     if self.config.prescale and (self.config.prescale.lower() != 'none'):
0375       # TO DO: check that the requested prescale column is valid
0376       self.data += """
0377 # force the use of a specific HLT prescale column
0378 if 'PrescaleService' in %(dict)s:
0379     %(process)s.PrescaleService.forceDefault     = True
0380     %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
0381 """
0382
0383
0384   def instrumentOpenMode(self):
0385     if self.config.open:
0386       # find all EDfilters
0387       filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
0388       re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
0389       # remove existing 'cms.ignore' and '~' modifiers
0390       self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
0391       self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
0392       # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
0393       for some in splitter(filters, 1000):
0394         re_filters  = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
0395         self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
0396
0397
0398   def instrumentErrorEventType(self):
0399     if self.config.errortype:
0400       # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
0401       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
0402       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
0403       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
0404
0405
0406   def overrideGlobalTag(self):
0407     # overwrite GlobalTag
0408     # the logic is:
0409     #   - if a GlobalTag is specified on the command line:
0410     #      - override the global tag
0411     #      - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
0412     #   - if a GlobalTag is NOT  specified on the command line:
0413     #      - when running on data, do nothing, and keep the global tag in the menu
0414     #      - when running on mc, take the GT from the configuration.type
0415
0416     # override the GlobalTag connection string and pfnPrefix
0417
0418     # when running on MC, override the global tag even if not specified on the command line
0419     if not self.config.data and not self.config.globaltag:
0420       if self.config.type in globalTag:
0421         self.config.globaltag = globalTag[self.config.type]
0422       else:
0423         self.config.globaltag = globalTag['GRun']
0424
0425     # if requested, override the L1 menu from the GlobalTag
0426     if self.config.l1.override:
0427       self.config.l1.tag    = self.config.l1.override
0428       self.config.l1.record = 'L1TUtmTriggerMenuRcd'
0429       self.config.l1.connect = ''
0430       self.config.l1.label  = ''
0431       if not self.config.l1.snapshotTime:
0432         self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
0433       self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
0434     else:
0435       self.config.l1cond = None
0436
0437     if self.config.globaltag or self.config.l1cond:
0438       text = """
0439 # override the GlobalTag, connection string and pfnPrefix
0440 if 'GlobalTag' in %(dict)s:
0441     from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
0442     %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
0443       if self.config.globaltag:
0444         text += ", globaltag = %s"  % repr(self.config.globaltag)
0445       if self.config.l1cond:
0446         text += ", conditions = %s" % repr(self.config.l1cond)
0447       text += ")\n"
0448       self.data += text
0449
0450   def overrideL1MenuXml(self):
0451     # if requested, override the GlobalTag's L1T menu from an Xml file
0452     if self.config.l1Xml.XmlFile:
0453       text = """
0454 # override the GlobalTag's L1T menu from an Xml file
0455 from HLTrigger.Configuration.CustomConfigs import L1XML
0456 %%(process)s = L1XML(%%(process)s,"%s")
0457 """ % (self.config.l1Xml.XmlFile)
0458       self.data += text
0459
0460   def runL1Emulator(self):
0461     # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0462     if self.config.emulator:
0463       text = """
0464 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0465 from HLTrigger.Configuration.CustomConfigs import L1REPACK
0466 %%(process)s = L1REPACK(%%(process)s,"%s")
0467 """ % (self.config.emulator)
0468       self.data += text
0469
0470   def overrideOutput(self):
0471     # if not runnign on Hilton, override the "online" ShmStreamConsumer output modules with "offline" PoolOutputModule's
0472     # note for Run3 ShmStreamConsumer has been replaced with EvFOutputModule and later GlobalEvFOutputModule
0473     # so we also do a replace there
0474     if not self.config.hilton:
0475       self.data = re.sub(
0476         r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"(ShmStreamConsumer)" *,',
0477         r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),',
0478         self.data
0479       )
0480       self.data = re.sub(
0481         r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"EvFOutputModule" *,\n    use_compression = cms.untracked.bool\( True \),\n    compression_algorithm = cms.untracked.string\( "ZLIB" \),\n    compression_level = cms.untracked.int32\( 1 \),\n    lumiSection_interval = cms.untracked.int32\( 0 \),\n(.+?),\n    psetMap = cms.untracked.InputTag\( "hltPSetMap" \)\n',
0482         r'\1hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),\n\3\n',
0483         self.data,0,re.DOTALL
0484       )
0485       self.data = re.sub(
0486         r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"GlobalEvFOutputModule" *,\n    use_compression = cms.untracked.bool\( True \),\n    compression_algorithm = cms.untracked.string\( "ZLIB" \),\n    compression_level = cms.untracked.int32\( 1 \),\n    lumiSection_interval = cms.untracked.int32\( 0 \),\n(.+?),\n    psetMap = cms.untracked.InputTag\( "hltPSetMap" \)\n',
0487         r'\1hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),\n\3\n',
0488         self.data,0,re.DOTALL
0489       )
0490     if not self.config.fragment and self.config.output == 'minimal':
0491       # add a single output to keep the TriggerResults and TriggerEvent
0492       self.data += """
0493 # add a single "keep *" output
0494 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
0495     fileName = cms.untracked.string( "output.root" ),
0496     fastCloning = cms.untracked.bool( False ),
0497     dataset = cms.untracked.PSet(
0498         dataTier = cms.untracked.string( 'AOD' ),
0499         filterName = cms.untracked.string( '' )
0500     ),
0501     outputCommands = cms.untracked.vstring( 'drop *',
0502         'keep edmTriggerResults_*_*_*',
0503         'keep triggerTriggerEvent_*_*_*',
0504         'keep GlobalAlgBlkBXVector_*_*_*',
0505         'keep GlobalExtBlkBXVector_*_*_*',
0506         'keep l1tEGammaBXVector_*_EGamma_*',
0507         'keep l1tEtSumBXVector_*_EtSum_*',
0508         'keep l1tJetBXVector_*_Jet_*',
0509         'keep l1tMuonBXVector_*_Muon_*',
0510         'keep l1tTauBXVector_*_Tau_*',
0511     )
0512 )
0513 %(process)s.MinimalOutput = cms.FinalPath( %(process)s.hltOutputMinimal )
0514 %(process)s.schedule.append( %(process)s.MinimalOutput )
0515 """
0516     elif not self.config.fragment and self.config.output == 'full':
0517       # add a single "keep *" output
0518       self.data += """
0519 # add a single "keep *" output
0520 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
0521     fileName = cms.untracked.string( "output.root" ),
0522     fastCloning = cms.untracked.bool( False ),
0523     dataset = cms.untracked.PSet(
0524         dataTier = cms.untracked.string( 'RECO' ),
0525         filterName = cms.untracked.string( '' )
0526     ),
0527     outputCommands = cms.untracked.vstring( 'keep *' )
0528 )
0529 %(process)s.FullOutput = cms.FinalPath( %(process)s.hltOutputFull )
0530 %(process)s.schedule.append( %(process)s.FullOutput )
0531 """
0532
0533   # select specific Eras
0534   def addEras(self):
0535     if self.config.eras is None:
0536       return
0537     from Configuration.StandardSequences.Eras import eras
0538     erasSplit = self.config.eras.split(',')
0539     self.data = re.sub(r'process = cms.Process\( *"\w+"', '\n'.join(eras.pythonCfgLines[era] for era in erasSplit)+'\n\g<0>, '+', '.join(era for era in erasSplit), self.data)
0540
0541   # select specific Eras
0542   def loadSetupCff(self):
0543     if self.config.setup is None:
0544       return
0545     processLine = self.data.find("\n",self.data.find("cms.Process"))
0546     self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
0547
0548   # override the process name and adapt the relevant filters
0549   def overrideProcessName(self):
0550     if self.config.name is None:
0551       return
0552
0553     # sanitise process name
0554     self.config.name = self.config.name.replace("_","")
0555     # override the process name
0556     quote = '[\'\"]'
0557     self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
0558
0559     # when --setup option is used, remove possible errors from PrescaleService due to missing HLT paths.
0560     if self.config.setup: self.data += """
0561 # avoid PrescaleService error due to missing HLT paths
0562 if 'PrescaleService' in process.__dict__:
0563     for pset in reversed(process.PrescaleService.prescaleTable):
0564         if not hasattr(process,pset.pathName.value()):
0565             process.PrescaleService.prescaleTable.remove(pset)
0566 """
0567
0568
0569   def updateMessageLogger(self):
0570     # request summary informations from the MessageLogger
0571     self.data += """
0572 # show summaries from trigger analysers used at HLT
0573 if 'MessageLogger' in %(dict)s:
0574     %(process)s.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet()
0575     %(process)s.MessageLogger.L1GtTrigReport = cms.untracked.PSet()
0576     %(process)s.MessageLogger.L1TGlobalSummary = cms.untracked.PSet()
0577     %(process)s.MessageLogger.HLTrigReport = cms.untracked.PSet()
0578     %(process)s.MessageLogger.FastReport = cms.untracked.PSet()
0579     %(process)s.MessageLogger.ThroughputService = cms.untracked.PSet()
0580 """
0581
0582
0583   def loadAdditionalConditions(self, comment, *conditions):
0584     # load additional conditions
0585     self.data += """
0586 # %s
0587 if 'GlobalTag' in %%(dict)s:
0588 """ % comment
0589     for condition in conditions:
0590       self.data += """    %%(process)s.GlobalTag.toGet.append(
0591         cms.PSet(
0592             record  = cms.string( '%(record)s' ),
0593             tag     = cms.string( '%(tag)s' ),
0594             label   = cms.untracked.string( '%(label)s' ),
0595         )
0596     )
0597 """ % condition
0598
0599
0600   def loadCffCommand(self, module):
0601     # load a cfi or cff module
0602     if self.config.fragment:
0603       return 'from %s import *\n' % module
0604     else:
0605       return 'process.load( "%s" )\n' % module
0606
0607   def loadCff(self, module):
0608     self.data += self.loadCffCommand(module)
0609
0610
0611   def overrideParameters(self, module, parameters):
0612     # override a module's parameter if the module is present in the configuration
0613     self.data += "if '%s' in %%(dict)s:\n" % module
0614     for (parameter, value) in parameters:
0615       self.data += "    %%(process)s.%s.%s = %s\n" % (module, parameter, value)
0616     self.data += "\n"
0617
0618
0619   def removeElementFromSequencesTasksAndPaths(self, label):
0620     if label in self.data:
0621       label_re = r'\b(process\.)?' + label
0622       self.data = re.sub(r' *(\+|,) *' + label_re, '', self.data)
0623       self.data = re.sub(label_re + r' *(\+|,) *', '', self.data)
0624       self.data = re.sub(label_re, '', self.data)
0625
0626
0627   def instrumentTiming(self):
0628
0629     if self.config.timing:
0630       self.data += """
0631 # instrument the menu with the modules and EndPath needed for timing studies
0632 """
0633
0634       self.data += '\n# configure the FastTimerService\n'
0635       self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
0636
0637       self.data += """# print a text summary at the end of the job
0638 %(process)s.FastTimerService.printEventSummary         = False
0639 %(process)s.FastTimerService.printRunSummary           = False
0640 %(process)s.FastTimerService.printJobSummary           = True
0641
0642 # enable DQM plots
0643 %(process)s.FastTimerService.enableDQM                 = True
0644
0645 # enable per-path DQM plots (starting with CMSSW 9.2.3-patch2)
0646 %(process)s.FastTimerService.enableDQMbyPath           = True
0647
0648 # enable per-module DQM plots
0649 %(process)s.FastTimerService.enableDQMbyModule         = True
0650
0651 # enable per-event DQM plots vs lumisection
0652 %(process)s.FastTimerService.enableDQMbyLumiSection    = True
0653 %(process)s.FastTimerService.dqmLumiSectionsRange      = 2500
0654
0655 # set the time resolution of the DQM plots
0656 %(process)s.FastTimerService.dqmTimeRange              = 2000.
0657 %(process)s.FastTimerService.dqmTimeResolution         =   10.
0658 %(process)s.FastTimerService.dqmPathTimeRange          = 1000.
0659 %(process)s.FastTimerService.dqmPathTimeResolution     =    5.
0660 %(process)s.FastTimerService.dqmModuleTimeRange        =  200.
0661 %(process)s.FastTimerService.dqmModuleTimeResolution   =    1.
0662
0663 # set the base DQM folder for the plots
0664 %(process)s.FastTimerService.dqmPath                   = 'HLT/TimerService'
0665 %(process)s.FastTimerService.enableDQMbyProcesses      = False
0666 """
0667
0668
0669   def instrumentDQM(self):
0670     if not self.config.hilton:
0671       # remove any reference to the hltDQMFileSaver and hltDQMFileSaverPB:
0672       # note the convert options remove the module itself,
0673       # here we are just removing the references in paths, sequences, etc
0674       self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaverPB')
0675       self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaver')
0676
0677       # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
0678       dqmstore  = "\n# load the DQMStore and DQMRootOutputModule\n"
0679       dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
0680       dqmstore += """
0681 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
0682     fileName = cms.untracked.string("DQMIO.root")
0683 )
0684 """
0685       empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\( *\).*')
0686       other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\()(.*)')
0687       if empty_path.search(self.data):
0688         # replace an empty DQMOutput path
0689         self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n', self.data)
0690       elif other_path.search(self.data):
0691         # prepend the dqmOutput to the DQMOutput path
0692         self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<4>', self.data)
0693       else:
0694         # create a new DQMOutput path with the dqmOutput module
0695         self.data += dqmstore
0696         self.data += '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n'
0697         self.data += '%(process)s.schedule.append( %(process)s.DQMOutput )\n'
0698
0699
0700   @staticmethod
0701   def dumppaths(paths):
0702     sys.stderr.write('Path selection:\n')
0703     for path in paths:
0704       sys.stderr.write('\t%s\n' % path)
0705     sys.stderr.write('\n\n')
0706
0707   def buildPathList(self):
0708     self.all_paths = self.getPathList()
0709
0710     if self.config.paths:
0711       # no path list was requested, dump the full table, minus unsupported / unwanted paths
0712       paths = self.config.paths.split(',')
0713     else:
0714       # dump only the requested paths, plus the eventual output endpaths
0715       paths = []
0716
0717     # 'none'    should remove all outputs
0718     # 'dqm'     should remove all outputs but DQMHistograms
0719     # 'minimal' should remove all outputs but DQMHistograms, and add a single output module to keep the TriggerResults and TriggerEvent
0720     # 'full'    should remove all outputs but DQMHistograms, and add a single output module to "keep *"
0721     # See also the `overrideOutput` method
0722     if self.config.fragment or self.config.output in ('none', ):
0723       if self.config.paths:
0724         # keep only the Paths and EndPaths requested explicitly
0725         pass
0726       else:
0727         # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring and DQMHistograms
0728         paths.append( "-*Output" )
0729         paths.append( "-RatesMonitoring")
0730         paths.append( "-DQMHistograms")
0731         if self.config.fragment: paths.append( "Scouting*Output" )
0732
0733     elif self.config.output in ('dqm', 'minimal', 'full'):
0734       if self.config.paths:
0735         # keep only the Paths and EndPaths requested explicitly, and the DQMHistograms
0736         paths.append( "DQMHistograms" )
0737       else:
0738         # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring
0739         paths.append( "-*Output" )
0740         paths.append( "-RatesMonitoring")
0741         if self.config.fragment: paths.append( "Scouting*Output" )
0742
0743     else:
0744       if self.config.paths:
0745         # keep all output EndPaths, including the DQMHistograms
0746         paths.append( "*Output" )
0747         paths.append( "DQMHistograms" )
0748       else:
0749         # keep all Paths and EndPaths
0750         pass
0751
0752     # drop unwanted paths for profiling (and timing studies)
0753     if self.config.profiling:
0754       paths.append( "-HLTAnalyzerEndpath" )
0755
0756     # this should never be in any dump (nor online menu)
0757     paths.append( "-OfflineOutput" )
0758
0759     # expand all wildcards
0760     paths = self.expandWildcards(paths, self.all_paths)
0761
0762     if self.config.paths:
0763       # do an "additive" consolidation
0764       paths = self.consolidatePositiveList(paths)
0765       if not paths:
0766         raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
0767     else:
0768       # do a "subtractive" consolidation
0769       paths = self.consolidateNegativeList(paths)
0770     self.options['paths'] = paths
0771
0772   def buildOptions(self):
0773     # common configuration for all scenarios
0774     self.options['services'].append( "-DQM" )
0775     self.options['services'].append( "-FUShmDQMOutputService" )
0776     self.options['services'].append( "-MicroStateService" )
0777     self.options['services'].append( "-ModuleWebRegistry" )
0778     self.options['services'].append( "-TimeProfilerService" )
0779
0780     # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
0781     # unless a hilton-like configuration has been requested
0782     if not self.config.hilton:
0783       self.options['services'].append( "-EvFDaqDirector" )
0784       self.options['services'].append( "-FastMonitoringService" )
0785       self.options['services'].append( "-DQMStore" )
0786       self.options['modules'].append( "-hltDQMFileSaver" )
0787       self.options['modules'].append( "-hltDQMFileSaverPB" )
0788
0789     if self.config.fragment:
0790       # extract a configuration file fragment
0791       self.options['essources'].append( "-GlobalTag" )
0792       self.options['essources'].append( "-HepPDTESSource" )
0793       self.options['essources'].append( "-XMLIdealGeometryESSource" )
0794       self.options['essources'].append( "-eegeom" )
0795       self.options['essources'].append( "-es_hardcode" )
0796       self.options['essources'].append( "-magfield" )
0797
0798       self.options['esmodules'].append( "-SlaveField0" )
0799       self.options['esmodules'].append( "-SlaveField20" )
0800       self.options['esmodules'].append( "-SlaveField30" )
0801       self.options['esmodules'].append( "-SlaveField35" )
0802       self.options['esmodules'].append( "-SlaveField38" )
0803       self.options['esmodules'].append( "-SlaveField40" )
0804       self.options['esmodules'].append( "-VBF0" )
0805       self.options['esmodules'].append( "-VBF20" )
0806       self.options['esmodules'].append( "-VBF30" )
0807       self.options['esmodules'].append( "-VBF35" )
0808       self.options['esmodules'].append( "-VBF38" )
0809       self.options['esmodules'].append( "-VBF40" )
0810       self.options['esmodules'].append( "-CSCGeometryESModule" )
0811       self.options['esmodules'].append( "-CaloGeometryBuilder" )
0812       self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
0813       self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
0814       self.options['esmodules'].append( "-DTGeometryESModule" )
0815       self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
0816       self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
0817       self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
0818       self.options['esmodules'].append( "-EcalLaserCorrectionService" )
0819       self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
0820       self.options['esmodules'].append( "-GEMGeometryESModule" )
0821       self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
0822       self.options['esmodules'].append( "-HcalTopologyIdealEP" )
0823       self.options['esmodules'].append( "-MuonNumberingInitialization" )
0824       self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
0825       self.options['esmodules'].append( "-RPCGeometryESModule" )
0826       self.options['esmodules'].append( "-SiStripGainESProducer" )
0827       self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
0828       self.options['esmodules'].append( "-SiStripQualityESProducer" )
0829       self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
0830       self.options['esmodules'].append( "-TrackerAdditionalParametersPerDetESModule" )
0831       self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
0832       self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
0833       self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
0834       self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
0835       self.options['esmodules'].append( "-hcal_db_producer" )
0836       self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
0837       self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
0838       self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
0839       self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
0840       self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
0841       self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
0842       self.options['esmodules'].append( "-trackerTopology" )
0843
0844       self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
0845       self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
0846       self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
0847       self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
0848       self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
0849       self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
0850       self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
0851       self.options['esmodules'].append( "-XMLFromDBSource" )
0852       self.options['esmodules'].append( "-sistripconn" )
0853
0854       self.options['services'].append( "-MessageLogger" )
0855
0856       self.options['psets'].append( "-maxEvents" )
0857       self.options['psets'].append( "-options" )
0858
0859       # remove Scouting OutputModules even though the EndPaths are kept
0860       self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
0861       self.options['modules'].append( "-hltOutputScoutingPF" )
0862
0863     if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
0864       self.options['services'].append( "-PrescaleService" )
0865
0866     if self.config.fragment or self.config.timing:
0867       self.options['services'].append( "-FastTimerService" )
0868
0869
0870   def append_filenames(self, name, filenames):
0871     if len(filenames) > 255:
0872       token_open  = "( *("
0873       token_close = ") )"
0874     else:
0875       token_open  = "("
0876       token_close = ")"
0877
0878     self.data += "    %s = cms.untracked.vstring%s\n" % (name, token_open)
0879     for line in filenames:
0880       self.data += "        '%s',\n" % line
0881     self.data += "    %s,\n" % (token_close)
0882
0883
0884   def expand_filenames(self, input):
0885     # check if the input is a dataset or a list of files
0886     if input[0:8] == 'dataset:':
0887       from .dasFileQuery import dasFileQuery
0888       # extract the dataset name, and use DAS to fine the list of LFNs
0889       dataset = input[8:]
0890       files = dasFileQuery(dataset)
0891     else:
0892       # assume a comma-separated list of input files
0893       files = input.split(',')
0894     return files
0895
0896   def build_source(self):
0897     if self.config.hilton:
0898       # use the DAQ source
0899       return
0900
0901     if self.config.input:
0902       # if a dataset or a list of input files was given, use it
0903       self.source = self.expand_filenames(self.config.input)
0904     elif self.config.data:
0905       # offline we can run on data...
0906       self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
0907     else:
0908       # ...or on mc
0909       self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
0910
0911     if self.config.parent:
0912       # if a dataset or a list of input files was given for the parent data, use it
0913       self.parent = self.expand_filenames(self.config.parent)
0914
0915     self.data += """
0916 # source module (EDM inputs)
0917 %(process)s.source = cms.Source( "PoolSource",
0918 """
0919     self.append_filenames("fileNames", self.source)
0920     if (self.parent):
0921       self.append_filenames("secondaryFileNames", self.parent)
0922     self.data += """\
0923     inputCommands = cms.untracked.vstring(
0924         'keep *'
0925     )
0926 )
0927 """