python/Tools/confdb.py

0001 #!/usr/bin/env python
0002
0003 from __future__ import absolute_import
0004 import sys
0005 import re
0006 import os
0007 from .pipe import pipe as _pipe
0008 from .options import globalTag
0009 from itertools import islice
0010
0011 def splitter(iterator, n):
0012   i = iterator.__iter__()
0013   while True:
0014     l = list(islice(i, n))
0015     if l:
0016       yield l
0017     else:
0018       break
0019
0020
0021 class HLTProcess(object):
0022
0023   def __init__(self, configuration):
0024     self.config = configuration
0025     self.data   = None
0026     self.source = []
0027     self.parent = []
0028
0029     self.options = {
0030       'essources' : [],
0031       'esmodules' : [],
0032       'modules'   : [],
0033       'sequences' : [],
0034       'services'  : [],
0035       'paths'     : [],
0036       'psets'     : [],
0037       'blocks'    : [],
0038     }
0039
0040     self.labels = {}
0041     if self.config.fragment:
0042       self.labels['process'] = 'fragment'
0043       self.labels['dict']    = 'fragment.__dict__'
0044     else:
0045       self.labels['process'] = 'process'
0046       self.labels['dict']    = 'process.__dict__'
0047
0048     if self.config.prescale and (self.config.prescale.lower() != 'none'):
0049       self.labels['prescale'] = self.config.prescale
0050
0051     # get the configuration from ConfdB
0052     from .confdbOfflineConverter import OfflineConverter
0053     self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port, tunnel = self.config.tunnel, tunnelPort = self.config.tunnel_port)
0054     self.buildPathList()
0055     self.buildOptions()
0056     self.getSetupConfigurationFromDB()
0057     self.getRawConfigurationFromDB()
0058     self.customize()
0059
0060   def getSetupConfigurationFromDB(self):
0061     if not self.config.setup:
0062         return
0063     ## if --setup is a python file, use directly that file as setup_cff.py
0064     if ".py" in self.config.setup:
0065         self.config.setupFile = self.config.setup.split(".py")[0]
0066         return
0067     args = ['--configName', self.config.setup ]
0068     args.append('--noedsources')
0069     args.append('--nopaths')
0070     for key, vals in self.options.items():
0071       if vals:
0072         args.extend(('--'+key, ','.join(vals)))
0073     args.append('--cff')
0074     data, err = self.converter.query( *args )
0075     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0076         sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
0077         sys.stderr.write(err + "\n\n")
0078         sys.exit(1)
0079     self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
0080     outfile = open(self.config.setupFile+".py","w+")
0081     outfile.write("# This file is automatically generated by hltGetConfiguration.\n" + data)
0082
0083   def getRawConfigurationFromDB(self):
0084     if self.config.menu.run:
0085       args = ['--runNumber', self.config.menu.run]
0086     else:
0087       args = ['--configName', self.config.menu.name ]
0088     if not self.config.hilton:
0089         # keep the original Source when running on Hilton
0090         args.append('--noedsources')
0091     for key, vals in self.options.items():
0092       if vals:
0093         args.extend(('--'+key, ','.join(vals)))
0094     data, err = self.converter.query( *args )
0095     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0096         sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0097         sys.stderr.write(err + "\n\n")
0098         sys.exit(1)
0099     self.data = data
0100
0101   def getPathList(self):
0102     if self.config.menu.run:
0103       args = ['--runNumber', self.config.menu.run]
0104     else:
0105       args = ['--configName', self.config.menu.name]
0106     args.extend( (
0107       '--cff',
0108       '--noedsources',
0109       '--noes',
0110       '--noservices',
0111       '--nosequences',
0112       '--nomodules'
0113     ) )
0114
0115     data, err = self.converter.query( *args )
0116     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0117         sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0118         sys.stderr.write(err + "\n\n")
0119         sys.exit(1)
0120     filter = re.compile(r' *= *cms.(End|Final)?Path.*')
0121     paths  = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
0122     return paths
0123
0124
0125   @staticmethod
0126   def expandWildcards(globs, collection):
0127     # expand a list of unix-style wildcards matching a given collection
0128     # wildcards with no matches are silently discarded
0129     matches = []
0130     for glob in globs:
0131       negate = ''
0132       if glob[0] == '-':
0133         negate = '-'
0134         glob   = glob[1:]
0135       # translate a unix-style glob expression into a regular expression
0136       filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
0137       matches.extend( negate + element for element in collection if filter.match(element) )
0138     return matches
0139
0140
0141   @staticmethod
0142   def consolidateNegativeList(elements):
0143     # consolidate a list of path exclusions and re-inclusions
0144     # the result is the list of paths to be removed from the dump
0145     result = set()
0146     for element in elements:
0147       if element[0] == '-':
0148         result.add( element )
0149       else:
0150         result.discard( '-' + element )
0151     return sorted( element for element in result )
0152
0153   @staticmethod
0154   def consolidatePositiveList(elements):
0155     # consolidate a list of path selection and re-exclusions
0156     # the result is the list of paths to be included in the dump
0157     result = set()
0158     for element in elements:
0159       if element[0] == '-':
0160         result.discard( element[1:] )
0161       else:
0162         result.add( element )
0163     return sorted( element for element in result )
0164
0165
0166   # dump the final configuration
0167   def dump(self):
0168     self.data = self.data % self.labels
0169     if self.config.fragment:
0170       self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
0171       self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
0172     return self.data
0173
0174
0175   # add specific customizations
0176   def specificCustomize(self):
0177     # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
0178     if self.config.fragment:
0179       self.data += """
0180 # add specific customizations
0181 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0182 fragment = customizeHLTforAll(fragment,"%s")
0183 """ % (self.config.type)
0184     elif self.config.hilton:
0185       # do not apply the STORM-specific customisation
0186       pass
0187     else:
0188       if self.config.type=="Fake":
0189         prefix = "run1"
0190       elif self.config.type in ("Fake1","Fake2","2018"):
0191         prefix = "run2"
0192       else:
0193         prefix = "run3"
0194       _gtData = "auto:"+prefix+"_hlt_"+self.config.type
0195       _gtMc   = "auto:"+prefix+"_mc_" +self.config.type
0196       self.data += """
0197 # add specific customizations
0198 _customInfo = {}
0199 _customInfo['menuType'  ]= "%s"
0200 _customInfo['globalTags']= {}
0201 _customInfo['globalTags'][True ] = "%s"
0202 _customInfo['globalTags'][False] = "%s"
0203 _customInfo['inputFiles']={}
0204 _customInfo['inputFiles'][True]  = "file:RelVal_Raw_%s_DATA.root"
0205 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
0206 _customInfo['maxEvents' ]=  %s
0207 _customInfo['globalTag' ]= "%s"
0208 _customInfo['inputFile' ]=  %s
0209 _customInfo['realData'  ]=  %s
0210
0211 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0212 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
0213 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
0214
0215     self.data += """
0216 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
0217 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
0218 """ % (self.config.type)
0219
0220     # Eras-based customisations
0221     self.data += """
0222 # Eras-based customisations
0223 from HLTrigger.Configuration.Eras import modifyHLTforEras
0224 modifyHLTforEras(%(process)s)
0225 """
0226     # add the user-defined customization functions, if any
0227     if self.config.customise:
0228         self.data += "\n"
0229         self.data += "#User-defined customization functions\n"
0230         for customise in self.config.customise.split(","):
0231             customiseValues = customise.split(".")
0232             if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
0233             if len(customiseValues)==1:
0234                  customiseValues.append("customise")
0235             customiseValues[0] = customiseValues[0].replace("/",".")
0236             self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
0237             self.data += "process = "+customiseValues[1]+"(process)\n"
0238
0239
0240   # customize the configuration according to the options
0241   def customize(self):
0242
0243     # adapt the source to the current scenario
0244     if not self.config.fragment:
0245       self.build_source()
0246
0247     # if requested, remove the HLT prescales
0248     self.fixPrescales()
0249
0250     # if requested, override all ED/HLTfilters to always pass ("open" mode)
0251     self.instrumentOpenMode()
0252
0253     # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
0254     self.instrumentErrorEventType()
0255
0256     # if requested, instrument the self with the modules and EndPath needed for timing studies
0257     self.instrumentTiming()
0258
0259     # if requested, override the L1 self from the GlobalTag (Xml)
0260     self.overrideL1MenuXml()
0261
0262     # if requested, run the L1 emulator
0263     self.runL1Emulator()
0264
0265     # add process.load("setup_cff")
0266     self.loadSetupCff()
0267
0268     if self.config.fragment:
0269       self.data += """
0270 # dummify hltGetConditions in cff's
0271 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
0272     %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
0273         result = cms.bool( True )
0274     )
0275     %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
0276 """
0277
0278       # the scouting path issue:
0279       # 1) for config fragments, we remove all output modules
0280       # 2) however in old style datasets, the scouting output paths also run the unpackers which are needed
0281       # 3) therefore they have to keep the scouting path but remove the scouting output module
0282       # 4) in new style datasets, aka datasetpaths & finalpaths, the scouting unpackers are on another path and all of this is unnecessary
0283       # 5) however its hard to detect whether we have new style or old style so we run this for both
0284       # 6) therefore we end up with a superfluous Scouting*OutputPaths which are empty
0285       for path in self.all_paths:
0286         match = re.match(r'(Scouting\w+)Output$', path)
0287         if match:
0288           module = 'hltOutput' + match.group(1)
0289           self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
0290           self.data = self.data.replace(' + process.'+module, '')
0291           self.data = self.data.replace(' process.'+module, '')
0292     else:
0293
0294       # override the process name and adapt the relevant filters
0295       self.overrideProcessName()
0296
0297       # select specific Eras
0298       self.addEras()
0299
0300       # override the output modules to output root files
0301       self.overrideOutput()
0302
0303       # add global options
0304       self.addGlobalOptions()
0305
0306       # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
0307       self.overrideGlobalTag()
0308
0309       # request summary informations from the MessageLogger
0310       self.updateMessageLogger()
0311
0312       # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
0313       self.instrumentDQM()
0314
0315     # add specific customisations
0316     self.specificCustomize()
0317
0318
0319   def addGlobalOptions(self):
0320     # add global options
0321     self.data += """
0322 # limit the number of events to be processed
0323 %%(process)s.maxEvents = cms.untracked.PSet(
0324     input = cms.untracked.int32( %d )
0325 )
0326 """ % self.config.events
0327
0328     self.data += """
0329 # enable TrigReport, TimeReport and MultiThreading
0330 %(process)s.options.wantSummary = True
0331 %(process)s.options.numberOfThreads = 4
0332 %(process)s.options.numberOfStreams = 0
0333 """
0334
0335   def _fix_parameter(self, **args):
0336     """arguments:
0337         name:     parameter name (optional)
0338         type:     parameter type (look for tracked and untracked variants)
0339         value:    original value
0340         replace:  replacement value
0341     """
0342     if 'name' in args:
0343       self.data = re.sub(
0344           r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0345           r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0346           self.data)
0347     else:
0348       self.data = re.sub(
0349           r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0350           r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0351           self.data)
0352
0353
0354   def fixPrescales(self):
0355     # update the PrescaleService to match the new list of paths
0356     if self.options['paths']:
0357       if self.options['paths'][0][0] == '-':
0358         # drop requested paths
0359         for minuspath in self.options['paths']:
0360           path = minuspath[1:]
0361           self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
0362       else:
0363         # keep requested paths
0364         for path in self.all_paths:
0365           if path not in self.options['paths']:
0366             self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
0367
0368     if self.config.prescale and (self.config.prescale.lower() != 'none'):
0369       # TO DO: check that the requested prescale column is valid
0370       self.data += """
0371 # force the use of a specific HLT prescale column
0372 if 'PrescaleService' in %(dict)s:
0373     %(process)s.PrescaleService.forceDefault     = True
0374     %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
0375 """
0376
0377
0378   def instrumentOpenMode(self):
0379     if self.config.open:
0380       # find all EDfilters
0381       filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
0382       re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
0383       # remove existing 'cms.ignore' and '~' modifiers
0384       self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
0385       self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
0386       # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
0387       for some in splitter(filters, 1000):
0388         re_filters  = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
0389         self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
0390
0391
0392   def instrumentErrorEventType(self):
0393     if self.config.errortype:
0394       # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
0395       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
0396       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
0397       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
0398
0399
0400   def overrideGlobalTag(self):
0401     # overwrite GlobalTag
0402     # the logic is:
0403     #   - if a GlobalTag is specified on the command line:
0404     #      - override the global tag
0405     #      - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
0406     #   - if a GlobalTag is NOT  specified on the command line:
0407     #      - when running on data, do nothing, and keep the global tag in the menu
0408     #      - when running on mc, take the GT from the configuration.type
0409
0410     # override the GlobalTag connection string and pfnPrefix
0411
0412     # when running on MC, override the global tag even if not specified on the command line
0413     if not self.config.data and not self.config.globaltag:
0414       if self.config.type in globalTag:
0415         self.config.globaltag = globalTag[self.config.type]
0416       else:
0417         self.config.globaltag = globalTag['GRun']
0418
0419     # if requested, override the L1 menu from the GlobalTag
0420     if self.config.l1.override:
0421       self.config.l1.tag    = self.config.l1.override
0422       self.config.l1.record = 'L1TUtmTriggerMenuRcd'
0423       self.config.l1.connect = ''
0424       self.config.l1.label  = ''
0425       if not self.config.l1.snapshotTime:
0426         self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
0427       self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
0428     else:
0429       self.config.l1cond = None
0430
0431     if self.config.globaltag or self.config.l1cond:
0432       text = """
0433 # override the GlobalTag, connection string and pfnPrefix
0434 if 'GlobalTag' in %(dict)s:
0435     from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
0436     %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
0437       if self.config.globaltag:
0438         text += ", globaltag = %s"  % repr(self.config.globaltag)
0439       if self.config.l1cond:
0440         text += ", conditions = %s" % repr(self.config.l1cond)
0441       text += ")\n"
0442       self.data += text
0443
0444   def overrideL1MenuXml(self):
0445     # if requested, override the GlobalTag's L1T menu from an Xml file
0446     if self.config.l1Xml.XmlFile:
0447       text = """
0448 # override the GlobalTag's L1T menu from an Xml file
0449 from HLTrigger.Configuration.CustomConfigs import L1XML
0450 %%(process)s = L1XML(%%(process)s,"%s")
0451 """ % (self.config.l1Xml.XmlFile)
0452       self.data += text
0453
0454   def runL1Emulator(self):
0455     # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0456     if self.config.emulator:
0457       text = """
0458 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0459 from HLTrigger.Configuration.CustomConfigs import L1REPACK
0460 %%(process)s = L1REPACK(%%(process)s,"%s")
0461 """ % (self.config.emulator)
0462       self.data += text
0463
0464   def overrideOutput(self):
0465     # if not running on Hilton, override the "online" output modules with the "offline" one (i.e. PoolOutputModule)
0466     # in Run 1 and Run 2, the online output modules were instances of ShmStreamConsumer
0467     # in Run 3, ShmStreamConsumer has been replaced with EvFOutputModule, and later GlobalEvFOutputModule
0468     if not self.config.hilton:
0469       self.data = re.sub(
0470         r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"(ShmStreamConsumer)" *,',
0471         r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),',
0472         self.data
0473       )
0474
0475       self.data = re.sub("""\
0476 \\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *['"](EvFOutputModule|GlobalEvFOutputModule)['"] *,
0477     use_compression = cms.untracked.bool\( (True|False) \),
0478     compression_algorithm = cms.untracked.string\( ['"](.+?)['"] \),
0479     compression_level = cms.untracked.int32\( (-?\d+) \),
0480     lumiSection_interval = cms.untracked.int32\( (-?\d+) \),
0481 (.+?),
0482     psetMap = cms.untracked.InputTag\( ['"]hltPSetMap['"] \)
0483 ""","""\
0484 %(process)s.hltOutput\g<2> = cms.OutputModule( "PoolOutputModule",
0485     fileName = cms.untracked.string( "output\g<2>.root" ),
0486     compressionAlgorithm = cms.untracked.string( "\g<5>" ),
0487     compressionLevel = cms.untracked.int32( \g<6> ),
0488     fastCloning = cms.untracked.bool( False ),
0489     dataset = cms.untracked.PSet(
0490         filterName = cms.untracked.string( "" ),
0491         dataTier = cms.untracked.string( "RAW" )
0492     ),
0493 \g<8>
0494 """, self.data, 0, re.DOTALL)
0495
0496     if not self.config.fragment and self.config.output == 'minimal':
0497       # add a single output to keep the TriggerResults and TriggerEvent
0498       self.data += """
0499 # add a single "keep *" output
0500 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
0501     fileName = cms.untracked.string( "output.root" ),
0502     fastCloning = cms.untracked.bool( False ),
0503     dataset = cms.untracked.PSet(
0504         dataTier = cms.untracked.string( 'AOD' ),
0505         filterName = cms.untracked.string( '' )
0506     ),
0507     outputCommands = cms.untracked.vstring( 'drop *',
0508         'keep edmTriggerResults_*_*_*',
0509         'keep triggerTriggerEvent_*_*_*',
0510         'keep GlobalAlgBlkBXVector_*_*_*',
0511         'keep GlobalExtBlkBXVector_*_*_*',
0512         'keep l1tEGammaBXVector_*_EGamma_*',
0513         'keep l1tEtSumBXVector_*_EtSum_*',
0514         'keep l1tJetBXVector_*_Jet_*',
0515         'keep l1tMuonBXVector_*_Muon_*',
0516         'keep l1tTauBXVector_*_Tau_*',
0517     )
0518 )
0519 %(process)s.MinimalOutput = cms.FinalPath( %(process)s.hltOutputMinimal )
0520 %(process)s.schedule.append( %(process)s.MinimalOutput )
0521 """
0522     elif not self.config.fragment and self.config.output == 'full':
0523       # add a single "keep *" output
0524       self.data += """
0525 # add a single "keep *" output
0526 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
0527     fileName = cms.untracked.string( "output.root" ),
0528     fastCloning = cms.untracked.bool( False ),
0529     dataset = cms.untracked.PSet(
0530         dataTier = cms.untracked.string( 'RECO' ),
0531         filterName = cms.untracked.string( '' )
0532     ),
0533     outputCommands = cms.untracked.vstring( 'keep *' )
0534 )
0535 %(process)s.FullOutput = cms.FinalPath( %(process)s.hltOutputFull )
0536 %(process)s.schedule.append( %(process)s.FullOutput )
0537 """
0538
0539   # select specific Eras
0540   def addEras(self):
0541     if self.config.eras is None:
0542       return
0543     from Configuration.StandardSequences.Eras import eras
0544     erasSplit = self.config.eras.split(',')
0545     self.data = re.sub(r'process = cms.Process\( *"\w+"', '\n'.join(eras.pythonCfgLines[era] for era in erasSplit)+'\n\g<0>, '+', '.join(era for era in erasSplit), self.data)
0546
0547   # select specific Eras
0548   def loadSetupCff(self):
0549     if self.config.setup is None:
0550       return
0551     processLine = self.data.find("\n",self.data.find("cms.Process"))
0552     self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
0553
0554   # override the process name and adapt the relevant filters
0555   def overrideProcessName(self):
0556     if self.config.name is None:
0557       return
0558
0559     # sanitise process name
0560     self.config.name = self.config.name.replace("_","")
0561     # override the process name
0562     quote = '[\'\"]'
0563     self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
0564
0565     # when --setup option is used, remove possible errors from PrescaleService due to missing HLT paths.
0566     if self.config.setup: self.data += """
0567 # avoid PrescaleService error due to missing HLT paths
0568 if 'PrescaleService' in process.__dict__:
0569     for pset in reversed(process.PrescaleService.prescaleTable):
0570         if not hasattr(process,pset.pathName.value()):
0571             process.PrescaleService.prescaleTable.remove(pset)
0572 """
0573
0574
0575   def updateMessageLogger(self):
0576     # request summary informations from the MessageLogger
0577     self.data += """
0578 # show summaries from trigger analysers used at HLT
0579 if 'MessageLogger' in %(dict)s:
0580     %(process)s.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet()
0581     %(process)s.MessageLogger.L1GtTrigReport = cms.untracked.PSet()
0582     %(process)s.MessageLogger.L1TGlobalSummary = cms.untracked.PSet()
0583     %(process)s.MessageLogger.HLTrigReport = cms.untracked.PSet()
0584     %(process)s.MessageLogger.FastReport = cms.untracked.PSet()
0585     %(process)s.MessageLogger.ThroughputService = cms.untracked.PSet()
0586 """
0587
0588
0589   def loadAdditionalConditions(self, comment, *conditions):
0590     # load additional conditions
0591     self.data += """
0592 # %s
0593 if 'GlobalTag' in %%(dict)s:
0594 """ % comment
0595     for condition in conditions:
0596       self.data += """    %%(process)s.GlobalTag.toGet.append(
0597         cms.PSet(
0598             record  = cms.string( '%(record)s' ),
0599             tag     = cms.string( '%(tag)s' ),
0600             label   = cms.untracked.string( '%(label)s' ),
0601         )
0602     )
0603 """ % condition
0604
0605
0606   def loadCffCommand(self, module):
0607     # load a cfi or cff module
0608     if self.config.fragment:
0609       return 'from %s import *\n' % module
0610     else:
0611       return 'process.load( "%s" )\n' % module
0612
0613   def loadCff(self, module):
0614     self.data += self.loadCffCommand(module)
0615
0616
0617   def overrideParameters(self, module, parameters):
0618     # override a module's parameter if the module is present in the configuration
0619     self.data += "if '%s' in %%(dict)s:\n" % module
0620     for (parameter, value) in parameters:
0621       self.data += "    %%(process)s.%s.%s = %s\n" % (module, parameter, value)
0622     self.data += "\n"
0623
0624
0625   def removeElementFromSequencesTasksAndPaths(self, label):
0626     if label in self.data:
0627       label_re = r'\b(process\.)?' + label
0628       self.data = re.sub(r' *(\+|,) *' + label_re, '', self.data)
0629       self.data = re.sub(label_re + r' *(\+|,) *', '', self.data)
0630       self.data = re.sub(label_re, '', self.data)
0631
0632
0633   def instrumentTiming(self):
0634
0635     if self.config.timing:
0636       self.data += """
0637 # instrument the menu with the modules and EndPath needed for timing studies
0638 """
0639
0640       self.data += '\n# configure the FastTimerService\n'
0641       self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
0642
0643       self.data += """# print a text summary at the end of the job
0644 %(process)s.FastTimerService.printEventSummary         = False
0645 %(process)s.FastTimerService.printRunSummary           = False
0646 %(process)s.FastTimerService.printJobSummary           = True
0647
0648 # enable DQM plots
0649 %(process)s.FastTimerService.enableDQM                 = True
0650
0651 # enable per-path DQM plots
0652 %(process)s.FastTimerService.enableDQMbyPath           = True
0653
0654 # enable per-module DQM plots
0655 %(process)s.FastTimerService.enableDQMbyModule         = True
0656
0657 # enable per-event DQM plots vs lumisection
0658 %(process)s.FastTimerService.enableDQMbyLumiSection    = True
0659 %(process)s.FastTimerService.dqmLumiSectionsRange      = 2500
0660
0661 # set the time resolution of the DQM plots
0662 %(process)s.FastTimerService.dqmTimeRange              = 2000.
0663 %(process)s.FastTimerService.dqmTimeResolution         =   10.
0664 %(process)s.FastTimerService.dqmPathTimeRange          = 1000.
0665 %(process)s.FastTimerService.dqmPathTimeResolution     =    5.
0666 %(process)s.FastTimerService.dqmModuleTimeRange        =  200.
0667 %(process)s.FastTimerService.dqmModuleTimeResolution   =    1.
0668
0669 # set the base DQM folder for the DQM plots
0670 %(process)s.FastTimerService.dqmPath                   = 'HLT/TimerService'
0671 %(process)s.FastTimerService.enableDQMbyProcesses      = False
0672
0673 # write a JSON file with the information to be displayed in a pie chart
0674 %(process)s.FastTimerService.writeJSONSummary          = True
0675 %(process)s.FastTimerService.jsonFileName              = 'resources.json'
0676 """
0677
0678       self.data += '\n# configure the ThroughputService\n'
0679       self.loadCff('HLTrigger.Timer.ThroughputService_cfi')
0680
0681       self.data += """# enable DQM plots
0682 %(process)s.ThroughputService.enableDQM                = True
0683
0684 # set the resolution of the DQM plots
0685 %(process)s.ThroughputService.eventRange               = 10000
0686 %(process)s.ThroughputService.eventResolution          = 1
0687 %(process)s.ThroughputService.timeRange                = 60000
0688 %(process)s.ThroughputService.timeResolution           = 10
0689
0690 # set the base DQM folder for the DQM plots
0691 %(process)s.ThroughputService.dqmPath                  = 'HLT/Throughput'
0692 %(process)s.ThroughputService.dqmPathByProcesses       = False
0693 """
0694
0695
0696   def instrumentDQM(self):
0697     if not self.config.hilton:
0698       # remove any reference to the hltDQMFileSaver and hltDQMFileSaverPB:
0699       # note the convert options remove the module itself,
0700       # here we are just removing the references in paths, sequences, etc
0701       self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaverPB')
0702       self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaver')
0703
0704       # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
0705       dqmstore  = "\n# load the DQMStore and DQMRootOutputModule\n"
0706       dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
0707       dqmstore += """
0708 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
0709     fileName = cms.untracked.string("DQMIO.root")
0710 )
0711 """
0712       empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\( *\).*')
0713       other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\()(.*)')
0714       if empty_path.search(self.data):
0715         # replace an empty DQMOutput path
0716         self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n', self.data)
0717       elif other_path.search(self.data):
0718         # prepend the dqmOutput to the DQMOutput path
0719         self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<4>', self.data)
0720       else:
0721         # create a new DQMOutput path with the dqmOutput module
0722         self.data += dqmstore
0723         self.data += '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n'
0724         self.data += '%(process)s.schedule.append( %(process)s.DQMOutput )\n'
0725
0726
0727   @staticmethod
0728   def dumppaths(paths):
0729     sys.stderr.write('Path selection:\n')
0730     for path in paths:
0731       sys.stderr.write('\t%s\n' % path)
0732     sys.stderr.write('\n\n')
0733
0734   def buildPathList(self):
0735     self.all_paths = self.getPathList()
0736
0737     if self.config.paths:
0738       # no path list was requested, dump the full table, minus unsupported / unwanted paths
0739       paths = self.config.paths.split(',')
0740     else:
0741       # dump only the requested paths, plus the eventual output endpaths
0742       paths = []
0743
0744     # 'none'    should remove all outputs
0745     # 'dqm'     should remove all outputs but DQMHistograms
0746     # 'minimal' should remove all outputs but DQMHistograms, and add a single output module to keep the TriggerResults and TriggerEvent
0747     # 'full'    should remove all outputs but DQMHistograms, and add a single output module to "keep *"
0748     # See also the `overrideOutput` method
0749     if self.config.fragment or self.config.output in ('none', ):
0750       if self.config.paths:
0751         # keep only the Paths and EndPaths requested explicitly
0752         pass
0753       else:
0754         # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring and DQMHistograms
0755         paths.append( "-*Output" )
0756         paths.append( "-RatesMonitoring")
0757         paths.append( "-DQMHistograms")
0758         if self.config.fragment: paths.append( "Scouting*Output" )
0759
0760     elif self.config.output in ('dqm', 'minimal', 'full'):
0761       if self.config.paths:
0762         # keep only the Paths and EndPaths requested explicitly, and the DQMHistograms
0763         paths.append( "DQMHistograms" )
0764       else:
0765         # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring
0766         paths.append( "-*Output" )
0767         paths.append( "-RatesMonitoring")
0768         if self.config.fragment: paths.append( "Scouting*Output" )
0769
0770     else:
0771       if self.config.paths:
0772         # keep all output EndPaths, including the DQMHistograms
0773         paths.append( "*Output" )
0774         paths.append( "DQMHistograms" )
0775       else:
0776         # keep all Paths and EndPaths
0777         pass
0778
0779     # drop unwanted paths for profiling (and timing studies)
0780     if self.config.profiling:
0781       paths.append( "-HLTAnalyzerEndpath" )
0782
0783     # this should never be in any dump (nor online menu)
0784     paths.append( "-OfflineOutput" )
0785
0786     # expand all wildcards
0787     paths = self.expandWildcards(paths, self.all_paths)
0788
0789     if self.config.paths:
0790       # do an "additive" consolidation
0791       paths = self.consolidatePositiveList(paths)
0792       if not paths:
0793         raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
0794     else:
0795       # do a "subtractive" consolidation
0796       paths = self.consolidateNegativeList(paths)
0797     self.options['paths'] = paths
0798
0799   def buildOptions(self):
0800     # common configuration for all scenarios
0801     self.options['services'].append( "-DQM" )
0802     self.options['services'].append( "-FUShmDQMOutputService" )
0803     self.options['services'].append( "-MicroStateService" )
0804     self.options['services'].append( "-ModuleWebRegistry" )
0805     self.options['services'].append( "-TimeProfilerService" )
0806
0807     # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
0808     # unless a hilton-like configuration has been requested
0809     if not self.config.hilton:
0810       self.options['services'].append( "-EvFDaqDirector" )
0811       self.options['services'].append( "-FastMonitoringService" )
0812       self.options['services'].append( "-DQMStore" )
0813       self.options['modules'].append( "-hltDQMFileSaver" )
0814       self.options['modules'].append( "-hltDQMFileSaverPB" )
0815
0816     if self.config.fragment:
0817       # extract a configuration file fragment
0818       self.options['essources'].append( "-GlobalTag" )
0819       self.options['essources'].append( "-HepPDTESSource" )
0820       self.options['essources'].append( "-XMLIdealGeometryESSource" )
0821       self.options['essources'].append( "-eegeom" )
0822       self.options['essources'].append( "-es_hardcode" )
0823       self.options['essources'].append( "-magfield" )
0824
0825       self.options['esmodules'].append( "-SlaveField0" )
0826       self.options['esmodules'].append( "-SlaveField20" )
0827       self.options['esmodules'].append( "-SlaveField30" )
0828       self.options['esmodules'].append( "-SlaveField35" )
0829       self.options['esmodules'].append( "-SlaveField38" )
0830       self.options['esmodules'].append( "-SlaveField40" )
0831       self.options['esmodules'].append( "-VBF0" )
0832       self.options['esmodules'].append( "-VBF20" )
0833       self.options['esmodules'].append( "-VBF30" )
0834       self.options['esmodules'].append( "-VBF35" )
0835       self.options['esmodules'].append( "-VBF38" )
0836       self.options['esmodules'].append( "-VBF40" )
0837       self.options['esmodules'].append( "-CSCGeometryESModule" )
0838       self.options['esmodules'].append( "-CaloGeometryBuilder" )
0839       self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
0840       self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
0841       self.options['esmodules'].append( "-DTGeometryESModule" )
0842       self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
0843       self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
0844       self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
0845       self.options['esmodules'].append( "-EcalLaserCorrectionService" )
0846       self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
0847       self.options['esmodules'].append( "-GEMGeometryESModule" )
0848       self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
0849       self.options['esmodules'].append( "-HcalTopologyIdealEP" )
0850       self.options['esmodules'].append( "-MuonNumberingInitialization" )
0851       self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
0852       self.options['esmodules'].append( "-RPCGeometryESModule" )
0853       self.options['esmodules'].append( "-SiStripGainESProducer" )
0854       self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
0855       self.options['esmodules'].append( "-SiStripQualityESProducer" )
0856       self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
0857       self.options['esmodules'].append( "-TrackerAdditionalParametersPerDetESModule" )
0858       self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
0859       self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
0860       self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
0861       self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
0862       self.options['esmodules'].append( "-hcal_db_producer" )
0863       self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
0864       self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
0865       self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
0866       self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
0867       self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
0868       self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
0869       self.options['esmodules'].append( "-trackerTopology" )
0870
0871       self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
0872       self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
0873       self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
0874       self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
0875       self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
0876       self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
0877       self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
0878       self.options['esmodules'].append( "-XMLFromDBSource" )
0879       self.options['esmodules'].append( "-sistripconn" )
0880       self.options['esmodules'].append( "-siPixelQualityESProducer" )
0881
0882       self.options['services'].append( "-MessageLogger" )
0883
0884       self.options['psets'].append( "-maxEvents" )
0885       self.options['psets'].append( "-options" )
0886
0887       # remove Scouting OutputModules even though the EndPaths are kept
0888       self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
0889       self.options['modules'].append( "-hltOutputScoutingPF" )
0890
0891     if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
0892       self.options['services'].append( "-PrescaleService" )
0893
0894     if self.config.fragment or self.config.timing:
0895       self.options['services'].append( "-FastTimerService" )
0896       self.options['services'].append( "-ThroughputService" )
0897
0898
0899   def append_filenames(self, name, filenames):
0900     if len(filenames) > 255:
0901       token_open  = "( *("
0902       token_close = ") )"
0903     else:
0904       token_open  = "("
0905       token_close = ")"
0906
0907     self.data += "    %s = cms.untracked.vstring%s\n" % (name, token_open)
0908     for line in filenames:
0909       self.data += "        '%s',\n" % line
0910     self.data += "    %s,\n" % (token_close)
0911
0912
0913   def expand_filenames(self, input):
0914     # check if the input is a dataset or a list of files
0915     if input[0:8] == 'dataset:':
0916       from .dasFileQuery import dasFileQuery
0917       # extract the dataset name, and use DAS to fine the list of LFNs
0918       dataset = input[8:]
0919       files = dasFileQuery(dataset)
0920     else:
0921       # assume a comma-separated list of input files
0922       files = input.split(',')
0923     return files
0924
0925   def build_source(self):
0926     if self.config.hilton:
0927       # use the DAQ source
0928       return
0929
0930     if self.config.input:
0931       # if a dataset or a list of input files was given, use it
0932       self.source = self.expand_filenames(self.config.input)
0933     elif self.config.data:
0934       # offline we can run on data...
0935       self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
0936     else:
0937       # ...or on mc
0938       self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
0939
0940     if self.config.parent:
0941       # if a dataset or a list of input files was given for the parent data, use it
0942       self.parent = self.expand_filenames(self.config.parent)
0943
0944     self.data += """
0945 # source module (EDM inputs)
0946 %(process)s.source = cms.Source( "PoolSource",
0947 """
0948     self.append_filenames("fileNames", self.source)
0949     if (self.parent):
0950       self.append_filenames("secondaryFileNames", self.parent)
0951     self.data += """\
0952     inputCommands = cms.untracked.vstring(
0953         'keep *'
0954     )
0955 )
0956 """