python/Tools/confdb.py

0001 #!/usr/bin/env python
0002
0003 import sys
0004 import re
0005 import os
0006 from .pipe import pipe as _pipe
0007 from .options import globalTag
0008 from itertools import islice
0009
0010 def splitter(iterator, n):
0011   i = iterator.__iter__()
0012   while True:
0013     l = list(islice(i, n))
0014     if l:
0015       yield l
0016     else:
0017       break
0018
0019
0020 class HLTProcess(object):
0021
0022   def __init__(self, configuration):
0023     self.config = configuration
0024     self.data   = None
0025     self.source = []
0026     self.parent = []
0027
0028     self.options = {
0029       'essources' : [],
0030       'esmodules' : [],
0031       'modules'   : [],
0032       'sequences' : [],
0033       'services'  : [],
0034       'paths'     : [],
0035       'psets'     : [],
0036       'blocks'    : [],
0037     }
0038
0039     self.labels = {}
0040     if self.config.fragment:
0041       self.labels['process'] = 'fragment'
0042       self.labels['dict']    = 'fragment.__dict__'
0043     else:
0044       self.labels['process'] = 'process'
0045       self.labels['dict']    = 'process.__dict__'
0046
0047     if self.config.prescale and (self.config.prescale.lower() != 'none'):
0048       self.labels['prescale'] = self.config.prescale
0049
0050     # get the configuration from ConfdB
0051     from .confdbOfflineConverter import OfflineConverter
0052     self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port, tunnel = self.config.tunnel, tunnelPort = self.config.tunnel_port)
0053     self.buildPathList()
0054     self.buildOptions()
0055     self.getSetupConfigurationFromDB()
0056     self.getRawConfigurationFromDB()
0057     self.customize()
0058
0059   def getSetupConfigurationFromDB(self):
0060     if not self.config.setup:
0061         return
0062     ## if --setup is a python file, use directly that file as setup_cff.py
0063     if ".py" in self.config.setup:
0064         self.config.setupFile = self.config.setup.split(".py")[0]
0065         return
0066     args = ['--configName', self.config.setup ]
0067     args.append('--noedsources')
0068     args.append('--nopaths')
0069     for key, vals in self.options.items():
0070       if vals:
0071         args.extend(('--'+key, ','.join(vals)))
0072     args.append('--cff')
0073     data, err = self.converter.query( *args )
0074     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0075         sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
0076         sys.stderr.write(err + "\n\n")
0077         sys.exit(1)
0078     self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
0079     outfile = open(self.config.setupFile+".py","w+")
0080     outfile.write("# This file is automatically generated by hltGetConfiguration.\n" + data)
0081
0082   def getRawConfigurationFromDB(self):
0083     if self.config.menu.run:
0084       args = ['--runNumber', self.config.menu.run]
0085     else:
0086       args = ['--configName', self.config.menu.name ]
0087     if not self.config.hilton:
0088         # keep the original Source when running on Hilton
0089         args.append('--noedsources')
0090     for key, vals in self.options.items():
0091       if vals:
0092         args.extend(('--'+key, ','.join(vals)))
0093     data, err = self.converter.query( *args )
0094     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0095         sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0096         sys.stderr.write(err + "\n\n")
0097         sys.exit(1)
0098     self.data = data
0099
0100   def getPathList(self):
0101     if self.config.menu.run:
0102       args = ['--runNumber', self.config.menu.run]
0103     else:
0104       args = ['--configName', self.config.menu.name]
0105     args.extend( (
0106       '--cff',
0107       '--noedsources',
0108       '--noes',
0109       '--noservices',
0110       '--nosequences',
0111       '--nomodules'
0112     ) )
0113
0114     data, err = self.converter.query( *args )
0115     if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
0116         sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
0117         sys.stderr.write(err + "\n\n")
0118         sys.exit(1)
0119     filter = re.compile(r' *= *cms.(End|Final)?Path.*')
0120     paths  = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
0121     return paths
0122
0123
0124   @staticmethod
0125   def expandWildcards(globs, collection):
0126     # expand a list of unix-style wildcards matching a given collection
0127     # wildcards with no matches are silently discarded
0128     matches = []
0129     for glob in globs:
0130       negate = ''
0131       if glob[0] == '-':
0132         negate = '-'
0133         glob   = glob[1:]
0134       # translate a unix-style glob expression into a regular expression
0135       filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
0136       matches.extend( negate + element for element in collection if filter.match(element) )
0137     return matches
0138
0139
0140   @staticmethod
0141   def consolidateNegativeList(elements):
0142     # consolidate a list of path exclusions and re-inclusions
0143     # the result is the list of paths to be removed from the dump
0144     result = set()
0145     for element in elements:
0146       if element[0] == '-':
0147         result.add( element )
0148       else:
0149         result.discard( '-' + element )
0150     return sorted( element for element in result )
0151
0152   @staticmethod
0153   def consolidatePositiveList(elements):
0154     # consolidate a list of path selection and re-exclusions
0155     # the result is the list of paths to be included in the dump
0156     result = set()
0157     for element in elements:
0158       if element[0] == '-':
0159         result.discard( element[1:] )
0160       else:
0161         result.add( element )
0162     return sorted( element for element in result )
0163
0164
0165   # dump the final configuration
0166   def dump(self):
0167     self.data = self.data % self.labels
0168     if self.config.fragment:
0169       self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
0170       self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
0171     return self.data
0172
0173
0174   # add specific customizations
0175   def specificCustomize(self):
0176     # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
0177     if self.config.fragment:
0178       self.data += """
0179 # add specific customizations
0180 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0181 fragment = customizeHLTforAll(fragment,"%s")
0182 """ % (self.config.type)
0183     elif self.config.hilton:
0184       # do not apply the STORM-specific customisation
0185       pass
0186     else:
0187       if self.config.type=="Fake":
0188         prefix = "run1"
0189       elif self.config.type in ("Fake1","Fake2","2018"):
0190         prefix = "run2"
0191       else:
0192         prefix = "run3"
0193       _gtData = "auto:"+prefix+"_hlt_"+self.config.type
0194       _gtMc   = "auto:"+prefix+"_mc_" +self.config.type
0195       self.data += """
0196 # add specific customizations
0197 _customInfo = {}
0198 _customInfo['menuType'  ]= "%s"
0199 _customInfo['globalTags']= {}
0200 _customInfo['globalTags'][True ] = "%s"
0201 _customInfo['globalTags'][False] = "%s"
0202 _customInfo['inputFiles']={}
0203 _customInfo['inputFiles'][True]  = "file:RelVal_Raw_%s_DATA.root"
0204 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
0205 _customInfo['maxEvents' ]=  %s
0206 _customInfo['globalTag' ]= "%s"
0207 _customInfo['inputFile' ]=  %s
0208 _customInfo['realData'  ]=  %s
0209
0210 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
0211 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
0212 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
0213
0214     self.data += """
0215 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
0216 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
0217 """ % (self.config.type)
0218
0219     # Eras-based customisations
0220     self.data += """
0221 # Eras-based customisations
0222 from HLTrigger.Configuration.Eras import modifyHLTforEras
0223 modifyHLTforEras(%(process)s)
0224 """
0225     # add the user-defined customization functions, if any
0226     if self.config.customise:
0227         self.data += "\n"
0228         self.data += "#User-defined customization functions\n"
0229         for customise in self.config.customise.split(","):
0230             customiseValues = customise.split(".")
0231             if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
0232             if len(customiseValues)==1:
0233                  customiseValues.append("customise")
0234             customiseValues[0] = customiseValues[0].replace("/",".")
0235             self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
0236             self.data += "process = "+customiseValues[1]+"(process)\n"
0237
0238
0239   # customize the configuration according to the options
0240   def customize(self):
0241
0242     # adapt the source to the current scenario
0243     if not self.config.fragment:
0244       self.build_source()
0245
0246     # if requested, remove the HLT prescales
0247     self.fixPrescales()
0248
0249     # if requested, override all ED/HLTfilters to always pass ("open" mode)
0250     self.instrumentOpenMode()
0251
0252     # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
0253     self.instrumentErrorEventType()
0254
0255     # if requested, instrument the self with the modules and EndPath needed for timing studies
0256     self.instrumentTiming()
0257
0258     # if requested, override the L1 self from the GlobalTag (Xml)
0259     self.overrideL1MenuXml()
0260
0261     # if requested, run the L1 emulator
0262     self.runL1Emulator()
0263
0264     # add process.load("setup_cff")
0265     self.loadSetupCff()
0266
0267     if self.config.fragment:
0268       self.data += """
0269 # dummify hltGetConditions in cff's
0270 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
0271     %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
0272         result = cms.bool( True )
0273     )
0274     %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
0275 """
0276
0277       # the scouting path issue:
0278       # 1) for config fragments, we remove all output modules
0279       # 2) however in old style datasets, the scouting output paths also run the unpackers which are needed
0280       # 3) therefore they have to keep the scouting path but remove the scouting output module
0281       # 4) in new style datasets, aka datasetpaths & finalpaths, the scouting unpackers are on another path and all of this is unnecessary
0282       # 5) however its hard to detect whether we have new style or old style so we run this for both
0283       # 6) therefore we end up with a superfluous Scouting*OutputPaths which are empty
0284       for path in self.all_paths:
0285         match = re.match(r'(Scouting\w+)Output$', path)
0286         if match:
0287           module = 'hltOutput' + match.group(1)
0288           self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
0289           self.data = self.data.replace(' + process.'+module, '')
0290           self.data = self.data.replace(' process.'+module, '')
0291     else:
0292
0293       # override the process name and adapt the relevant filters
0294       self.overrideProcessName()
0295
0296       # select specific Eras
0297       self.addEras()
0298
0299       # override the output modules to output root files
0300       self.overrideOutput()
0301
0302       # add global options
0303       self.addGlobalOptions()
0304
0305       # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
0306       self.overrideGlobalTag()
0307
0308       # request summary informations from the MessageLogger
0309       self.updateMessageLogger()
0310
0311       # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
0312       self.instrumentDQM()
0313
0314     # add specific customisations
0315     self.specificCustomize()
0316
0317
0318   def addGlobalOptions(self):
0319     # add global options
0320     self.data += """
0321 # limit the number of events to be processed
0322 %%(process)s.maxEvents = cms.untracked.PSet(
0323     input = cms.untracked.int32( %d )
0324 )
0325 """ % self.config.events
0326
0327     self.data += """
0328 # enable TrigReport, TimeReport and MultiThreading
0329 %(process)s.options.wantSummary = True
0330 %(process)s.options.numberOfThreads = 4
0331 %(process)s.options.numberOfStreams = 0
0332 """
0333
0334   def _fix_parameter(self, **args):
0335     """arguments:
0336         name:     parameter name (optional)
0337         type:     parameter type (look for tracked and untracked variants)
0338         value:    original value
0339         replace:  replacement value
0340     """
0341     if 'name' in args:
0342       self.data = re.sub(
0343           r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0344           r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0345           self.data)
0346     else:
0347       self.data = re.sub(
0348           r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
0349           r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
0350           self.data)
0351
0352
0353   def fixPrescales(self):
0354     # update the PrescaleService to match the new list of paths
0355     if self.options['paths']:
0356       if self.options['paths'][0][0] == '-':
0357         # drop requested paths
0358         for minuspath in self.options['paths']:
0359           path = minuspath[1:]
0360           self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
0361       else:
0362         # keep requested paths
0363         for path in self.all_paths:
0364           if path not in self.options['paths']:
0365             self.data = re.sub(r'      cms.PSet\(  pathName = cms.string\( "%s" \),\n        prescales = cms.vuint32\( .* \)\n      \),?\n' % path, '', self.data)
0366
0367     if self.config.prescale and (self.config.prescale.lower() != 'none'):
0368       # TO DO: check that the requested prescale column is valid
0369       self.data += """
0370 # force the use of a specific HLT prescale column
0371 if 'PrescaleService' in %(dict)s:
0372     %(process)s.PrescaleService.forceDefault     = True
0373     %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
0374 """
0375
0376
0377   def instrumentOpenMode(self):
0378     if self.config.open:
0379       # find all EDfilters
0380       filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
0381       re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
0382       # remove existing 'cms.ignore' and '~' modifiers
0383       self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
0384       self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
0385       # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
0386       for some in splitter(filters, 1000):
0387         re_filters  = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
0388         self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
0389
0390
0391   def instrumentErrorEventType(self):
0392     if self.config.errortype:
0393       # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
0394       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
0395       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
0396       self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
0397
0398
0399   def overrideGlobalTag(self):
0400     # overwrite GlobalTag
0401     # the logic is:
0402     #   - if a GlobalTag is specified on the command line:
0403     #      - override the global tag
0404     #      - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
0405     #   - if a GlobalTag is NOT  specified on the command line:
0406     #      - when running on data, do nothing, and keep the global tag in the menu
0407     #      - when running on mc, take the GT from the configuration.type
0408
0409     # override the GlobalTag connection string and pfnPrefix
0410
0411     # when running on MC, override the global tag even if not specified on the command line
0412     if not self.config.data and not self.config.globaltag:
0413       if self.config.type in globalTag:
0414         self.config.globaltag = globalTag[self.config.type]
0415       else:
0416         self.config.globaltag = globalTag['GRun']
0417
0418     # if requested, override the L1 menu from the GlobalTag
0419     if self.config.l1.override:
0420       self.config.l1.tag    = self.config.l1.override
0421       self.config.l1.record = 'L1TUtmTriggerMenuRcd'
0422       self.config.l1.connect = ''
0423       self.config.l1.label  = ''
0424       if not self.config.l1.snapshotTime:
0425         self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
0426       self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
0427     else:
0428       self.config.l1cond = None
0429
0430     if self.config.globaltag or self.config.l1cond:
0431       text = """
0432 # override the GlobalTag, connection string and pfnPrefix
0433 if 'GlobalTag' in %(dict)s:
0434     from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
0435     %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
0436       if self.config.globaltag:
0437         text += ", globaltag = %s"  % repr(self.config.globaltag)
0438       if self.config.l1cond:
0439         text += ", conditions = %s" % repr(self.config.l1cond)
0440       text += ")\n"
0441       self.data += text
0442
0443   def overrideL1MenuXml(self):
0444     # if requested, override the GlobalTag's L1T menu from an Xml file
0445     if self.config.l1Xml.XmlFile:
0446       text = """
0447 # override the GlobalTag's L1T menu from an Xml file
0448 from HLTrigger.Configuration.CustomConfigs import L1XML
0449 %%(process)s = L1XML(%%(process)s,"%s")
0450 """ % (self.config.l1Xml.XmlFile)
0451       self.data += text
0452
0453   def runL1Emulator(self):
0454     # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0455     if self.config.emulator:
0456       text = """
0457 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
0458 from HLTrigger.Configuration.CustomConfigs import L1REPACK
0459 %%(process)s = L1REPACK(%%(process)s,"%s")
0460 """ % (self.config.emulator)
0461       self.data += text
0462
0463   def overrideOutput(self):
0464     # if not running on Hilton, override the "online" output modules with the "offline" one (i.e. PoolOutputModule)
0465     # in Run 1 and Run 2, the online output modules were instances of ShmStreamConsumer
0466     # in Run 3, ShmStreamConsumer has been replaced with EvFOutputModule, and later GlobalEvFOutputModule
0467     if not self.config.hilton:
0468       self.data = re.sub(
0469         r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"(ShmStreamConsumer)" *,',
0470         r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n    fileName = cms.untracked.string( "output\2.root" ),\n    fastCloning = cms.untracked.bool( False ),\n    dataset = cms.untracked.PSet(\n        filterName = cms.untracked.string( "" ),\n        dataTier = cms.untracked.string( "RAW" )\n    ),',
0471         self.data
0472       )
0473
0474       self.data = re.sub("""\
0475 \\b(process\\.)?hltOutput(\\w+) *= *cms\\.OutputModule\\( *['"](EvFOutputModule|GlobalEvFOutputModule)['"] *,
0476     use_compression = cms.untracked.bool\\( (True|False) \\),
0477     compression_algorithm = cms.untracked.string\\( ['"](.+?)['"] \\),
0478     compression_level = cms.untracked.int32\\( (-?\\d+) \\),
0479     lumiSection_interval = cms.untracked.int32\\( (-?\\d+) \\),
0480 (.+?),
0481     psetMap = cms.untracked.InputTag\\( ['"]hltPSetMap['"] \\)
0482 ""","""\
0483 %(process)s.hltOutput\\g<2> = cms.OutputModule( "PoolOutputModule",
0484     fileName = cms.untracked.string( "output\\g<2>.root" ),
0485     compressionAlgorithm = cms.untracked.string( "\\g<5>" ),
0486     compressionLevel = cms.untracked.int32( \\g<6> ),
0487     fastCloning = cms.untracked.bool( False ),
0488     dataset = cms.untracked.PSet(
0489         filterName = cms.untracked.string( "" ),
0490         dataTier = cms.untracked.string( "RAW" )
0491     ),
0492 \\g<8>
0493 """, self.data, 0, re.DOTALL)
0494
0495     if not self.config.fragment and self.config.output == 'minimal':
0496       # add a single output to keep the TriggerResults and TriggerEvent
0497       self.data += """
0498 # add a single "keep *" output
0499 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
0500     fileName = cms.untracked.string( "output.root" ),
0501     fastCloning = cms.untracked.bool( False ),
0502     dataset = cms.untracked.PSet(
0503         dataTier = cms.untracked.string( 'AOD' ),
0504         filterName = cms.untracked.string( '' )
0505     ),
0506     outputCommands = cms.untracked.vstring( 'drop *',
0507         'keep edmTriggerResults_*_*_*',
0508         'keep triggerTriggerEvent_*_*_*',
0509         'keep GlobalAlgBlkBXVector_*_*_*',
0510         'keep GlobalExtBlkBXVector_*_*_*',
0511         'keep l1tEGammaBXVector_*_EGamma_*',
0512         'keep l1tEtSumBXVector_*_EtSum_*',
0513         'keep l1tJetBXVector_*_Jet_*',
0514         'keep l1tMuonBXVector_*_Muon_*',
0515         'keep l1tTauBXVector_*_Tau_*',
0516     )
0517 )
0518 %(process)s.MinimalOutput = cms.FinalPath( %(process)s.hltOutputMinimal )
0519 %(process)s.schedule.append( %(process)s.MinimalOutput )
0520 """
0521     elif not self.config.fragment and self.config.output == 'full':
0522       # add a single "keep *" output
0523       self.data += """
0524 # add a single "keep *" output
0525 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
0526     fileName = cms.untracked.string( "output.root" ),
0527     fastCloning = cms.untracked.bool( False ),
0528     dataset = cms.untracked.PSet(
0529         dataTier = cms.untracked.string( 'RECO' ),
0530         filterName = cms.untracked.string( '' )
0531     ),
0532     outputCommands = cms.untracked.vstring( 'keep *' )
0533 )
0534 %(process)s.FullOutput = cms.FinalPath( %(process)s.hltOutputFull )
0535 %(process)s.schedule.append( %(process)s.FullOutput )
0536 """
0537
0538   # select specific Eras
0539   def addEras(self):
0540     if self.config.eras is None:
0541       return
0542     from Configuration.StandardSequences.Eras import eras
0543     erasSplit = self.config.eras.split(',')
0544     self.data = re.sub(r'process = cms.Process\( *"\w+"', '\n'.join(eras.pythonCfgLines[era] for era in erasSplit)+'\n\\g<0>, '+', '.join(era for era in erasSplit), self.data)
0545
0546   # select specific Eras
0547   def loadSetupCff(self):
0548     if self.config.setup is None:
0549       return
0550     processLine = self.data.find("\n",self.data.find("cms.Process"))
0551     self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
0552
0553   # override the process name and adapt the relevant filters
0554   def overrideProcessName(self):
0555     if self.config.name is None:
0556       return
0557
0558     # sanitise process name
0559     self.config.name = self.config.name.replace("_","")
0560     # override the process name
0561     quote = '[\'\"]'
0562     self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
0563
0564     # when --setup option is used, remove possible errors from PrescaleService due to missing HLT paths.
0565     if self.config.setup: self.data += """
0566 # avoid PrescaleService error due to missing HLT paths
0567 if 'PrescaleService' in process.__dict__:
0568     for pset in reversed(process.PrescaleService.prescaleTable):
0569         if not hasattr(process,pset.pathName.value()):
0570             process.PrescaleService.prescaleTable.remove(pset)
0571 """
0572
0573
0574   def updateMessageLogger(self):
0575     # request summary informations from the MessageLogger
0576     self.data += """
0577 # show summaries from trigger analysers used at HLT
0578 if 'MessageLogger' in %(dict)s:
0579     %(process)s.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet()
0580     %(process)s.MessageLogger.L1GtTrigReport = cms.untracked.PSet()
0581     %(process)s.MessageLogger.L1TGlobalSummary = cms.untracked.PSet()
0582     %(process)s.MessageLogger.HLTrigReport = cms.untracked.PSet()
0583     %(process)s.MessageLogger.FastReport = cms.untracked.PSet()
0584     %(process)s.MessageLogger.ThroughputService = cms.untracked.PSet()
0585 """
0586
0587
0588   def loadAdditionalConditions(self, comment, *conditions):
0589     # load additional conditions
0590     self.data += """
0591 # %s
0592 if 'GlobalTag' in %%(dict)s:
0593 """ % comment
0594     for condition in conditions:
0595       self.data += """    %%(process)s.GlobalTag.toGet.append(
0596         cms.PSet(
0597             record  = cms.string( '%(record)s' ),
0598             tag     = cms.string( '%(tag)s' ),
0599             label   = cms.untracked.string( '%(label)s' ),
0600         )
0601     )
0602 """ % condition
0603
0604
0605   def loadCffCommand(self, module):
0606     # load a cfi or cff module
0607     if self.config.fragment:
0608       return 'from %s import *\n' % module
0609     else:
0610       return 'process.load( "%s" )\n' % module
0611
0612   def loadCff(self, module):
0613     self.data += self.loadCffCommand(module)
0614
0615
0616   def overrideParameters(self, module, parameters):
0617     # override a module's parameter if the module is present in the configuration
0618     self.data += "if '%s' in %%(dict)s:\n" % module
0619     for (parameter, value) in parameters:
0620       self.data += "    %%(process)s.%s.%s = %s\n" % (module, parameter, value)
0621     self.data += "\n"
0622
0623
0624   def removeElementFromSequencesTasksAndPaths(self, label):
0625     if label in self.data:
0626       label_re = r'\b(process\.)?' + label
0627       self.data = re.sub(r' *(\+|,) *' + label_re, '', self.data)
0628       self.data = re.sub(label_re + r' *(\+|,) *', '', self.data)
0629       self.data = re.sub(label_re, '', self.data)
0630
0631
0632   def instrumentTiming(self):
0633
0634     if self.config.timing:
0635       self.data += """
0636 # instrument the menu with the modules and EndPath needed for timing studies
0637 """
0638
0639       self.data += '\n# configure the FastTimerService\n'
0640       self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
0641
0642       self.data += """# print a text summary at the end of the job
0643 %(process)s.FastTimerService.printEventSummary         = False
0644 %(process)s.FastTimerService.printRunSummary           = False
0645 %(process)s.FastTimerService.printJobSummary           = True
0646
0647 # enable DQM plots
0648 %(process)s.FastTimerService.enableDQM                 = True
0649
0650 # enable per-path DQM plots
0651 %(process)s.FastTimerService.enableDQMbyPath           = True
0652
0653 # enable per-module DQM plots
0654 %(process)s.FastTimerService.enableDQMbyModule         = True
0655
0656 # enable per-event DQM plots vs lumisection
0657 %(process)s.FastTimerService.enableDQMbyLumiSection    = True
0658 %(process)s.FastTimerService.dqmLumiSectionsRange      = 2500
0659
0660 # set the time resolution of the DQM plots
0661 %(process)s.FastTimerService.dqmTimeRange              = 2000.
0662 %(process)s.FastTimerService.dqmTimeResolution         =   10.
0663 %(process)s.FastTimerService.dqmPathTimeRange          = 1000.
0664 %(process)s.FastTimerService.dqmPathTimeResolution     =    5.
0665 %(process)s.FastTimerService.dqmModuleTimeRange        =  200.
0666 %(process)s.FastTimerService.dqmModuleTimeResolution   =    1.
0667
0668 # set the base DQM folder for the DQM plots
0669 %(process)s.FastTimerService.dqmPath                   = 'HLT/TimerService'
0670 %(process)s.FastTimerService.enableDQMbyProcesses      = False
0671
0672 # write a JSON file with the information to be displayed in a pie chart
0673 %(process)s.FastTimerService.writeJSONSummary          = True
0674 %(process)s.FastTimerService.jsonFileName              = 'resources.json'
0675 """
0676
0677       self.data += '\n# configure the ThroughputService\n'
0678       self.loadCff('HLTrigger.Timer.ThroughputService_cfi')
0679
0680       self.data += """# enable DQM plots
0681 %(process)s.ThroughputService.enableDQM                = True
0682
0683 # set the resolution of the DQM plots
0684 %(process)s.ThroughputService.eventRange               = 10000
0685 %(process)s.ThroughputService.eventResolution          = 1
0686 %(process)s.ThroughputService.timeRange                = 60000
0687 %(process)s.ThroughputService.timeResolution           = 10
0688
0689 # set the base DQM folder for the DQM plots
0690 %(process)s.ThroughputService.dqmPath                  = 'HLT/Throughput'
0691 %(process)s.ThroughputService.dqmPathByProcesses       = False
0692 """
0693
0694
0695   def instrumentDQM(self):
0696     if not self.config.hilton:
0697       # remove any reference to the hltDQMFileSaver and hltDQMFileSaverPB:
0698       # note the convert options remove the module itself,
0699       # here we are just removing the references in paths, sequences, etc
0700       self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaverPB')
0701       self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaver')
0702
0703       # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
0704       dqmstore  = "\n# load the DQMStore and DQMRootOutputModule\n"
0705       dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
0706       dqmstore += """
0707 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
0708     fileName = cms.untracked.string("DQMIO.root")
0709 )
0710 """
0711       empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\( *\).*')
0712       other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.(Final|End)Path\()(.*)')
0713       if empty_path.search(self.data):
0714         # replace an empty DQMOutput path
0715         self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n', self.data)
0716       elif other_path.search(self.data):
0717         # prepend the dqmOutput to the DQMOutput path
0718         self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<4>', self.data)
0719       else:
0720         # create a new DQMOutput path with the dqmOutput module
0721         self.data += dqmstore
0722         self.data += '\n%(process)s.DQMOutput = cms.FinalPath( %(process)s.dqmOutput )\n'
0723         self.data += '%(process)s.schedule.append( %(process)s.DQMOutput )\n'
0724
0725
0726   @staticmethod
0727   def dumppaths(paths):
0728     sys.stderr.write('Path selection:\n')
0729     for path in paths:
0730       sys.stderr.write('\t%s\n' % path)
0731     sys.stderr.write('\n\n')
0732
0733   def buildPathList(self):
0734     self.all_paths = self.getPathList()
0735
0736     if self.config.paths:
0737       # no path list was requested, dump the full table, minus unsupported / unwanted paths
0738       paths = self.config.paths.split(',')
0739     else:
0740       # dump only the requested paths, plus the eventual output endpaths
0741       paths = []
0742
0743     # 'none'    should remove all outputs
0744     # 'dqm'     should remove all outputs but DQMHistograms
0745     # 'minimal' should remove all outputs but DQMHistograms, and add a single output module to keep the TriggerResults and TriggerEvent
0746     # 'full'    should remove all outputs but DQMHistograms, and add a single output module to "keep *"
0747     # See also the `overrideOutput` method
0748     if self.config.fragment or self.config.output in ('none', ):
0749       if self.config.paths:
0750         # keep only the Paths and EndPaths requested explicitly
0751         pass
0752       else:
0753         # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring and DQMHistograms
0754         paths.append( "-*Output" )
0755         paths.append( "-RatesMonitoring")
0756         paths.append( "-DQMHistograms")
0757         if self.config.fragment: paths.append( "Scouting*Output" )
0758
0759     elif self.config.output in ('dqm', 'minimal', 'full'):
0760       if self.config.paths:
0761         # keep only the Paths and EndPaths requested explicitly, and the DQMHistograms
0762         paths.append( "DQMHistograms" )
0763       else:
0764         # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring
0765         paths.append( "-*Output" )
0766         paths.append( "-RatesMonitoring")
0767         if self.config.fragment: paths.append( "Scouting*Output" )
0768
0769     else:
0770       if self.config.paths:
0771         # keep all output EndPaths, including the DQMHistograms
0772         paths.append( "*Output" )
0773         paths.append( "DQMHistograms" )
0774       else:
0775         # keep all Paths and EndPaths
0776         pass
0777
0778     # drop unwanted paths for profiling (and timing studies)
0779     if self.config.profiling:
0780       paths.append( "-HLTAnalyzerEndpath" )
0781
0782     # this should never be in any dump (nor online menu)
0783     paths.append( "-OfflineOutput" )
0784
0785     # expand all wildcards
0786     paths = self.expandWildcards(paths, self.all_paths)
0787
0788     if self.config.paths:
0789       # do an "additive" consolidation
0790       paths = self.consolidatePositiveList(paths)
0791       if not paths:
0792         raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
0793     else:
0794       # do a "subtractive" consolidation
0795       paths = self.consolidateNegativeList(paths)
0796     self.options['paths'] = paths
0797
0798   def buildOptions(self):
0799     # common configuration for all scenarios
0800     self.options['services'].append( "-DQM" )
0801     self.options['services'].append( "-FUShmDQMOutputService" )
0802     self.options['services'].append( "-MicroStateService" )
0803     self.options['services'].append( "-ModuleWebRegistry" )
0804     self.options['services'].append( "-TimeProfilerService" )
0805
0806     # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
0807     # unless a hilton-like configuration has been requested
0808     if not self.config.hilton:
0809       self.options['services'].append( "-EvFDaqDirector" )
0810       self.options['services'].append( "-FastMonitoringService" )
0811       self.options['services'].append( "-DQMStore" )
0812       self.options['modules'].append( "-hltDQMFileSaver" )
0813       self.options['modules'].append( "-hltDQMFileSaverPB" )
0814
0815     if self.config.fragment:
0816       # extract a configuration file fragment
0817       self.options['essources'].append( "-GlobalTag" )
0818       self.options['essources'].append( "-HepPDTESSource" )
0819       self.options['essources'].append( "-XMLIdealGeometryESSource" )
0820       self.options['essources'].append( "-eegeom" )
0821       self.options['essources'].append( "-es_hardcode" )
0822       self.options['essources'].append( "-magfield" )
0823
0824       self.options['esmodules'].append( "-SlaveField0" )
0825       self.options['esmodules'].append( "-SlaveField20" )
0826       self.options['esmodules'].append( "-SlaveField30" )
0827       self.options['esmodules'].append( "-SlaveField35" )
0828       self.options['esmodules'].append( "-SlaveField38" )
0829       self.options['esmodules'].append( "-SlaveField40" )
0830       self.options['esmodules'].append( "-VBF0" )
0831       self.options['esmodules'].append( "-VBF20" )
0832       self.options['esmodules'].append( "-VBF30" )
0833       self.options['esmodules'].append( "-VBF35" )
0834       self.options['esmodules'].append( "-VBF38" )
0835       self.options['esmodules'].append( "-VBF40" )
0836       self.options['esmodules'].append( "-CSCGeometryESModule" )
0837       self.options['esmodules'].append( "-CaloGeometryBuilder" )
0838       self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
0839       self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
0840       self.options['esmodules'].append( "-DTGeometryESModule" )
0841       self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
0842       self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
0843       self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
0844       self.options['esmodules'].append( "-EcalLaserCorrectionService" )
0845       self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
0846       self.options['esmodules'].append( "-GEMGeometryESModule" )
0847       self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
0848       self.options['esmodules'].append( "-HcalTopologyIdealEP" )
0849       self.options['esmodules'].append( "-MuonNumberingInitialization" )
0850       self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
0851       self.options['esmodules'].append( "-RPCGeometryESModule" )
0852       self.options['esmodules'].append( "-SiStripGainESProducer" )
0853       self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
0854       self.options['esmodules'].append( "-SiStripQualityESProducer" )
0855       self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
0856       self.options['esmodules'].append( "-TrackerAdditionalParametersPerDetESModule" )
0857       self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
0858       self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
0859       self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
0860       self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
0861       self.options['esmodules'].append( "-hcal_db_producer" )
0862       self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
0863       self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
0864       self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
0865       self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
0866       self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
0867       self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
0868       self.options['esmodules'].append( "-trackerTopology" )
0869
0870       self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
0871       self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
0872       self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
0873       self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
0874       self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
0875       self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
0876       self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
0877       self.options['esmodules'].append( "-XMLFromDBSource" )
0878       self.options['esmodules'].append( "-sistripconn" )
0879       self.options['esmodules'].append( "-siPixelQualityESProducer" )
0880
0881       self.options['services'].append( "-MessageLogger" )
0882
0883       self.options['psets'].append( "-maxEvents" )
0884       self.options['psets'].append( "-options" )
0885
0886       # remove Scouting OutputModules even though the EndPaths are kept
0887       self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
0888       self.options['modules'].append( "-hltOutputScoutingPF" )
0889
0890     if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
0891       self.options['services'].append( "-PrescaleService" )
0892
0893     if self.config.fragment or self.config.timing:
0894       self.options['services'].append( "-FastTimerService" )
0895       self.options['services'].append( "-ThroughputService" )
0896
0897
0898   def append_filenames(self, name, filenames):
0899     if len(filenames) > 255:
0900       token_open  = "( *("
0901       token_close = ") )"
0902     else:
0903       token_open  = "("
0904       token_close = ")"
0905
0906     self.data += "    %s = cms.untracked.vstring%s\n" % (name, token_open)
0907     for line in filenames:
0908       self.data += "        '%s',\n" % line
0909     self.data += "    %s,\n" % (token_close)
0910
0911
0912   def expand_filenames(self, input):
0913     # check if the input is a dataset or a list of files
0914     if input[0:8] == 'dataset:':
0915       from .dasFileQuery import dasFileQuery
0916       # extract the dataset name, and use DAS to fine the list of LFNs
0917       dataset = input[8:]
0918       files = dasFileQuery(dataset)
0919     else:
0920       # assume a comma-separated list of input files
0921       files = input.split(',')
0922     return files
0923
0924   def build_source(self):
0925     if self.config.hilton:
0926       # use the DAQ source
0927       return
0928
0929     if self.config.input:
0930       # if a dataset or a list of input files was given, use it
0931       self.source = self.expand_filenames(self.config.input)
0932     elif self.config.data:
0933       # offline we can run on data...
0934       self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
0935     else:
0936       # ...or on mc
0937       self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
0938
0939     if self.config.parent:
0940       # if a dataset or a list of input files was given for the parent data, use it
0941       self.parent = self.expand_filenames(self.config.parent)
0942
0943     self.data += """
0944 # source module (EDM inputs)
0945 %(process)s.source = cms.Source( "PoolSource",
0946 """
0947     self.append_filenames("fileNames", self.source)
0948     if (self.parent):
0949       self.append_filenames("secondaryFileNames", self.parent)
0950     self.data += """\
0951     inputCommands = cms.untracked.vstring(
0952         'keep *'
0953     )
0954 )
0955 """