Configuration/test/getEventContent.py

0001 #!/usr/bin/env python3
0002 """getEventContent.py: print EventContent cff fragment of a ConfDB configuration
0003 """
0004 import argparse
0005 import subprocess
0006 import os
0007 import re
0008
0009 import FWCore.ParameterSet.Config as cms
0010 import HLTrigger.Configuration.Tools.pipe as pipe
0011 import HLTrigger.Configuration.Tools.options as options
0012
0013 def getHLTProcessBlocks(config, blocks):
0014   """return cms.Process containing the OutputModules of the HLT configuration
0015   """
0016   # cmd-line args to select HLT configuration
0017   if config.menu.run:
0018     configline = f'--runNumber {config.menu.run}'
0019   else:
0020     configline = f'--{config.menu.database} --{config.menu.version} --configName {config.menu.name}'
0021
0022   # cmd to download HLT configuration
0023   cmdline = f'hltConfigFromDB {configline}'
0024   if config.proxy:
0025     cmdline += f' --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}'
0026
0027   cmdline += ' --noedsources --noes --nopsets --noservices --nopaths --format python'
0028   cmdline += ' --blocks '+','.join({foo+'::outputCommands' for foo in blocks})
0029
0030   # load HLT configuration
0031   try:
0032     foo = {}
0033     exec(pipe.pipe(cmdline).decode(), foo)
0034   except:
0035     raise Exception(f'query did not return a valid python file:\n query="{cmdline}"')
0036
0037   ret = {}
0038   for block in blocks:
0039     key = 'block_'+block
0040     ret[key] = foo[key] if key in foo else None
0041     if ret[key] != None and not isinstance(ret[key], cms.PSet):
0042       raise Exception(f'query did not return valid HLT blocks:\n query="{cmdline}"')
0043
0044   return ret
0045
0046 def getHLTProcessBlockGroups(config, blockGroupDict):
0047   ret = {}
0048   blockDict = getHLTProcessBlocks(config, {bar for foo in blockGroupDict.values() for bar in foo})
0049   for groupName in blockGroupDict:
0050     ret[groupName] = cms.PSet()
0051     for blockKey in blockGroupDict[groupName]:
0052       blockName = 'block_'+blockKey
0053       if blockDict[blockName] != None:
0054         setattr(ret[groupName], blockName, blockDict[blockName])
0055   return ret
0056
0057 def makePSet(statements):
0058   statements = sorted(statements)
0059   block = cms.PSet(
0060     outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*', )
0061   )
0062   block.outputCommands.extend( statements )
0063   return block
0064
0065 def makePSetNoDrop(statements):
0066   statements = sorted(statements)
0067   block = cms.PSet(
0068     outputCommands = cms.untracked.vstring()
0069   )
0070   block.outputCommands.extend( statements )
0071   return block
0072
0073 def buildPSet(blocks):
0074   statements = set()
0075   for block in blocks:
0076     statements.update( statement for statement in block if statement.find('drop') != 0 )
0077   return makePSet(statements)
0078
0079 def buildPSetNoDrop(blocks):
0080   statements = set()
0081   for block in blocks:
0082     statements.update( statement for statement in block if statement.find('drop') != 0 )
0083   return makePSetNoDrop(statements)
0084
0085 def buildPSetWithoutRAWs(blocks):
0086   statements = set()
0087   for block in blocks:
0088     statements.update( statement for statement in block if statement.find('drop') != 0 and statement.find('keep FEDRawDataCollection') != 0)
0089   return makePSet(statements)
0090
0091 # customisation of AOD event content, requested by David Dagenhart
0092 def dropL1GlobalTriggerObjectMapRecord(block):
0093   """drop the old L1GlobalTriggerObjectMapRecord data format from the block (meant for the AOD data tier)"""
0094   try:
0095     # look for the hltL1GtObjectMap keep statement
0096     position = block.outputCommands.index('keep *_hltL1GtObjectMap_*_*')
0097   except ValueError:
0098     pass
0099   else:
0100     # add just after it a drop statement for the old data format
0101     block.outputCommands.insert(position  + 1, 'drop L1GlobalTriggerObjectMapRecord_hltL1GtObjectMap_*_*')
0102
0103 def printHLTriggerEventContentCff(process):
0104
0105   blockGroups = getHLTProcessBlockGroups(config, {
0106     'hltOutputA_cff': [
0107       'hltOutputA',
0108       'hltOutputPhysicsCommissioning',
0109     ],
0110     'hltOutputALCA_cff': [
0111       'hltOutputALCAPHISYM',
0112       'hltOutputALCAP0',
0113       'hltOutputALCAPPSExpress',
0114       'hltOutputALCAPPSPrompt',
0115       'hltOutputALCALumiPixelsCountsExpress',
0116       'hltOutputALCALumiPixelsCountsPrompt',
0117     ],
0118     'hltOutputMON_cff': [
0119       'hltOutputA',
0120       'hltOutputPhysicsCommissioning',
0121       'hltOutputDQM',
0122       'hltOutputHLTMonitor',
0123       'hltOutputReleaseValidation',
0124     ],
0125     'hltOutputScouting_cff': [
0126       'hltOutputScoutingPF',
0127     ],
0128   })
0129
0130   hltOutputA_cff = blockGroups['hltOutputA_cff']
0131   hltOutputALCA_cff = blockGroups['hltOutputALCA_cff']
0132   hltOutputMON_cff = blockGroups['hltOutputMON_cff']
0133   hltOutputScouting_cff = blockGroups['hltOutputScouting_cff']
0134
0135   # hltDebugOutput
0136
0137   if not hasattr(hltOutputMON_cff,'block_hltOutputA'):
0138     hltOutputMON_cff.block_hltOutputA = hltOutputMON_cff.block_hltOutputPhysicsCommissioning
0139   if not hasattr(hltOutputMON_cff,'block_hltOutputDQM'):
0140     hltOutputMON_cff.block_hltOutputDQM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0141   if not hasattr(hltOutputMON_cff,'block_hltOutputHLTMonitor'):
0142     hltOutputMON_cff.block_hltOutputHLTMonitor = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*' ))
0143   if not hasattr(hltOutputMON_cff,'block_hltOutputReleaseValidation'):
0144     hltOutputMON_cff.block_hltOutputReleaseValidation = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0145
0146   hltDebugOutputBlocks = (
0147     # the DQM and HLTMON streams have the HLT debug outputs used online
0148     hltOutputMON_cff.block_hltOutputA.outputCommands,
0149     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0150     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0151     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0152   )
0153   hltDebugOutputContent = buildPSet(hltDebugOutputBlocks)
0154
0155   # hltDebugWithAlCaOutput
0156
0157   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPHISYM'):
0158     hltOutputALCA_cff.block_hltOutputALCAPHISYM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0159   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAP0'):
0160     hltOutputALCA_cff.block_hltOutputALCAP0 = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0161   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSExpress'):
0162     hltOutputALCA_cff.block_hltOutputALCAPPSExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0163   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSPrompt'):
0164     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0165   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsExpress'):
0166     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0167   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsPrompt'):
0168     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0169   hltDebugWithAlCaOutputBlocks = (
0170     # the DQM and HLTMON streams have the HLT debug outputs used online
0171     hltOutputMON_cff.block_hltOutputA.outputCommands,
0172     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0173     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0174     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0175     # the ALCA streams have the AlCa outputs
0176     hltOutputALCA_cff.block_hltOutputALCAPHISYM.outputCommands,
0177     hltOutputALCA_cff.block_hltOutputALCAP0.outputCommands,
0178     hltOutputALCA_cff.block_hltOutputALCAPPSExpress.outputCommands,
0179     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt.outputCommands,
0180     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress.outputCommands,
0181     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt.outputCommands,
0182   )
0183   hltDebugWithAlCaOutputContent = buildPSet(hltDebugWithAlCaOutputBlocks)
0184
0185   # hltScoutingOutput
0186
0187   if not hasattr(hltOutputScouting_cff,'block_hltOutputScoutingPF'):
0188     hltOutputScouting_cff.block_hltOutputScoutingPF = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0189
0190   hltScoutingOutputBlocks = (
0191     # the Scouting streams have the Scouting outputs
0192     hltOutputScouting_cff.block_hltOutputScoutingPF.outputCommands,
0193   )
0194   hltScoutingOutputContent = buildPSet(hltScoutingOutputBlocks)
0195   hltScoutingOutputContentNoDrop = buildPSetNoDrop(hltScoutingOutputBlocks)
0196
0197   # hltDefaultOutput
0198   if not hasattr(hltOutputA_cff,'block_hltOutputA'):
0199     hltOutputA_cff.block_hltOutputA = hltOutputA_cff.block_hltOutputPhysicsCommissioning
0200   hltDefaultOutputBlocks = (
0201     # the A stream has the HLT default output, with FEDs - strip out the FEDRawDataCollection keep statements for hltDefaultOutput
0202     hltOutputA_cff.block_hltOutputA.outputCommands,
0203   )
0204   hltDefaultOutputContent         = buildPSetWithoutRAWs(hltDefaultOutputBlocks)
0205   hltDefaultOutputWithFEDsContent = buildPSet(hltDefaultOutputBlocks)
0206
0207   # define the CMSSW default event content configurations
0208
0209   # RAW event content
0210   HLTriggerRAW = cms.PSet(
0211       outputCommands = cms.vstring()
0212   )
0213   HLTriggerRAW.outputCommands.extend(hltDefaultOutputWithFEDsContent.outputCommands)
0214   HLTriggerRAW.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0215
0216   # RECO event content
0217   HLTriggerRECO = cms.PSet(
0218       outputCommands = cms.vstring()
0219   )
0220   HLTriggerRECO.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0221   HLTriggerRECO.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0222
0223   # AOD event content
0224   HLTriggerAOD = cms.PSet(
0225       outputCommands = cms.vstring()
0226   )
0227   HLTriggerAOD.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0228   HLTriggerAOD.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0229   dropL1GlobalTriggerObjectMapRecord(HLTriggerAOD)
0230
0231   # MINIAOD event content
0232   HLTriggerMINIAOD = cms.PSet(
0233       outputCommands = cms.vstring()
0234   )
0235   HLTriggerMINIAOD.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0236
0237   # HLTDEBUG RAW event content
0238   HLTDebugRAW = cms.PSet(
0239       outputCommands = cms.vstring()
0240   )
0241   HLTDebugRAW.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0242   HLTDebugRAW.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0243
0244   # HLTDEBUG FEVT event content
0245   HLTDebugFEVT = cms.PSet(
0246       outputCommands = cms.vstring()
0247   )
0248   HLTDebugFEVT.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0249   HLTDebugFEVT.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0250
0251   # Scouting event content
0252   HLTScouting = cms.PSet(
0253       outputCommands = cms.vstring()
0254   )
0255   HLTScouting.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0256
0257   # print the expanded event content configurations to stdout
0258   print('''import FWCore.ParameterSet.Config as cms
0259
0260 # EventContent for HLT-related products.
0261
0262 # This file exports the following EventContent blocks:
0263 #   HLTrigger(RAW|RECO|AOD|MINIAOD) [without DEBUG products]
0264 #   HLTDebug(RAW|FEVT)              [with    DEBUG products]
0265 #   HLTScouting                     [only Scouting products]
0266 #
0267 # as these are used in Configuration/EventContent
0268 #''')
0269
0270   def psetString(name, outputCommands):
0271       str_products = ',\n'.join(f'        \'{str_cmd}\'' for str_cmd in outputCommands)
0272       return f'\n{name} = cms.PSet(\n    outputCommands = cms.vstring( *(\n{str_products}\n    ) )\n)'
0273
0274   print(psetString('HLTriggerRAW', HLTriggerRAW.outputCommands))
0275   print(psetString('HLTriggerRECO', HLTriggerRECO.outputCommands))
0276   print(psetString('HLTriggerAOD', HLTriggerAOD.outputCommands))
0277   print(psetString('HLTriggerMINIAOD', HLTriggerMINIAOD.outputCommands))
0278   print(psetString('HLTDebugRAW', HLTDebugRAW.outputCommands))
0279   print(psetString('HLTDebugFEVT', HLTDebugFEVT.outputCommands))
0280   print(psetString('HLTScouting', HLTScouting.outputCommands))
0281
0282 ###
0283 ### main
0284 ###
0285 if __name__ == '__main__':
0286
0287   # defaults of cmd-line arguments
0288   defaults = options.HLTProcessOptions()
0289
0290   parser = argparse.ArgumentParser(
0291     prog = './'+os.path.basename(__file__),
0292     formatter_class = argparse.RawDescriptionHelpFormatter,
0293     description = __doc__
0294   )
0295
0296   # required argument
0297   parser.add_argument('menu',
0298                       action  = 'store',
0299                       type    = options.ConnectionHLTMenu,
0300                       metavar = 'MENU',
0301                       help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
0302
0303   # options
0304   parser.add_argument('--dbproxy',
0305                       dest    = 'proxy',
0306                       action  = 'store_true',
0307                       default = defaults.proxy,
0308                       help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
0309   parser.add_argument('--dbproxyport',
0310                       dest    = 'proxy_port',
0311                       action  = 'store',
0312                       metavar = 'PROXYPORT',
0313                       default = defaults.proxy_port,
0314                        help    = 'Port of the socks proxy (default: 8080)' )
0315   parser.add_argument('--dbproxyhost',
0316                       dest    = 'proxy_host',
0317                       action  = 'store',
0318                       metavar = 'PROXYHOST',
0319                       default = defaults.proxy_host,
0320                       help    = 'Host of the socks proxy (default: "localhost")' )
0321
0322   # parse command line arguments and options
0323   config = parser.parse_args()
0324
0325   printHLTriggerEventContentCff(config)