Configuration/test/getEventContent.py

0001 #!/usr/bin/env python3
0002 """getEventContent.py: print EventContent cff fragment of a ConfDB configuration
0003 """
0004 import argparse
0005 import subprocess
0006 import os
0007 import re
0008
0009 import FWCore.ParameterSet.Config as cms
0010 import HLTrigger.Configuration.Tools.pipe as pipe
0011 import HLTrigger.Configuration.Tools.options as options
0012
0013 def getHLTProcessBlocks(config, blocks):
0014   """return cms.Process containing the OutputModules of the HLT configuration
0015   """
0016   # cmd-line args to select HLT configuration
0017   if config.menu.run:
0018     configline = f'--runNumber {config.menu.run}'
0019   else:
0020     configline = f'--{config.menu.database} --{config.menu.version} --configName {config.menu.name}'
0021
0022   # cmd to download HLT configuration
0023   cmdline = f'hltConfigFromDB {configline}'
0024   if config.proxy:
0025     cmdline += f' --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}'
0026
0027   cmdline += ' --noedsources --noes --nopsets --noservices --nopaths --format python'
0028   cmdline += ' --blocks '+','.join({foo+'::outputCommands' for foo in blocks})
0029
0030   # load HLT configuration
0031   try:
0032     foo = {}
0033     exec(pipe.pipe(cmdline).decode(), foo)
0034   except:
0035     raise Exception(f'query did not return a valid python file:\n query="{cmdline}"')
0036
0037   ret = {}
0038   for block in blocks:
0039     key = 'block_'+block
0040     ret[key] = foo[key] if key in foo else None
0041     if ret[key] != None and not isinstance(ret[key], cms.PSet):
0042       raise Exception(f'query did not return valid HLT blocks:\n query="{cmdline}"')
0043
0044   return ret
0045
0046 def getHLTProcessBlockGroups(config, blockGroupDict):
0047   ret = {}
0048   blockDict = getHLTProcessBlocks(config, {bar for foo in blockGroupDict.values() for bar in foo})
0049   for groupName in blockGroupDict:
0050     ret[groupName] = cms.PSet()
0051     for blockKey in blockGroupDict[groupName]:
0052       blockName = 'block_'+blockKey
0053       if blockDict[blockName] != None:
0054         setattr(ret[groupName], blockName, blockDict[blockName])
0055   return ret
0056
0057 def makePSet(statements):
0058   statements = sorted(statements)
0059   block = cms.PSet(
0060     outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*', )
0061   )
0062   block.outputCommands.extend( statements )
0063   return block
0064
0065 def makePSetNoDrop(statements):
0066   statements = sorted(statements)
0067   block = cms.PSet(
0068     outputCommands = cms.untracked.vstring()
0069   )
0070   block.outputCommands.extend( statements )
0071   return block
0072
0073 def buildPSet(blocks):
0074   statements = set()
0075   for block in blocks:
0076     statements.update( statement for statement in block if statement.find('drop') != 0 )
0077   return makePSet(statements)
0078
0079 def buildPSetNoDrop(blocks):
0080   statements = set()
0081   for block in blocks:
0082     statements.update( statement for statement in block if statement.find('drop') != 0 )
0083   return makePSetNoDrop(statements)
0084
0085 def buildPSetWithoutRAWs(blocks):
0086   statements = set()
0087   for block in blocks:
0088     statements.update( statement for statement in block if statement.find('drop') != 0 and statement.find('keep FEDRawDataCollection') != 0)
0089   return makePSet(statements)
0090
0091 # customisation of AOD event content, requested by David Dagenhart
0092 def dropL1GlobalTriggerObjectMapRecord(block):
0093   """drop the old L1GlobalTriggerObjectMapRecord data format from the block (meant for the AOD data tier)"""
0094   try:
0095     # look for the hltL1GtObjectMap keep statement
0096     position = block.outputCommands.index('keep *_hltL1GtObjectMap_*_*')
0097   except ValueError:
0098     pass
0099   else:
0100     # add just after it a drop statement for the old data format
0101     block.outputCommands.insert(position  + 1, 'drop L1GlobalTriggerObjectMapRecord_hltL1GtObjectMap_*_*')
0102
0103 def printHLTriggerEventContentCff(process):
0104
0105   blockGroups = getHLTProcessBlockGroups(config, {
0106     'hltOutputA_cff': [
0107       'hltOutputA',
0108       'hltOutputPhysicsCommissioning',
0109     ],
0110     'hltOutputALCA_cff': [
0111       'hltOutputALCAPHISYM',
0112       'hltOutputALCAP0',
0113       'hltOutputALCAPPSExpress',
0114       'hltOutputALCAPPSPrompt',
0115       'hltOutputALCALumiPixelsCountsExpress',
0116       'hltOutputALCALumiPixelsCountsPrompt',
0117       'hltOutputRPCMON',
0118     ],
0119     'hltOutputMON_cff': [
0120       'hltOutputA',
0121       'hltOutputPhysicsCommissioning',
0122       'hltOutputDQM',
0123       'hltOutputHLTMonitor',
0124       'hltOutputReleaseValidation',
0125     ],
0126     'hltOutputScouting_cff': [
0127       'hltOutputScoutingPF',
0128     ],
0129   })
0130
0131   hltOutputA_cff = blockGroups['hltOutputA_cff']
0132   hltOutputALCA_cff = blockGroups['hltOutputALCA_cff']
0133   hltOutputMON_cff = blockGroups['hltOutputMON_cff']
0134   hltOutputScouting_cff = blockGroups['hltOutputScouting_cff']
0135
0136   # hltDebugOutput
0137
0138   if not hasattr(hltOutputMON_cff,'block_hltOutputA'):
0139     hltOutputMON_cff.block_hltOutputA = hltOutputMON_cff.block_hltOutputPhysicsCommissioning
0140   if not hasattr(hltOutputMON_cff,'block_hltOutputDQM'):
0141     hltOutputMON_cff.block_hltOutputDQM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0142   if not hasattr(hltOutputMON_cff,'block_hltOutputHLTMonitor'):
0143     hltOutputMON_cff.block_hltOutputHLTMonitor = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*' ))
0144   if not hasattr(hltOutputMON_cff,'block_hltOutputReleaseValidation'):
0145     hltOutputMON_cff.block_hltOutputReleaseValidation = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0146
0147   hltDebugOutputBlocks = (
0148     # the DQM and HLTMON streams have the HLT debug outputs used online
0149     hltOutputMON_cff.block_hltOutputA.outputCommands,
0150     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0151     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0152     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0153   )
0154   hltDebugOutputContent = buildPSet(hltDebugOutputBlocks)
0155
0156   # hltDebugWithAlCaOutput
0157
0158   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPHISYM'):
0159     hltOutputALCA_cff.block_hltOutputALCAPHISYM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0160   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAP0'):
0161     hltOutputALCA_cff.block_hltOutputALCAP0 = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0162   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSExpress'):
0163     hltOutputALCA_cff.block_hltOutputALCAPPSExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0164   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSPrompt'):
0165     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0166   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsExpress'):
0167     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0168   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsPrompt'):
0169     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0170   if not hasattr(hltOutputALCA_cff,'block_hltOutputRPCMON'):
0171     hltOutputALCA_cff.block_hltOutputRPCMON = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0172   hltDebugWithAlCaOutputBlocks = (
0173     # the DQM and HLTMON streams have the HLT debug outputs used online
0174     hltOutputMON_cff.block_hltOutputA.outputCommands,
0175     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0176     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0177     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0178     # the ALCA streams have the AlCa outputs
0179     hltOutputALCA_cff.block_hltOutputALCAPHISYM.outputCommands,
0180     hltOutputALCA_cff.block_hltOutputALCAP0.outputCommands,
0181     hltOutputALCA_cff.block_hltOutputALCAPPSExpress.outputCommands,
0182     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt.outputCommands,
0183     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress.outputCommands,
0184     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt.outputCommands,
0185     hltOutputALCA_cff.block_hltOutputRPCMON.outputCommands,
0186   )
0187   hltDebugWithAlCaOutputContent = buildPSet(hltDebugWithAlCaOutputBlocks)
0188
0189   # hltScoutingOutput
0190
0191   if not hasattr(hltOutputScouting_cff,'block_hltOutputScoutingPF'):
0192     hltOutputScouting_cff.block_hltOutputScoutingPF = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0193
0194   hltScoutingOutputBlocks = (
0195     # the Scouting streams have the Scouting outputs
0196     hltOutputScouting_cff.block_hltOutputScoutingPF.outputCommands,
0197   )
0198   hltScoutingOutputContent = buildPSetNoDrop(hltScoutingOutputBlocks)
0199
0200   # hltDefaultOutput
0201   if not hasattr(hltOutputA_cff,'block_hltOutputA'):
0202     hltOutputA_cff.block_hltOutputA = hltOutputA_cff.block_hltOutputPhysicsCommissioning
0203   hltDefaultOutputBlocks = (
0204     # the A stream has the HLT default output, with FEDs - strip out the FEDRawDataCollection keep statements for hltDefaultOutput
0205     hltOutputA_cff.block_hltOutputA.outputCommands,
0206   )
0207   hltDefaultOutputContent         = buildPSetWithoutRAWs(hltDefaultOutputBlocks)
0208   hltDefaultOutputWithFEDsContent = buildPSet(hltDefaultOutputBlocks)
0209
0210   # define the CMSSW default event content configurations
0211
0212   # RAW event content
0213   HLTriggerRAW = cms.PSet(
0214       outputCommands = cms.vstring()
0215   )
0216   HLTriggerRAW.outputCommands.extend(hltDefaultOutputWithFEDsContent.outputCommands)
0217   HLTriggerRAW.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0218
0219   # RECO event content
0220   HLTriggerRECO = cms.PSet(
0221       outputCommands = cms.vstring()
0222   )
0223   HLTriggerRECO.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0224   HLTriggerRECO.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0225
0226   # AOD event content
0227   HLTriggerAOD = cms.PSet(
0228       outputCommands = cms.vstring()
0229   )
0230   HLTriggerAOD.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0231   HLTriggerAOD.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0232   dropL1GlobalTriggerObjectMapRecord(HLTriggerAOD)
0233
0234   # HLTDEBUG RAW event content
0235   HLTDebugRAW = cms.PSet(
0236       outputCommands = cms.vstring()
0237   )
0238   HLTDebugRAW.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0239   HLTDebugRAW.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0240
0241   # HLTDEBUG FEVT event content
0242   HLTDebugFEVT = cms.PSet(
0243       outputCommands = cms.vstring()
0244   )
0245   HLTDebugFEVT.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0246   HLTDebugFEVT.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0247
0248   # Scouting event content
0249   HLTScouting = cms.PSet(
0250       outputCommands = cms.vstring()
0251   )
0252   HLTScouting.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0253
0254   # print the expanded event content configurations to stdout
0255   print('''import FWCore.ParameterSet.Config as cms
0256
0257 # EventContent for HLT related products.
0258
0259 # This file exports the following EventContent blocks:
0260 #   HLTriggerRAW  HLTriggerRECO  HLTriggerAOD (without DEBUG products)
0261 #   HLTDebugRAW   HLTDebugFEVT                (with    DEBUG products)
0262 #   HLTScouting                               (with Scouting products)
0263 #
0264 # as these are used in Configuration/EventContent
0265 #''')
0266   print('HLTriggerRAW  = cms.PSet(\n    outputCommands = cms.vstring( *(\n%s\n    ) )\n)\n'  % ',\n'.join( '        \'%s\'' % keep for keep in HLTriggerRAW.outputCommands))
0267   print('HLTriggerRECO = cms.PSet(\n    outputCommands = cms.vstring( *(\n%s\n    ) )\n)\n'  % ',\n'.join( '        \'%s\'' % keep for keep in HLTriggerRECO.outputCommands))
0268   print('HLTriggerAOD  = cms.PSet(\n    outputCommands = cms.vstring( *(\n%s\n    ) )\n)\n'  % ',\n'.join( '        \'%s\'' % keep for keep in HLTriggerAOD.outputCommands))
0269   print('HLTDebugRAW   = cms.PSet(\n    outputCommands = cms.vstring( *(\n%s\n    ) )\n)\n'  % ',\n'.join( '        \'%s\'' % keep for keep in HLTDebugRAW.outputCommands))
0270   print('HLTDebugFEVT  = cms.PSet(\n    outputCommands = cms.vstring( *(\n%s\n    ) )\n)\n'  % ',\n'.join( '        \'%s\'' % keep for keep in HLTDebugFEVT.outputCommands))
0271   print('HLTScouting   = cms.PSet(\n    outputCommands = cms.vstring( *(\n%s\n    ) )\n)\n'  % ',\n'.join( '        \'%s\'' % keep for keep in HLTScouting.outputCommands))
0272
0273 ###
0274 ### main
0275 ###
0276 if __name__ == '__main__':
0277
0278   # defaults of cmd-line arguments
0279   defaults = options.HLTProcessOptions()
0280
0281   parser = argparse.ArgumentParser(
0282     prog = './'+os.path.basename(__file__),
0283     formatter_class = argparse.RawDescriptionHelpFormatter,
0284     description = __doc__
0285   )
0286
0287   # required argument
0288   parser.add_argument('menu',
0289                       action  = 'store',
0290                       type    = options.ConnectionHLTMenu,
0291                       metavar = 'MENU',
0292                       help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
0293
0294   # options
0295   parser.add_argument('--dbproxy',
0296                       dest    = 'proxy',
0297                       action  = 'store_true',
0298                       default = defaults.proxy,
0299                       help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
0300   parser.add_argument('--dbproxyport',
0301                       dest    = 'proxy_port',
0302                       action  = 'store',
0303                       metavar = 'PROXYPORT',
0304                       default = defaults.proxy_port,
0305                        help    = 'Port of the socks proxy (default: 8080)' )
0306   parser.add_argument('--dbproxyhost',
0307                       dest    = 'proxy_host',
0308                       action  = 'store',
0309                       metavar = 'PROXYHOST',
0310                       default = defaults.proxy_host,
0311                       help    = 'Host of the socks proxy (default: "localhost")' )
0312
0313   # parse command line arguments and options
0314   config = parser.parse_args()
0315
0316   printHLTriggerEventContentCff(config)