Configuration/test/getEventContent.py

0001 #!/usr/bin/env python3
0002 """getEventContent.py: print EventContent cff fragment of a ConfDB configuration
0003 """
0004 import argparse
0005 import os
0006
0007 import FWCore.ParameterSet.Config as cms
0008 import HLTrigger.Configuration.Tools.pipe as pipe
0009 import HLTrigger.Configuration.Tools.options as options
0010
0011 def getHLTProcessBlocks(config, blocks):
0012   """return cms.Process containing the OutputModules of the HLT configuration
0013   """
0014   # cmd-line args to select HLT configuration
0015   if config.menu.run:
0016     configline = f'--runNumber {config.menu.run}'
0017   else:
0018     configline = f'--{config.menu.database} --{config.menu.version} --configName {config.menu.name}'
0019
0020   # cmd to download HLT configuration
0021   cmdline = f'hltConfigFromDB {configline}'
0022   if config.proxy:
0023     cmdline += f' --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}'
0024
0025   cmdline += ' --noedsources --noes --nopsets --noservices --nopaths --format python'
0026   cmdline += ' --blocks '+','.join({foo+'::outputCommands' for foo in blocks})
0027
0028   # load HLT configuration
0029   try:
0030     foo = {}
0031     exec(pipe.pipe(cmdline).decode(), foo)
0032   except:
0033     raise Exception(f'query did not return a valid python file:\n query="{cmdline}"')
0034
0035   ret = {}
0036   for block in blocks:
0037     key = 'block_'+block
0038     ret[key] = foo[key] if key in foo else None
0039     if ret[key] != None and not isinstance(ret[key], cms.PSet):
0040       raise Exception(f'query did not return valid HLT blocks:\n query="{cmdline}"')
0041
0042   return ret
0043
0044 def getHLTProcessBlockGroups(config, blockGroupDict):
0045   ret = {}
0046   blockDict = getHLTProcessBlocks(config, {bar for foo in blockGroupDict.values() for bar in foo})
0047   for groupName in blockGroupDict:
0048     ret[groupName] = cms.PSet()
0049     for blockKey in blockGroupDict[groupName]:
0050       blockName = 'block_'+blockKey
0051       if blockDict[blockName] != None:
0052         setattr(ret[groupName], blockName, blockDict[blockName])
0053   return ret
0054
0055 def makePSet(statements):
0056   statements = sorted(statements)
0057   block = cms.PSet(
0058     outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*', )
0059   )
0060   block.outputCommands.extend( statements )
0061   return block
0062
0063 def makePSetNoDrop(statements):
0064   statements = sorted(statements)
0065   block = cms.PSet(
0066     outputCommands = cms.untracked.vstring()
0067   )
0068   block.outputCommands.extend( statements )
0069   return block
0070
0071 def buildPSet(blocks):
0072   statements = set()
0073   for block in blocks:
0074     statements.update( statement for statement in block if statement.find('drop') != 0 )
0075   return makePSet(statements)
0076
0077 def buildPSetNoDrop(blocks):
0078   statements = set()
0079   for block in blocks:
0080     statements.update( statement for statement in block if statement.find('drop') != 0 )
0081   return makePSetNoDrop(statements)
0082
0083 def buildPSetWithoutRAWs(blocks):
0084   statements = set()
0085   for block in blocks:
0086     statements.update( statement for statement in block if statement.find('drop') != 0 and statement.find('keep FEDRawDataCollection') != 0)
0087   return makePSet(statements)
0088
0089 # customisation of AOD event content, requested by David Dagenhart
0090 def dropL1GlobalTriggerObjectMapRecord(block):
0091   """drop the old L1GlobalTriggerObjectMapRecord data format from the block (meant for the AOD data tier)"""
0092   try:
0093     # look for the hltL1GtObjectMap keep statement
0094     position = block.outputCommands.index('keep *_hltL1GtObjectMap_*_*')
0095   except ValueError:
0096     pass
0097   else:
0098     # add just after it a drop statement for the old data format
0099     block.outputCommands.insert(position  + 1, 'drop L1GlobalTriggerObjectMapRecord_hltL1GtObjectMap_*_*')
0100
0101 def printHLTriggerEventContentCff(process):
0102
0103   blockGroups = getHLTProcessBlockGroups(config, {
0104     'hltOutputA_cff': [
0105       'hltOutputA',
0106       'hltOutputPhysicsCommissioning',
0107     ],
0108     'hltOutputALCA_cff': [
0109       'hltOutputALCAHcalIsoTrk',
0110       'hltOutputALCALowPtJet',
0111       'hltOutputALCALumiPixelsCountsExpress',
0112       'hltOutputALCALumiPixelsCountsPrompt',
0113       'hltOutputALCAP0',
0114       'hltOutputALCAPHISYM',
0115       'hltOutputALCAPPSExpress',
0116       'hltOutputALCAPPSPrompt',
0117     ],
0118     'hltOutputMON_cff': [
0119       'hltOutputA',
0120       'hltOutputPhysicsCommissioning',
0121       'hltOutputDQM',
0122       'hltOutputHLTMonitor',
0123       'hltOutputReleaseValidation',
0124     ],
0125     'hltOutputScouting_cff': [
0126       'hltOutputScoutingPF0',
0127     ],
0128   })
0129
0130   hltOutputA_cff = blockGroups['hltOutputA_cff']
0131   hltOutputALCA_cff = blockGroups['hltOutputALCA_cff']
0132   hltOutputMON_cff = blockGroups['hltOutputMON_cff']
0133   hltOutputScouting_cff = blockGroups['hltOutputScouting_cff']
0134
0135   # hltDebugOutput
0136
0137   if not hasattr(hltOutputMON_cff,'block_hltOutputA'):
0138     hltOutputMON_cff.block_hltOutputA = hltOutputMON_cff.block_hltOutputPhysicsCommissioning
0139   if not hasattr(hltOutputMON_cff,'block_hltOutputDQM'):
0140     hltOutputMON_cff.block_hltOutputDQM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0141   if not hasattr(hltOutputMON_cff,'block_hltOutputHLTMonitor'):
0142     hltOutputMON_cff.block_hltOutputHLTMonitor = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*' ))
0143   if not hasattr(hltOutputMON_cff,'block_hltOutputReleaseValidation'):
0144     hltOutputMON_cff.block_hltOutputReleaseValidation = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0145
0146   hltDebugOutputBlocks = (
0147     # the DQM and HLTMON streams have the HLT debug outputs used online
0148     hltOutputMON_cff.block_hltOutputA.outputCommands,
0149     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0150     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0151     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0152   )
0153   hltDebugOutputContent = buildPSet(hltDebugOutputBlocks)
0154
0155   # hltDebugWithAlCaOutput
0156
0157   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAHcalIsoTrk'):
0158     hltOutputALCA_cff.block_hltOutputALCAHcalIsoTrk = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0159   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALowPtJet'):
0160     hltOutputALCA_cff.block_hltOutputALCALowPtJet = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0161   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsExpress'):
0162     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0163   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsPrompt'):
0164     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0165   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAP0'):
0166     hltOutputALCA_cff.block_hltOutputALCAP0 = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0167   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPHISYM'):
0168     hltOutputALCA_cff.block_hltOutputALCAPHISYM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0169   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSExpress'):
0170     hltOutputALCA_cff.block_hltOutputALCAPPSExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0171   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSPrompt'):
0172     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0173
0174   hltDebugWithAlCaOutputBlocks = (
0175     # the DQM and HLTMON streams have the HLT debug outputs used online
0176     hltOutputMON_cff.block_hltOutputA.outputCommands,
0177     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0178     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0179     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0180     # the ALCA streams have the AlCa outputs
0181     hltOutputALCA_cff.block_hltOutputALCAHcalIsoTrk.outputCommands,
0182     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress.outputCommands,
0183     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt.outputCommands,
0184     hltOutputALCA_cff.block_hltOutputALCAP0.outputCommands,
0185     hltOutputALCA_cff.block_hltOutputALCAPHISYM.outputCommands,
0186     hltOutputALCA_cff.block_hltOutputALCAPPSExpress.outputCommands,
0187     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt.outputCommands,
0188     hltOutputALCA_cff.block_hltOutputALCALowPtJet.outputCommands,
0189   )
0190   hltDebugWithAlCaOutputContent = buildPSet(hltDebugWithAlCaOutputBlocks)
0191
0192   # hltScoutingOutput
0193
0194   if not hasattr(hltOutputScouting_cff,'block_hltOutputScoutingPF0'):
0195     hltOutputScouting_cff.block_hltOutputScoutingPF0 = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0196
0197   hltScoutingOutputBlocks = (
0198     # the Scouting streams have the Scouting outputs
0199     hltOutputScouting_cff.block_hltOutputScoutingPF0.outputCommands,
0200   )
0201
0202   hltScoutingOutputContent = buildPSet(hltScoutingOutputBlocks)
0203   hltScoutingOutputContentNoDrop = buildPSetNoDrop(hltScoutingOutputBlocks)
0204
0205   # hltDefaultOutput
0206   if not hasattr(hltOutputA_cff,'block_hltOutputA'):
0207     hltOutputA_cff.block_hltOutputA = hltOutputA_cff.block_hltOutputPhysicsCommissioning
0208   hltDefaultOutputBlocks = (
0209     # the A stream has the HLT default output, with FEDs - strip out the FEDRawDataCollection keep statements for hltDefaultOutput
0210     hltOutputA_cff.block_hltOutputA.outputCommands,
0211   )
0212   hltDefaultOutputContent         = buildPSetWithoutRAWs(hltDefaultOutputBlocks)
0213   hltDefaultOutputWithFEDsContent = buildPSet(hltDefaultOutputBlocks)
0214
0215   # define the CMSSW default event content configurations
0216
0217   # RAW event content
0218   HLTriggerRAW = cms.PSet(
0219       outputCommands = cms.vstring()
0220   )
0221   HLTriggerRAW.outputCommands.extend(hltDefaultOutputWithFEDsContent.outputCommands)
0222   HLTriggerRAW.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0223
0224   # RECO event content
0225   HLTriggerRECO = cms.PSet(
0226       outputCommands = cms.vstring()
0227   )
0228   HLTriggerRECO.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0229   HLTriggerRECO.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0230
0231   # AOD event content
0232   HLTriggerAOD = cms.PSet(
0233       outputCommands = cms.vstring()
0234   )
0235   HLTriggerAOD.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0236   HLTriggerAOD.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0237   dropL1GlobalTriggerObjectMapRecord(HLTriggerAOD)
0238
0239   # MINIAOD event content
0240   HLTriggerMINIAOD = cms.PSet(
0241       outputCommands = cms.vstring()
0242   )
0243   HLTriggerMINIAOD.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0244
0245   # MINIAODSIM event content
0246   HLTriggerMINIAODSIM = cms.PSet(
0247       outputCommands = cms.vstring()
0248   )
0249
0250   _hltScoutingOutputContentForMINIAODSIM =  hltScoutingOutputContent.outputCommands.copy()
0251   _hltScoutingOutputContentForMINIAODSIM.remove('keep *_hltScoutingRecHitPacker_*_*')
0252   HLTriggerMINIAODSIM.outputCommands.extend(_hltScoutingOutputContentForMINIAODSIM)
0253
0254   # HLTDEBUG RAW event content
0255   HLTDebugRAW = cms.PSet(
0256       outputCommands = cms.vstring()
0257   )
0258   HLTDebugRAW.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0259   HLTDebugRAW.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0260
0261   # HLTDEBUG FEVT event content
0262   HLTDebugFEVT = cms.PSet(
0263       outputCommands = cms.vstring()
0264   )
0265   HLTDebugFEVT.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0266   HLTDebugFEVT.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0267
0268   # Scouting event content
0269   HLTScouting = cms.PSet(
0270       outputCommands = cms.vstring()
0271   )
0272   HLTScouting.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0273
0274   # print the expanded event content configurations to stdout
0275   print('''import FWCore.ParameterSet.Config as cms
0276
0277 # EventContent for HLT-related products.
0278
0279 # This file exports the following EventContent blocks:
0280 #   HLTrigger(RAW|RECO|AOD|MINIAOD{SIM}) [without DEBUG products]
0281 #   HLTDebug(RAW|FEVT)                   [with    DEBUG products]
0282 #   HLTScouting                          [only Scouting products]
0283 #
0284 # as these are used in Configuration/EventContent
0285 #''')
0286
0287   def psetString(name, outputCommands):
0288       str_products = ',\n'.join(f'        \'{str_cmd}\'' for str_cmd in outputCommands)
0289       return f'\n{name} = cms.PSet(\n    outputCommands = cms.vstring( *(\n{str_products}\n    ) )\n)'
0290
0291   print(psetString('HLTriggerRAW', HLTriggerRAW.outputCommands))
0292   print(psetString('HLTriggerRECO', HLTriggerRECO.outputCommands))
0293   print(psetString('HLTriggerAOD', HLTriggerAOD.outputCommands))
0294   print(psetString('HLTriggerMINIAOD', HLTriggerMINIAOD.outputCommands))
0295   print(psetString('HLTriggerMINIAODSIM', HLTriggerMINIAODSIM.outputCommands))
0296   print(psetString('HLTDebugRAW', HLTDebugRAW.outputCommands))
0297   print(psetString('HLTDebugFEVT', HLTDebugFEVT.outputCommands))
0298   print(psetString('HLTScouting', HLTScouting.outputCommands))
0299
0300 ###
0301 ### main
0302 ###
0303 if __name__ == '__main__':
0304
0305   # defaults of cmd-line arguments
0306   defaults = options.HLTProcessOptions()
0307
0308   parser = argparse.ArgumentParser(
0309     prog = './'+os.path.basename(__file__),
0310     formatter_class = argparse.RawDescriptionHelpFormatter,
0311     description = __doc__
0312   )
0313
0314   # required argument
0315   parser.add_argument('menu',
0316                       action  = 'store',
0317                       type    = options.ConnectionHLTMenu,
0318                       metavar = 'MENU',
0319                       help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
0320
0321   # options
0322   parser.add_argument('--dbproxy',
0323                       dest    = 'proxy',
0324                       action  = 'store_true',
0325                       default = defaults.proxy,
0326                       help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
0327   parser.add_argument('--dbproxyport',
0328                       dest    = 'proxy_port',
0329                       action  = 'store',
0330                       metavar = 'PROXYPORT',
0331                       default = defaults.proxy_port,
0332                        help    = 'Port of the socks proxy (default: 8080)' )
0333   parser.add_argument('--dbproxyhost',
0334                       dest    = 'proxy_host',
0335                       action  = 'store',
0336                       metavar = 'PROXYHOST',
0337                       default = defaults.proxy_host,
0338                       help    = 'Host of the socks proxy (default: "localhost")' )
0339
0340   # parse command line arguments and options
0341   config = parser.parse_args()
0342
0343   printHLTriggerEventContentCff(config)