Configuration/test/getEventContent.py

0001 #!/usr/bin/env python3
0002 """getEventContent.py: print EventContent cff fragment of a ConfDB configuration
0003 """
0004 import argparse
0005 import subprocess
0006 import os
0007 import re
0008
0009 import FWCore.ParameterSet.Config as cms
0010 import HLTrigger.Configuration.Tools.pipe as pipe
0011 import HLTrigger.Configuration.Tools.options as options
0012
0013 def getHLTProcessBlocks(config, blocks):
0014   """return cms.Process containing the OutputModules of the HLT configuration
0015   """
0016   # cmd-line args to select HLT configuration
0017   if config.menu.run:
0018     configline = f'--runNumber {config.menu.run}'
0019   else:
0020     configline = f'--{config.menu.database} --{config.menu.version} --configName {config.menu.name}'
0021
0022   # cmd to download HLT configuration
0023   cmdline = f'hltConfigFromDB {configline}'
0024   if config.proxy:
0025     cmdline += f' --dbproxy --dbproxyhost {config.proxy_host} --dbproxyport {config.proxy_port}'
0026
0027   cmdline += ' --noedsources --noes --nopsets --noservices --nopaths --format python'
0028   cmdline += ' --blocks '+','.join({foo+'::outputCommands' for foo in blocks})
0029
0030   # load HLT configuration
0031   try:
0032     foo = {}
0033     exec(pipe.pipe(cmdline).decode(), foo)
0034   except:
0035     raise Exception(f'query did not return a valid python file:\n query="{cmdline}"')
0036
0037   ret = {}
0038   for block in blocks:
0039     key = 'block_'+block
0040     ret[key] = foo[key] if key in foo else None
0041     if ret[key] != None and not isinstance(ret[key], cms.PSet):
0042       raise Exception(f'query did not return valid HLT blocks:\n query="{cmdline}"')
0043
0044   return ret
0045
0046 def getHLTProcessBlockGroups(config, blockGroupDict):
0047   ret = {}
0048   blockDict = getHLTProcessBlocks(config, {bar for foo in blockGroupDict.values() for bar in foo})
0049   for groupName in blockGroupDict:
0050     ret[groupName] = cms.PSet()
0051     for blockKey in blockGroupDict[groupName]:
0052       blockName = 'block_'+blockKey
0053       if blockDict[blockName] != None:
0054         setattr(ret[groupName], blockName, blockDict[blockName])
0055   return ret
0056
0057 def makePSet(statements):
0058   statements = sorted(statements)
0059   block = cms.PSet(
0060     outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*', )
0061   )
0062   block.outputCommands.extend( statements )
0063   return block
0064
0065 def makePSetNoDrop(statements):
0066   statements = sorted(statements)
0067   block = cms.PSet(
0068     outputCommands = cms.untracked.vstring()
0069   )
0070   block.outputCommands.extend( statements )
0071   return block
0072
0073 def buildPSet(blocks):
0074   statements = set()
0075   for block in blocks:
0076     statements.update( statement for statement in block if statement.find('drop') != 0 )
0077   return makePSet(statements)
0078
0079 def buildPSetNoDrop(blocks):
0080   statements = set()
0081   for block in blocks:
0082     statements.update( statement for statement in block if statement.find('drop') != 0 )
0083   return makePSetNoDrop(statements)
0084
0085 def buildPSetWithoutRAWs(blocks):
0086   statements = set()
0087   for block in blocks:
0088     statements.update( statement for statement in block if statement.find('drop') != 0 and statement.find('keep FEDRawDataCollection') != 0)
0089   return makePSet(statements)
0090
0091 # customisation of AOD event content, requested by David Dagenhart
0092 def dropL1GlobalTriggerObjectMapRecord(block):
0093   """drop the old L1GlobalTriggerObjectMapRecord data format from the block (meant for the AOD data tier)"""
0094   try:
0095     # look for the hltL1GtObjectMap keep statement
0096     position = block.outputCommands.index('keep *_hltL1GtObjectMap_*_*')
0097   except ValueError:
0098     pass
0099   else:
0100     # add just after it a drop statement for the old data format
0101     block.outputCommands.insert(position  + 1, 'drop L1GlobalTriggerObjectMapRecord_hltL1GtObjectMap_*_*')
0102
0103 def printHLTriggerEventContentCff(process):
0104
0105   blockGroups = getHLTProcessBlockGroups(config, {
0106     'hltOutputA_cff': [
0107       'hltOutputA',
0108       'hltOutputPhysicsCommissioning',
0109     ],
0110     'hltOutputALCA_cff': [
0111       'hltOutputALCAHcalIsoTrk',
0112       'hltOutputALCALowPtJet',
0113       'hltOutputALCALumiPixelsCountsExpress',
0114       'hltOutputALCALumiPixelsCountsPrompt',
0115       'hltOutputALCAP0',
0116       'hltOutputALCAPHISYM',
0117       'hltOutputALCAPPSExpress',
0118       'hltOutputALCAPPSPrompt',
0119     ],
0120     'hltOutputMON_cff': [
0121       'hltOutputA',
0122       'hltOutputPhysicsCommissioning',
0123       'hltOutputDQM',
0124       'hltOutputHLTMonitor',
0125       'hltOutputReleaseValidation',
0126     ],
0127     'hltOutputScouting_cff': [
0128       'hltOutputScoutingPF',
0129     ],
0130   })
0131
0132   hltOutputA_cff = blockGroups['hltOutputA_cff']
0133   hltOutputALCA_cff = blockGroups['hltOutputALCA_cff']
0134   hltOutputMON_cff = blockGroups['hltOutputMON_cff']
0135   hltOutputScouting_cff = blockGroups['hltOutputScouting_cff']
0136
0137   # hltDebugOutput
0138
0139   if not hasattr(hltOutputMON_cff,'block_hltOutputA'):
0140     hltOutputMON_cff.block_hltOutputA = hltOutputMON_cff.block_hltOutputPhysicsCommissioning
0141   if not hasattr(hltOutputMON_cff,'block_hltOutputDQM'):
0142     hltOutputMON_cff.block_hltOutputDQM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0143   if not hasattr(hltOutputMON_cff,'block_hltOutputHLTMonitor'):
0144     hltOutputMON_cff.block_hltOutputHLTMonitor = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *_hlt*_*_*' ))
0145   if not hasattr(hltOutputMON_cff,'block_hltOutputReleaseValidation'):
0146     hltOutputMON_cff.block_hltOutputReleaseValidation = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0147
0148   hltDebugOutputBlocks = (
0149     # the DQM and HLTMON streams have the HLT debug outputs used online
0150     hltOutputMON_cff.block_hltOutputA.outputCommands,
0151     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0152     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0153     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0154   )
0155   hltDebugOutputContent = buildPSet(hltDebugOutputBlocks)
0156
0157   # hltDebugWithAlCaOutput
0158
0159   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAHcalIsoTrk'):
0160     hltOutputALCA_cff.block_hltOutputALCAHcalIsoTrk = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0161   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALowPtJet'):
0162     hltOutputALCA_cff.block_hltOutputALCALowPtJet = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0163   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsExpress'):
0164     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0165   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCALumiPixelsCountsPrompt'):
0166     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0167   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAP0'):
0168     hltOutputALCA_cff.block_hltOutputALCAP0 = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0169   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPHISYM'):
0170     hltOutputALCA_cff.block_hltOutputALCAPHISYM = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0171   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSExpress'):
0172     hltOutputALCA_cff.block_hltOutputALCAPPSExpress = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0173   if not hasattr(hltOutputALCA_cff,'block_hltOutputALCAPPSPrompt'):
0174     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0175
0176   hltDebugWithAlCaOutputBlocks = (
0177     # the DQM and HLTMON streams have the HLT debug outputs used online
0178     hltOutputMON_cff.block_hltOutputA.outputCommands,
0179     hltOutputMON_cff.block_hltOutputDQM.outputCommands,
0180     hltOutputMON_cff.block_hltOutputHLTMonitor.outputCommands,
0181     hltOutputMON_cff.block_hltOutputReleaseValidation.outputCommands,
0182     # the ALCA streams have the AlCa outputs
0183     hltOutputALCA_cff.block_hltOutputALCAHcalIsoTrk.outputCommands,
0184     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsExpress.outputCommands,
0185     hltOutputALCA_cff.block_hltOutputALCALumiPixelsCountsPrompt.outputCommands,
0186     hltOutputALCA_cff.block_hltOutputALCAP0.outputCommands,
0187     hltOutputALCA_cff.block_hltOutputALCAPHISYM.outputCommands,
0188     hltOutputALCA_cff.block_hltOutputALCAPPSExpress.outputCommands,
0189     hltOutputALCA_cff.block_hltOutputALCAPPSPrompt.outputCommands,
0190     hltOutputALCA_cff.block_hltOutputALCALowPtJet.outputCommands,
0191   )
0192   hltDebugWithAlCaOutputContent = buildPSet(hltDebugWithAlCaOutputBlocks)
0193
0194   # hltScoutingOutput
0195
0196   if not hasattr(hltOutputScouting_cff,'block_hltOutputScoutingPF'):
0197     hltOutputScouting_cff.block_hltOutputScoutingPF = cms.PSet(outputCommands = cms.untracked.vstring( 'drop *' ))
0198
0199   hltScoutingOutputBlocks = (
0200     # the Scouting streams have the Scouting outputs
0201     hltOutputScouting_cff.block_hltOutputScoutingPF.outputCommands,
0202   )
0203   hltScoutingOutputContent = buildPSet(hltScoutingOutputBlocks)
0204   hltScoutingOutputContentNoDrop = buildPSetNoDrop(hltScoutingOutputBlocks)
0205
0206   # hltDefaultOutput
0207   if not hasattr(hltOutputA_cff,'block_hltOutputA'):
0208     hltOutputA_cff.block_hltOutputA = hltOutputA_cff.block_hltOutputPhysicsCommissioning
0209   hltDefaultOutputBlocks = (
0210     # the A stream has the HLT default output, with FEDs - strip out the FEDRawDataCollection keep statements for hltDefaultOutput
0211     hltOutputA_cff.block_hltOutputA.outputCommands,
0212   )
0213   hltDefaultOutputContent         = buildPSetWithoutRAWs(hltDefaultOutputBlocks)
0214   hltDefaultOutputWithFEDsContent = buildPSet(hltDefaultOutputBlocks)
0215
0216   # define the CMSSW default event content configurations
0217
0218   # RAW event content
0219   HLTriggerRAW = cms.PSet(
0220       outputCommands = cms.vstring()
0221   )
0222   HLTriggerRAW.outputCommands.extend(hltDefaultOutputWithFEDsContent.outputCommands)
0223   HLTriggerRAW.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0224
0225   # RECO event content
0226   HLTriggerRECO = cms.PSet(
0227       outputCommands = cms.vstring()
0228   )
0229   HLTriggerRECO.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0230   HLTriggerRECO.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0231
0232   # AOD event content
0233   HLTriggerAOD = cms.PSet(
0234       outputCommands = cms.vstring()
0235   )
0236   HLTriggerAOD.outputCommands.extend(hltDefaultOutputContent.outputCommands)
0237   HLTriggerAOD.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0238   dropL1GlobalTriggerObjectMapRecord(HLTriggerAOD)
0239
0240   # MINIAOD event content
0241   HLTriggerMINIAOD = cms.PSet(
0242       outputCommands = cms.vstring()
0243   )
0244   HLTriggerMINIAOD.outputCommands.extend(hltScoutingOutputContent.outputCommands)
0245
0246   # HLTDEBUG RAW event content
0247   HLTDebugRAW = cms.PSet(
0248       outputCommands = cms.vstring()
0249   )
0250   HLTDebugRAW.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0251   HLTDebugRAW.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0252
0253   # HLTDEBUG FEVT event content
0254   HLTDebugFEVT = cms.PSet(
0255       outputCommands = cms.vstring()
0256   )
0257   HLTDebugFEVT.outputCommands.extend(hltDebugWithAlCaOutputContent.outputCommands)
0258   HLTDebugFEVT.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0259
0260   # Scouting event content
0261   HLTScouting = cms.PSet(
0262       outputCommands = cms.vstring()
0263   )
0264   HLTScouting.outputCommands.extend(hltScoutingOutputContentNoDrop.outputCommands)
0265
0266   # print the expanded event content configurations to stdout
0267   print('''import FWCore.ParameterSet.Config as cms
0268
0269 # EventContent for HLT-related products.
0270
0271 # This file exports the following EventContent blocks:
0272 #   HLTrigger(RAW|RECO|AOD|MINIAOD) [without DEBUG products]
0273 #   HLTDebug(RAW|FEVT)              [with    DEBUG products]
0274 #   HLTScouting                     [only Scouting products]
0275 #
0276 # as these are used in Configuration/EventContent
0277 #''')
0278
0279   def psetString(name, outputCommands):
0280       str_products = ',\n'.join(f'        \'{str_cmd}\'' for str_cmd in outputCommands)
0281       return f'\n{name} = cms.PSet(\n    outputCommands = cms.vstring( *(\n{str_products}\n    ) )\n)'
0282
0283   print(psetString('HLTriggerRAW', HLTriggerRAW.outputCommands))
0284   print(psetString('HLTriggerRECO', HLTriggerRECO.outputCommands))
0285   print(psetString('HLTriggerAOD', HLTriggerAOD.outputCommands))
0286   print(psetString('HLTriggerMINIAOD', HLTriggerMINIAOD.outputCommands))
0287   print(psetString('HLTDebugRAW', HLTDebugRAW.outputCommands))
0288   print(psetString('HLTDebugFEVT', HLTDebugFEVT.outputCommands))
0289   print(psetString('HLTScouting', HLTScouting.outputCommands))
0290
0291 ###
0292 ### main
0293 ###
0294 if __name__ == '__main__':
0295
0296   # defaults of cmd-line arguments
0297   defaults = options.HLTProcessOptions()
0298
0299   parser = argparse.ArgumentParser(
0300     prog = './'+os.path.basename(__file__),
0301     formatter_class = argparse.RawDescriptionHelpFormatter,
0302     description = __doc__
0303   )
0304
0305   # required argument
0306   parser.add_argument('menu',
0307                       action  = 'store',
0308                       type    = options.ConnectionHLTMenu,
0309                       metavar = 'MENU',
0310                       help    = 'HLT menu to dump from the database. Supported formats are:\n  - /path/to/configuration[/Vn]\n  - [[{v1|v2|v3}/]{run3|run2|online|adg}:]/path/to/configuration[/Vn]\n  - run:runnumber\nThe possible converters are "v1", "v2, and "v3" (default).\nThe possible databases are "run3" (default, used for offline development), "run2" (used for accessing run2 offline development menus), "online" (used to extract online menus within Point 5) and "adg" (used to extract the online menus outside Point 5).\nIf no menu version is specified, the latest one is automatically used.\nIf "run:" is used instead, the HLT menu used for the given run number is looked up and used.\nNote other converters and databases exist as options but they are only for expert/special use.' )
0311
0312   # options
0313   parser.add_argument('--dbproxy',
0314                       dest    = 'proxy',
0315                       action  = 'store_true',
0316                       default = defaults.proxy,
0317                       help    = 'Use a socks proxy to connect outside CERN network (default: False)' )
0318   parser.add_argument('--dbproxyport',
0319                       dest    = 'proxy_port',
0320                       action  = 'store',
0321                       metavar = 'PROXYPORT',
0322                       default = defaults.proxy_port,
0323                        help    = 'Port of the socks proxy (default: 8080)' )
0324   parser.add_argument('--dbproxyhost',
0325                       dest    = 'proxy_host',
0326                       action  = 'store',
0327                       metavar = 'PROXYHOST',
0328                       default = defaults.proxy_host,
0329                       help    = 'Host of the socks proxy (default: "localhost")' )
0330
0331   # parse command line arguments and options
0332   config = parser.parse_args()
0333
0334   printHLTriggerEventContentCff(config)