L1TCalorimeter/python/convertParamsToOnlineFormat.py

0001 #!/bin/env python
0002
0003 from __future__ import print_function
0004 import argparse
0005 import FWCore.ParameterSet.Config as cms
0006 from importlib import import_module
0007 import os
0008 import sys
0009 import xml.etree.ElementTree as ET
0010
0011
0012 # Pairwise generator: returns pairs of adjacent elements in a list / other iterable
0013 def pairwiseGen(aList):
0014     for i in xrange(len(aList)-1):
0015         yield (aList[i], aList[i+1])
0016
0017 def parseOfflineLUTfile(aRelPath, aExpectedSize, aPaddingValue = None, aTruncate = False):
0018     # Find file by looking under directories listed in 'CMSSW_SEARCH_PATH' as outlined in https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideEdmFileInPath
0019     searchPaths = os.getenv('CMSSW_SEARCH_PATH').split(':')
0020     resolvedPath = None
0021     for baseDir in searchPaths:
0022         print("Looking for '" + aRelPath + "' under '" + baseDir + "'")
0023         if os.path.isfile(os.path.join(baseDir, aRelPath)):
0024             print("   success!")
0025             resolvedPath = os.path.join(baseDir, aRelPath)
0026             break
0027     if resolvedPath is None:
0028         raise RuntimeError("Could not find LUT file '" + aRelPath + "' under directories in 'CMSSW_SEARCH_PATH'")
0029
0030     with open(resolvedPath) as f:
0031         entries = []
0032         line_nr = 0
0033         for line in f:
0034             line_nr += 1
0035             # Ignore comment lines
0036             if line.startswith('#') or line == '\n':
0037                 continue
0038
0039             # Remove trailing comments from data lines
0040             stripped_line = line[:line.find('#')]
0041
0042             # Split line into list of whitespace-separated items
0043             items = stripped_line.split()
0044             if len(items) != 2:
0045                print("ERROR parsing file", resolvedPath, "on line", line_nr, "'" + line + "' : Splitting on whitespace produced", len(items), "items")
0046                sys.exit(1)
0047
0048             entries.append( (int(items[0]), int(items[1])) )
0049
0050         # Sort the LUT
0051         entries.sort(key= lambda x : x[0])
0052         # Check that the LUT is not empty
0053         if len(entries) == 0:
0054             print("ERROR parsing file", resolvedPath, ": No LUT entries defined in the file")
0055             sys.exit(1)
0056
0057         # Check that no items from the LUT are missing
0058         if entries[0][0] != 0:
0059             print("ERROR parsing file", resolvedPath, ": LUT entries before index", entries[0][0], "are not defined")
0060             sys.exit(1)
0061
0062         for x1, x2 in pairwiseGen(entries):
0063             if x1[0] != (x2[0]-1):
0064                 print("ERROR parsing file", resolvedPath, ": ", x2[0] - x1[0] - 1,"LUT entries between indices", x1[0], "and", x2[0], "are not defined")
0065                 sys.exit(1)
0066
0067         result = [x[1] for x in entries]
0068
0069         if (len(result) < aExpectedSize) and not (aPaddingValue is None):
0070             print ("WARNING : Padding", str(len(result))+"-entry LUT with value", aPaddingValue, "to have", aExpectedSize, "entries")
0071             result += ([aPaddingValue] * (aExpectedSize - len(result)))
0072         elif (len(result) > aExpectedSize) and aTruncate:
0073             print ("WARNING : Truncating", str(len(result))+"-entry LUT to have", aExpectedSize, "entries")
0074             result = result[0:aExpectedSize]
0075         elif len(result) != aExpectedSize:
0076             print ("ERROR parsing file", resolvedPath, ": Expected LUT of size", aExpectedSize, ", but", len(result), "entries were specified (and no padding/truncation requested)")
0077             sys.exit(1)
0078
0079         return result
0080
0081
0082 def getFullListOfParameters(aModule):
0083
0084     def divideByEgLsb(aParam):
0085         return int(aParam.value() / aModule.egLsb.value())
0086
0087     def divideByTauLsb(aParam):
0088         return int(aParam.value() / aModule.tauLsb.value())
0089
0090     def divideByJetLsb(aParam):
0091         return int(aParam.value() / aModule.jetLsb.value())
0092
0093
0094     result = [
0095       (('mp_common', 'sdfile'),               None,                         ''),
0096       (('mp_common', 'algoRev'),              None,                         ''),
0097       (('mp_common', 'leptonSeedThreshold'),  '2_ClusterSeedThreshold.mif', divideByEgLsb(aModule.egSeedThreshold)),
0098       (('mp_common', 'leptonTowerThreshold'), '3_ClusterThreshold.mif',     divideByEgLsb(aModule.egNeighbourThreshold)),
0099       (('mp_common', 'pileUpTowerThreshold'), '4_PileUpThreshold.mif',      0x0)
0100     ]
0101
0102     result += [
0103       (('mp_egamma', 'egammaRelaxationThreshold'),  '10_EgRelaxThr.mif',              divideByEgLsb(aModule.egMaxPtHOverE)),
0104       (('mp_egamma', 'egammaMaxEta'),               'egammaMaxEta.mif',          aModule.egEtaCut.value()),
0105       (('mp_egamma', 'egammaBypassCuts'),           'BypassEgVeto.mif',               bool(aModule.egBypassEGVetos.value())),
0106       (('mp_egamma', 'egammaBypassShape'),          'BypassEgShape.mif',              bool(aModule.egBypassShape.value())),
0107       (('mp_egamma', 'egammaBypassEcalFG'),         'BypassEcalFG.mif',               bool(aModule.egBypassECALFG.value())),
0108       (('mp_egamma', 'egammaBypassExtendedHOverE'), '_BypassExtHE.mif',               bool(aModule.egBypassExtHOverE)),
0109       (('mp_egamma', 'egammaHOverECut_iEtaLT15'),   '_RatioCutLt15.mif',              aModule.egHOverEcutBarrel.value()),
0110       (('mp_egamma', 'egammaHOverECut_iEtaGTEq15'), '_RatioCutGe15.mif',              aModule.egHOverEcutEndcap.value()),
0111       (('mp_egamma', 'egammaEnergyCalibLUT'),       'C_EgammaCalibration_12to18.mif', parseOfflineLUTfile(aModule.egCalibrationLUTFile.value(), 4096)),
0112       (('mp_egamma', 'egammaIsoLUT1'),              'D_EgammaIsolation1_13to9.mif',   parseOfflineLUTfile(aModule.egIsoLUTFile.value(), 8192)),
0113       (('mp_egamma', 'egammaIsoLUT2'),              'D_EgammaIsolation2_13to9.mif',   parseOfflineLUTfile(aModule.egIsoLUTFile2.value(), 8192))
0114     ]
0115
0116     result += [
0117       (('mp_tau', 'tauMaxEta'),         'tauMaxEta.mif',               aModule.isoTauEtaMax.value()),
0118       (('mp_tau', 'tauEnergyCalibLUT'), 'I_TauCalibration_11to18.mif', parseOfflineLUTfile(aModule.tauCalibrationLUTFile.value(), 2048, 0x0)),
0119       (('mp_tau', 'tauIsoLUT'),         'H_TauIsolation_12to9.mif',    parseOfflineLUTfile(aModule.tauIsoLUTFile.value(), 4096)),
0120       (('mp_tau', 'tauTrimmingLUT'),    'P_TauTrimming_13to8.mif',     parseOfflineLUTfile(aModule.tauTrimmingShapeVetoLUTFile.value(), 8192))
0121     ]
0122
0123     result += [
0124       (('mp_jet', 'jetSeedThreshold'),   '1_JetSeedThreshold.mif',      divideByJetLsb(aModule.jetSeedThreshold)),
0125       (('mp_jet', 'jetMaxEta'),          '6_JetEtaMax.mif',             0x00028),
0126       (('mp_jet', 'jetBypassPileUpSub'), 'BypassJetPUS.mif',            bool(aModule.jetBypassPUS.value())),
0127       (('mp_jet', 'jetPUSUsePhiRing'), 'PhiRingPUS.mif',        bool(aModule.jetPUSUsePhiRing.value())),
0128       (('mp_jet', 'jetEnergyCalibLUT'),  'L_JetCalibration_11to18.mif', parseOfflineLUTfile(aModule.jetCalibrationLUTFile.value(), 2048)),
0129       (('mp_jet', 'HTMHT_maxJetEta'),    'HTMHT_maxJetEta.mif',         aModule.etSumEtaMax[1]), # assert == etSumEtaMax[3] ?
0130       (('mp_jet', 'HT_jetThreshold'),    '8_HtThreshold.mif',           int(aModule.etSumEtThreshold[1] / aModule.etSumLsb.value())),
0131       (('mp_jet', 'MHT_jetThreshold'),   '9_MHtThreshold.mif',          int(aModule.etSumEtThreshold[3] / aModule.etSumLsb.value())),
0132     ]
0133
0134     result += [
0135       (('mp_sums', 'towerCountThreshold'),      'HeavyIonThr.mif',       int(aModule.etSumEtThreshold[4] / aModule.etSumLsb.value()) ),
0136       (('mp_sums', 'towerCountMaxEta'),         'HeavyIonEta.mif',       aModule.etSumEtaMax[4]),
0137       (('mp_sums', 'ETMET_maxTowerEta'),        'ETMET_maxTowerEta.mif', aModule.etSumEtaMax[0]), # assert == etSumEtaMax[2] ?
0138       (('mp_sums', 'ecalET_towerThresholdLUT'), 'X_EcalTHR_11to9.mif',   parseOfflineLUTfile(aModule.etSumEcalSumPUSLUTFile.value(), 2048, aTruncate = True)),
0139       (('mp_sums', 'ET_towerThresholdLUT'),     'X_ETTHR_11to9.mif',     parseOfflineLUTfile(aModule.etSumEttPUSLUTFile.value(), 2048, aTruncate = True)),
0140       (('mp_sums', 'MET_towerThresholdLUT'),    'X_METTHR_11to9.mif',    parseOfflineLUTfile(aModule.etSumMetPUSLUTFile.value(), 2048))
0141     ]
0142
0143     result += [
0144       (('demux', 'sdfile'),  None, ''),
0145       (('demux', 'algoRev'), None, 0xcafe),
0146       (('demux', 'ET_centralityLowerThresholds'), 'CentralityLowerThrs.mif', [ int(round(loBound / aModule.etSumLsb.value())) for loBound in aModule.etSumCentralityLower.value()]),
0147       (('demux', 'ET_centralityUpperThresholds'), 'CentralityUpperThrs.mif', [ int(round(upBound / aModule.etSumLsb.value())) for upBound in aModule.etSumCentralityUpper.value()]),
0148       (('demux', 'MET_energyCalibLUT'),       'M_METnoHFenergyCalibration_12to18.mif',     parseOfflineLUTfile(aModule.metCalibrationLUTFile.value(), 4096, aTruncate = True)),
0149       (('demux', 'METHF_energyCalibLUT'),     'M_METwithHFenergyCalibration_12to18.mif',   parseOfflineLUTfile(aModule.metHFCalibrationLUTFile.value(), 4096, aTruncate = True)),
0150       (('demux', 'ET_energyCalibLUT'),        'S_ETcalibration_12to18.mif',       parseOfflineLUTfile(aModule.etSumEttCalibrationLUTFile.value(), 4096, aTruncate = True)),
0151       (('demux', 'ecalET_energyCalibLUT'),    'R_EcalCalibration_12to18.mif',     parseOfflineLUTfile(aModule.etSumEcalSumCalibrationLUTFile.value(), 4096, aTruncate = True)),
0152       (('demux', 'MET_phiCalibLUT'),          'Q_METnoHFphiCalibration_12to18.mif',     parseOfflineLUTfile(aModule.metPhiCalibrationLUTFile.value(), 4096, aTruncate = True)),
0153       (('demux', 'METHF_phiCalibLUT'),        'Q_METwithHFphiCalibration_12to18.mif',   parseOfflineLUTfile(aModule.metHFPhiCalibrationLUTFile.value(), 4096, aTruncate = True)),
0154     ]
0155
0156     result = [(a, b, parseOfflineLUTfile(c.value()) if isinstance(c, cms.FileInPath) else c) for a, b, c in result]
0157
0158     return result
0159
0160
0161 def getXmlParameterMap(aModule):
0162     result = {}
0163     for xmlDetails, mifName, value in getFullListOfParameters(aModule):
0164         if xmlDetails is not None:
0165             if xmlDetails[0] in result:
0166                 result[xmlDetails[0]] += [(xmlDetails[1], value)]
0167             else:
0168                 result[xmlDetails[0]] = [(xmlDetails[1], value)]
0169
0170     return result
0171
0172
0173 def getMifParameterMap(aModule):
0174
0175     fullList = getFullListOfParameters(aModule)
0176
0177     return {mifFileName : value for (_, mifFileName, value) in fullList if mifFileName is not None}
0178
0179
0180 # Stolen from https://stackoverflow.com/questions/3095434/inserting-newlines-in-xml-file-generated-via-xml-etree-elementtree-in-python
0181 def indent(elem, level=0):
0182     i = "\n" + level*"  "
0183     if len(elem):
0184         if not elem.text or not elem.text.strip():
0185             elem.text = i + "  "
0186         if not elem.tail or not elem.tail.strip():
0187             elem.tail = i
0188         for elem in elem:
0189             indent(elem, level+1)
0190         if not elem.tail or not elem.tail.strip():
0191             elem.tail = i
0192     else:
0193         if level and (not elem.tail or not elem.tail.strip()):
0194             elem.tail = i
0195
0196 def createMIF(aFilePath, aValue):
0197     print("Writing MIF file:", aFilePath)
0198     with open(aFilePath, 'w') as f:
0199         if isinstance(aValue, bool):
0200             aValue = (1 if aValue else 0)
0201
0202         if isinstance(aValue, int):
0203             f.write( hex(aValue) )
0204         elif isinstance(aValue, list):
0205             f.write("\n".join([hex(x) for x in aValue]))
0206         else:
0207             raise RuntimeError("Do not know how to deal with parameter of type " + str(type(aValue)))
0208
0209
0210 def createXML(parameters, contextId, outputFilePath):
0211     topNode = ET.Element('algo', id='calol2')
0212     contextNode = ET.SubElement(topNode, 'context', id=contextId)
0213     for paramId, value in parameters:
0214         if isinstance(value, bool):
0215             ET.SubElement(contextNode, 'param', id=paramId, type='bool').text = str(value).lower()
0216         elif isinstance(value, int):
0217             ET.SubElement(contextNode, 'param', id=paramId, type='uint').text = "0x{0:05X}".format(value)
0218         elif isinstance(value, str):
0219             ET.SubElement(contextNode, 'param', id=paramId, type='string').text = value
0220         elif isinstance(value, list):
0221             ET.SubElement(contextNode, 'param', id=paramId, type='vector:uint').text  = "\n      " + ",\n      ".join(["0x{0:05X}".format(x) for x in value]) + "\n    "
0222         else:
0223             raise RuntimeError("Do not know how to deal with parameter '" + paramId + "' of type " + str(type(value)))
0224     indent(topNode)
0225
0226     print("Writing XML file:", outputFilePath)
0227     with open(outputFilePath, 'w') as f:
0228         f.write(ET.tostring(topNode))
0229
0230
0231
0232 if __name__ == '__main__':
0233
0234     parser = argparse.ArgumentParser()
0235
0236     parser.add_argument('params_cfi', help='Name of CMSSW cfi python file specifying the values for the calo parameters')
0237     parser.add_argument('output_dir', help='Directory for MIF/XML output files')
0238
0239     outputFormatGroup = parser.add_mutually_exclusive_group(required=True)
0240     outputFormatGroup.add_argument('--mif', action='store_true')
0241     outputFormatGroup.add_argument('--xml', action='store_true')
0242
0243     args = parser.parse_args()
0244
0245     moduleName = 'L1Trigger.L1TCalorimeter.' + args.params_cfi
0246     print("Importing calo params from module:", moduleName)
0247     caloParams = import_module(moduleName).caloStage2Params
0248
0249     print(caloParams.egCalibrationLUTFile.value())
0250     print(caloParams.egIsoLUTFile.value())
0251     print(caloParams.egIsoLUTFile2.value())
0252     os.mkdir(args.output_dir)
0253
0254     if args.mif:
0255         for fileName, value in getMifParameterMap(caloParams).items():
0256             createMIF(args.output_dir + '/' + fileName, value)
0257     else:
0258         for fileTag, paramList in getXmlParameterMap(caloParams).items():
0259             createXML(paramList, 'MainProcessor' if fileTag.startswith('mp') else 'Demux', args.output_dir + '/algo_' + fileTag + '.xml')