File indexing completed on 2024-12-17 02:43:54
0001
0002 class ModuleInfo(object):
0003 def __init__(self, label, type_):
0004 self._label = label
0005 self._type = type_
0006 self._streamInfo = dict()
0007 self._eventInfo = list()
0008 def __repr__(self):
0009 return self._label+" "+self._type+" "+str(self._eventInfo)
0010 class ModuleCall(object):
0011 def __init__(self, data):
0012 self._temp = data[0]
0013 self._nTemp = data[1]
0014 self._unmatched = data[2]
0015 self._nUnmatched = data[3]
0016 self._new = data[4]
0017 self._nNew = data[5]
0018
0019 class ModuleEventInfo(object):
0020 def __init__(self, modCall, data, streamID):
0021 self._temp = modCall._temp
0022 self._nTemp = modCall._nTemp
0023 self._unmatched = modCall._unmatched
0024 self._nUnmatched = modCall._nUnmatched
0025 self._dataProdAlloc = data[0]
0026 self._nDataProdAlloc = data[1]
0027 self._new = modCall._new - self._dataProdAlloc
0028 self._nNew = modCall._nNew - self._nDataProdAlloc
0029 self._streamID = streamID
0030 def __repr__(self):
0031 return "temp("+str(self._temp)+","+str(self._nTemp)+") un("+str(self._unmatched)+","+str(self._nUnmatched)+") prod("+str(self._dataProdAlloc)+","+str(self._nDataProdAlloc)+") new("+str(self._new)+","+str(self._nNew)+")"
0032 class FileParser(object):
0033 def __init__(self):
0034 self.modules = dict()
0035 self.acquires = dict()
0036 self.nStreams = 0
0037 pass
0038 def parse(self,file):
0039 for l in file:
0040 self._parseLine(l[:-1])
0041 def _parseLine(self, line):
0042 if len(line) == 0:
0043 return
0044 if line[0] == '#':
0045 return
0046 if line[0] == '@':
0047 self.addModule(line[2:])
0048 if line[0] == 'M':
0049 self.moduleCall(line[2:])
0050 if line[0] == 'A':
0051 self.moduleAcquireCall(line[2:])
0052 if line[0] == 'D':
0053 self.eventDeallocCall(line[2:])
0054 def addModule(self, l):
0055 d = l.split(" ")
0056 name, type_, index = d[0:3]
0057 self.modules[int(index)] = ModuleInfo(name,type_)
0058 def moduleCall(self, l):
0059 d = [int(x) for x in l.split(" ")]
0060 m = ModuleCall(d[2:])
0061 moduleName, streamID = d[0:2]
0062 if streamID+1 > self.nStreams:
0063 self.nStreams = streamID + 1
0064 self.modules[moduleName]._streamInfo[streamID] = m
0065 if d[0] in self.acquires:
0066 a = self.acquires[moduleName][streamID]
0067 m._temp += a._temp
0068 m._nTemp += a._nTemp
0069 m._unmatched += a._unmatched
0070 m._nUnmatched += a._nUnmatched
0071 def moduleAcquireCall(self,l):
0072 d = [int(x) for x in l.split(" ")]
0073 moduleName, streamID = d[0:2]
0074 m = self.modules[moduleName]
0075 self.acquires.setdefault(moduleName,dict())[streamID] = ModuleCall(d[2:])
0076 pass
0077 def eventDeallocCall(self,l):
0078 d = [int(x) for x in l.split(" ")]
0079 moduleName, streamID = d[0:2]
0080 streamInfo = self.modules[moduleName]._streamInfo[streamID]
0081 del self.modules[moduleName]._streamInfo[streamID]
0082 self.modules[moduleName]._eventInfo.append(ModuleEventInfo(streamInfo,d[2:],streamID))
0083
0084 def reportModulesWithMemoryGrowth(fileParser, showEvents):
0085 ret = []
0086 for m in fileParser.modules.values():
0087 mem = 0
0088 if len(m._eventInfo):
0089 l = list()
0090 previousNewInStream = [0]*fileParser.nStreams
0091
0092 for e in m._eventInfo:
0093 l.append((previousNewInStream[e._streamID], e._unmatched))
0094 mem += previousNewInStream[e._streamID] - e._unmatched
0095 previousNewInStream[e._streamID] = e._new
0096 if mem and mem > m._eventInfo[0]._new:
0097 increment = []
0098 for n,u in l:
0099 if len(increment):
0100 increment[-1] -= u
0101 increment.append(n)
0102 if showEvents:
0103 ret.append((m._label,m._type, mem,increment))
0104 else:
0105 ret.append((m._label, m._type, mem))
0106 return ret
0107
0108 def reportModuleDataProductMemory(fileParser, showEvents):
0109 ret = []
0110 for m in fileParser.modules.values():
0111 l = list()
0112 retained = False
0113 sum = 0
0114 for e in m._eventInfo:
0115 l.append(e._dataProdAlloc)
0116 if e._dataProdAlloc > 0:
0117 sum += e._dataProdAlloc
0118 retained = True
0119 if retained:
0120 if showEvents:
0121 ret.append((m._label, m._type, float(sum)/len(l), l))
0122 else:
0123 ret.append((m._label, m._type, float(sum)/len(l)))
0124 return ret
0125
0126 def reportModuleRetainingMemory(fileParser, showEvents):
0127 ret =[]
0128 for m in fileParser.modules.values():
0129 l = list()
0130 retained = False
0131 sum = 0
0132 for e in m._eventInfo[1:]:
0133 l.append(e._new)
0134 if e._new > 0:
0135 sum += e._new
0136 retained = True
0137 if retained:
0138 if showEvents:
0139 ret.append((m._label, m._type, float(sum)/len(l), l))
0140 else:
0141 ret.append((m._label, m._type, float(sum)/len(l)))
0142 return ret
0143
0144 def reportModuleTemporary(fileParser, showEvents):
0145 ret = []
0146 for m in fileParser.modules.values():
0147 l = list()
0148 retained = False
0149 sum = 0
0150 for e in m._eventInfo:
0151 l.append(e._temp)
0152 if e._temp > 0:
0153 sum += e._temp
0154 retained = True
0155 if retained:
0156 if showEvents:
0157 ret.append((m._label, m._type, float(sum)/len(l), l))
0158 else:
0159 ret.append((m._label, m._type, float(sum)/len(l)))
0160 return ret
0161
0162 def reportModuleNTemporary(fileParser, showEvents):
0163 ret = []
0164 for m in fileParser.modules.values():
0165 l = list()
0166 retained = False
0167 sum = 0
0168 for e in m._eventInfo:
0169 l.append(e._nTemp)
0170 if e._temp > 0:
0171 sum += e._nTemp
0172 retained = True
0173 if retained:
0174 if showEvents:
0175 ret.append((m._label, m._type, float(sum)/len(l), l))
0176 else:
0177 ret.append((m._label, m._type, float(sum)/len(l)))
0178 return ret
0179
0180
0181 def printReport(values, showEvents, summary, eventSummary, maxColumn):
0182 values.sort(key=lambda x: x[2])
0183 label = "module label"
0184 classType = "module class type"
0185 if maxColumn == 0:
0186 columnWidths = [len(label),len(classType),len(summary)]
0187 for v in values:
0188 for c in (0,1,2):
0189 if c == 2:
0190 width = len(f"{v[c]}:.2f")
0191 else:
0192 width = len(v[c])
0193 if width > columnWidths[c]:
0194 columnWidths[c] = width
0195 else:
0196 columnWidths = [maxColumn, maxColumn, maxColumn]
0197 label = label[:maxColumn]
0198 classType = classType[:maxColumn]
0199 print(f"{label:{columnWidths[0]}} {classType:{columnWidths[1]}} {summary:{columnWidths[2]}}")
0200 if showEvents:
0201 print(f" [{eventSummary}]")
0202
0203 for v in values:
0204 label = v[0]
0205 classType = v[1]
0206 if maxColumn:
0207 label = label[:maxColumn]
0208 classType = classType[:maxColumn]
0209 print(f"{label:{columnWidths[0]}} {classType:{columnWidths[1]}} {v[2]:{columnWidths[2]}.2f}")
0210 if showEvents:
0211 print(f" {v[3]}")
0212
0213 if __name__=="__main__":
0214 import argparse
0215
0216 parser = argparse.ArgumentParser(description='Parses files generated from ModuleEventAlloc service')
0217 parser.add_argument('filename',
0218 type=argparse.FileType('r'),
0219 help='file to process')
0220 parser.add_argument('--grew', help='report which modules retained more memory over the job', action='store_true')
0221 parser.add_argument('--retained', help='report which modules retained memory between events (might be deleted at begin of next event)', action='store_true')
0222 parser.add_argument('--product', help="report how much memory each module put into the Event as data products", action='store_true')
0223 parser.add_argument('--tempSize', help="report how much temporary allocated memory each module used when processing the Event", action='store_true')
0224 parser.add_argument('--nTemp', help="report number of temporary allocations each module used when processing the Event", action='store_true')
0225
0226 parser.add_argument('--eventData', help='for each report, show the per event data associated to the report', action='store_true')
0227 parser.add_argument('--maxColumn', type=int, help='maximum column width for report, 0 for no constraint', default=0)
0228 args = parser.parse_args()
0229
0230 inputfile = args.filename
0231
0232 fileParser = FileParser()
0233 fileParser.parse(inputfile)
0234
0235 if args.grew:
0236 printReport(reportModulesWithMemoryGrowth(fileParser, args.eventData), args.eventData, "total memory growth", "growth each event", args.maxColumn)
0237 if args.retained:
0238 printReport(reportModuleRetainingMemory(fileParser, args.eventData), args.eventData, "average retained", "retained each event", args.maxColumn)
0239 if args.product:
0240 printReport(reportModuleDataProductMemory(fileParser, args.eventData), args.eventData, "average data products size", "data products size each event", args.maxColumn)
0241 if args.tempSize:
0242 printReport(reportModuleTemporary(fileParser, args.eventData), args.eventData, "average temporary allocation size", "temporary allocation size each event", args.maxColumn)
0243 if args.nTemp:
0244 printReport(reportModuleNTemporary(fileParser, args.eventData), args.eventData, "average # of temporary allocation", "# of temporary allocations each event", args.maxColumn)
0245
