File indexing completed on 2024-11-09 02:40:54
0001
0002 from builtins import range
0003 from itertools import groupby
0004 from operator import attrgetter,itemgetter
0005 import sys
0006 import json
0007 from collections import defaultdict
0008
0009 def printHelp():
0010 s = '''
0011 To Use: Add the ModuleAllocMonitor Service to the cmsRun job use something like this
0012 in the configuration:
0013
0014 process.add_(cms.Service("ModuleAllocMonitor", fileName = cms.untracked.string("moduleAlloc.log")))
0015
0016 After running the job, execute this script and pass the name of the
0017 ModuleAllocMonitor log file to the script.
0018
0019 This script will output a more human readable form of the data in the log file.'''
0020 return s
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040
0041
0042
0043
0044
0045
0046
0047 kMicroToSec = 0.000001
0048
0049 kSourceFindEvent = "sourceFindEvent"
0050 kSourceDelayedRead ="sourceDelayedRead"
0051
0052 kLargestLumiNumber = 4294967295
0053
0054
0055 class Phase (object):
0056 destruction = -16
0057 endJob = -12
0058 endStream = -11
0059 writeProcessBlock = -10
0060 endProcessBlock = -9
0061 globalWriteRun = -7
0062 globalEndRun = -6
0063 streamEndRun = -5
0064 globalWriteLumi = -4
0065 globalEndLumi = -3
0066 streamEndLumi = -2
0067 clearEvent = -1
0068 Event = 0
0069 streamBeginLumi = 2
0070 globalBeginLumi = 3
0071 streamBeginRun = 5
0072 globalBeginRun = 6
0073 accessInputProcessBlock = 8
0074 beginProcessBlock = 9
0075 openFile = 10
0076 beginStream = 11
0077 beginJob = 12
0078 esSync = 13
0079 esSyncEnqueue = 14
0080 getNextTransition = 15
0081 construction = 16
0082 startTracing = 17
0083
0084
0085 class Activity (object):
0086 prefetch = 0
0087 acquire = 1
0088 process = 2
0089 delayedGet = 3
0090 externalWork = 4
0091 temporary = 100
0092
0093 activityNames_ = { Activity.prefetch : 'prefetch',
0094 Activity.acquire : 'acquire',
0095 Activity.process : 'process',
0096 Activity.delayedGet : 'delayedGet',
0097 Activity.externalWork : 'externalWork' }
0098
0099 def activityName(activity):
0100 return activityNames_[activity]
0101
0102 transitionToNames_ = {
0103 Phase.startTracing: 'start tracing',
0104 Phase.construction: 'construction',
0105 Phase.destruction: 'destruction',
0106 Phase.beginJob: 'begin job',
0107 Phase.endJob: 'end job',
0108 Phase.beginStream: 'begin stream',
0109 Phase.endStream: 'end stream',
0110 Phase.beginProcessBlock: 'begin process block',
0111 Phase.endProcessBlock: 'end process block',
0112 Phase.accessInputProcessBlock: 'access input process block',
0113 Phase.writeProcessBlock: 'write process block',
0114 Phase.globalBeginRun: 'global begin run',
0115 Phase.globalEndRun: 'global end run',
0116 Phase.globalWriteRun: 'global write run',
0117 Phase.streamBeginRun: 'stream begin run',
0118 Phase.streamEndRun: 'stream end run',
0119 Phase.globalBeginLumi: 'global begin lumi',
0120 Phase.globalEndLumi: 'global end lumi',
0121 Phase.globalWriteLumi: 'global write lumi',
0122 Phase.streamBeginLumi: 'stream begin lumi',
0123 Phase.streamEndLumi: 'stream end lumi',
0124 Phase.esSyncEnqueue: 'EventSetup synchronization',
0125 Phase.esSync: 'EventSetup synchronization',
0126 Phase.Event: 'event',
0127 Phase.clearEvent: 'clear event',
0128 Phase.getNextTransition: 'get next transition',
0129 Phase.openFile : "open file"
0130 }
0131
0132 def transitionName(transition):
0133 return transitionToNames_[transition]
0134
0135 transitionToIndent_ = {
0136 Phase.startTracing: 0,
0137 Phase.construction: 0,
0138 Phase.destruction: 0,
0139 Phase.endJob: 0,
0140 Phase.beginJob: 0,
0141 Phase.beginStream: 0,
0142 Phase.endStream: 0,
0143 Phase.beginProcessBlock: 1,
0144 Phase.endProcessBlock: 1,
0145 Phase.accessInputProcessBlock: 1,
0146 Phase.writeProcessBlock: 1,
0147 Phase.globalBeginRun: 1,
0148 Phase.globalEndRun: 1,
0149 Phase.globalWriteRun: 1,
0150 Phase.streamBeginRun: 1,
0151 Phase.streamEndRun: 1,
0152 Phase.globalBeginLumi: 2,
0153 Phase.globalEndLumi: 2,
0154 Phase.globalWriteLumi: 2,
0155 Phase.streamBeginLumi: 2,
0156 Phase.streamEndLumi: 2,
0157 Phase.Event: 3,
0158 Phase.clearEvent: 3,
0159 Phase.esSyncEnqueue: 1,
0160 Phase.esSync: 1,
0161 Phase.getNextTransition: 1
0162 }
0163 def transitionIndentLevel(transition):
0164 return transitionToIndent_[transition]
0165
0166 globalTransitions_ = {
0167 Phase.startTracing,
0168 Phase.construction,
0169 Phase.destruction,
0170 Phase.endJob,
0171 Phase.beginJob,
0172 Phase.beginProcessBlock,
0173 Phase.endProcessBlock,
0174 Phase.accessInputProcessBlock,
0175 Phase.writeProcessBlock,
0176 Phase.globalBeginRun,
0177 Phase.globalEndRun,
0178 Phase.globalWriteRun,
0179 Phase.globalBeginLumi,
0180 Phase.globalEndLumi,
0181 Phase.globalWriteLumi,
0182 Phase.esSyncEnqueue,
0183 Phase.esSync,
0184 Phase.getNextTransition,
0185 Phase.openFile
0186 }
0187 def transitionIsGlobal(transition):
0188 return transition in globalTransitions_;
0189
0190 def textPrefix_(time, indentLevel):
0191
0192 return f'{time:>11} '+"++"*indentLevel
0193
0194 class AllocInfo(object):
0195 def __init__(self,payload):
0196 self.nAllocs = int(payload[0])
0197 self.nDeallocs = int(payload[1])
0198 self.added = int(payload[2])
0199 self.minTemp = int(payload[3])
0200 self.maxTemp = int(payload[4])
0201 self.max1Alloc = int(payload[5])
0202 def inject(self, transition):
0203 transition["nAllocs"]=self.nAllocs
0204 transition["nDeallocs"]=self.nDeallocs
0205 transition["added"]=self.added
0206 transition["minTemp"]=self.minTemp
0207 transition["maxTemp"]=self.maxTemp
0208 transition["max1Alloc"]=self.max1Alloc
0209 def __repr__(self):
0210 return "{{'nAlloc': {}, 'nDealloc': {}, 'added': {}, 'minTemp': {}, 'maxTemp': {}, 'max1Alloc': {} }}".format(self.nAllocs, self.nDeallocs, self.added, self.minTemp, self.maxTemp, self.max1Alloc)
0211 def toSimpleDict(self):
0212 return {'nAlloc' : self.nAllocs, 'nDealloc' :self.nDeallocs, 'added' : self.added, 'minTemp' : self.minTemp, 'maxTemp' : self.maxTemp, 'max1Alloc' : self.max1Alloc }
0213
0214 class SyncValues(object):
0215 def __init__(self):
0216 self._runs = []
0217 self._lumis = []
0218 self._streams = []
0219 def setRun(self, index, runNumber):
0220 while len(self._runs) <= index:
0221 self._runs.append(0)
0222 self._runs[index] = runNumber
0223 def runFor(self,index):
0224 return self._runs[index]
0225 def setLumi(self, index, runNumber, lumiNumber):
0226 while len(self._lumis) <= index:
0227 self._lumis.append((0,0))
0228 self._lumis[index] = (runNumber, lumiNumber)
0229 def lumiFor(self, index):
0230 return self._lumis[index]
0231 def setStream(self, index, runNumber, lumiNumber, eventNumber):
0232 while len(self._streams) <= index:
0233 self._streams.append((0,0,0))
0234 self._streams[index] = (runNumber, lumiNumber, eventNumber)
0235 def streamFor(self, index):
0236 return self._streams[index]
0237 def get(self, transition, index):
0238 if transition == Phase.construction or transition == Phase.destruction:
0239 return ()
0240 if transition == Phase.beginJob or transition == Phase.endJob or transition == Phase.openFile:
0241 return ()
0242 if transition == Phase.globalBeginRun or transition == Phase.globalEndRun or transition == Phase.globalWriteRun:
0243 return (self.runFor(index),)
0244 if transition == Phase.globalBeginLumi or transition == Phase.globalEndLumi or transition == Phase.globalWriteLumi:
0245 return self.lumiFor(index)
0246 if transition == Phase.getNextTransition:
0247 return ()
0248 if transition == Phase.writeProcessBlock:
0249 return ()
0250 if transition == Phase.beginStream:
0251 self.setStream(index, 0,0,0)
0252 return ()
0253 if not transitionIsGlobal(transition):
0254 return self.streamFor(index)
0255 raise RuntimeError("Unknown transition {}".format(transition))
0256
0257 class TempModuleTransitionInfos(object):
0258 def __init__(self):
0259 self._times = {}
0260 self._esTimes = {}
0261 def insertTime(self, label, transition, index, time):
0262 self._times[(label, transition, index)] = time
0263 def findTime(self, label, transition, index):
0264 time = self._times[(label, transition, index)]
0265 del self._times[(label, transition, index)]
0266 return time
0267 def insertTimeES(self, label, transition, index, record, call, time):
0268 self._esTimes[(label, transition, index, record, call)] = time
0269 def findTimeES(self, label, transition, index, record, call):
0270 time = self._esTimes[(label, transition, index, record, call)]
0271 del self._esTimes[(label, transition, index, record, call)]
0272 return time
0273
0274 class ModuleData(object):
0275 def __init__(self, start, stop, transition, sync, activity, allocInfo, recordName=None, callID=None):
0276 self.timeRange = (start, stop)
0277 self.transition = transition
0278 self.sync = sync
0279 self.activity = activity
0280 self.allocInfo = allocInfo
0281 self.record = (recordName, callID)
0282 def __repr__(self):
0283 if self.record[0]:
0284 return "{{ 'timeRange': {}, 'transition': {}, 'sync' :{}, 'activity':{}, 'record': {{'name' : {}, 'callID' :{} }}, 'alloc':{} }}".format(self.timeRange, self.transition, self.sync, self.activity, self.record[0], self.record[1], self.allocInfo)
0285
0286 return "{{ 'timeRange': {}, 'transition': {}, 'sync' :{}, 'activity':{}, 'alloc':{} }}".format(self.timeRange, self.transition, self.sync, self.activity, self.allocInfo)
0287 def syncToSimpleDict(self):
0288 if len(self.sync) == 0:
0289 return self.sync
0290 if len(self.sync) == 1:
0291 return {'run' : self.sync[0]}
0292 if len(self.sync) == 2:
0293 return {'run' : self.sync[0], 'lumi' : self.sync[1] }
0294 return {'run' : self.sync[0], 'lumi' : self.sync[1], 'event' : self.sync[2] }
0295 def toSimpleDict(self) :
0296 if self.record[0]:
0297 return {'timeRange': self.timeRange, 'transition': transitionName(self.transition), 'sync' : self.syncToSimpleDict(), 'activity' : activityName(self.activity), 'record' :{'name': self.record[0], 'callID' : self.record[1] }, 'alloc' : self.allocInfo.toSimpleDict() }
0298 return {'timeRange': self.timeRange, 'transition': transitionName(self.transition), 'sync' : self.syncToSimpleDict(), 'activity': activityName(self.activity), 'alloc' : self.allocInfo.toSimpleDict() }
0299
0300
0301 class ModuleCentricModuleData(object):
0302 def __init__(self):
0303 self._data = {}
0304 self._last = {}
0305 self._startTime = None
0306 def setStartTime(self, time):
0307 self._startTime = time
0308 def insert(self, label, start, stop, transition, index, sync, activity, allocInfo, recordName=None, callID=None):
0309 if label not in self._data:
0310 self._data[label] = []
0311 self._data[label].append(ModuleData(start, stop, transition, sync, activity, allocInfo, recordName, callID))
0312 self._last[(label, transition, index, activity)] = self._data[label][-1]
0313 def findLast(self, label, transition, index, activity):
0314 return self._last[(label, transition, index, activity)]
0315 def __repr__(self):
0316 return str(self._data)
0317 def data(self):
0318 return self._data
0319 def toSimpleDict(self):
0320 dct = {'startedMonitoring': self._startTime, 'source' :[], 'clearEvent': [], 'modules' :{}}
0321 modules = dct['modules']
0322 for m,lst in self._data.items():
0323 l = None
0324 if m == 'source':
0325 l = dct['source']
0326 elif m == 'clearEvent':
0327 l = dct['clearEvent']
0328 else:
0329 modules[m]=[]
0330 l = modules[m]
0331 for d in lst:
0332 l.append( d.toSimpleDict() )
0333 return dct
0334 def sortModulesBy(self, attribute):
0335 data = []
0336 for m, lst in self._data.items():
0337 data.append((m, max(lst, key = lambda x: getattr(x.allocInfo,attribute))) )
0338 data.sort( key = lambda x: getattr(x[1].allocInfo,attribute), reverse=True)
0339 return list(map(lambda x: (x[0], x[1].toSimpleDict()), data))
0340
0341 class TemporalModuleData(object):
0342 def __init__(self):
0343 self._data = []
0344 self._last = {}
0345 self._startTime = None
0346 def setStartTime(self, time):
0347 self._startTime = time
0348 def insert(self, label, start, stop, transition, index, sync, activity, allocInfo, recordName=None, callID=None):
0349 self._data.append((label, ModuleData(start, stop, transition, sync, activity, allocInfo, recordName, callID)))
0350 self._last[(label,transition, index, activity)] = self._data[-1]
0351 def findLast(self, label, transition,index, activity):
0352 v = self._last.get((label, transition, index, activity), None)
0353 if v:
0354 return v[1]
0355 return None
0356 def __repr__(self):
0357 return str(self._data)
0358 def data(self):
0359 return self._data
0360 def toSimpleDict(self):
0361 dct = {'startedMonitoring': self._startTime, 'measurements' :[]}
0362 measurements = dct['measurements']
0363 for d in self._data:
0364 entry = d[1].toSimpleDict()
0365 entry['label'] = d[0]
0366 measurements.append(entry)
0367 return dct
0368
0369
0370
0371
0372 class FrameworkTransitionParser (object):
0373 def __init__(self, payload):
0374 self.transition = int(payload[0])
0375 self.index = int(payload[1])
0376 self.sync = (int(payload[2]), int(payload[3]), int(payload[4]))
0377 self.time = int(payload[5])
0378 def indentLevel(self):
0379 return transitionIndentLevel(self.transition)
0380 def textPrefix(self):
0381 return textPrefix_(self.time, self.indentLevel())
0382 def syncText(self):
0383 if self.transition == Phase.globalBeginRun or Phase.globalEndRun == self.transition:
0384 return f'run={self.sync[0]}'
0385 if self.transition == Phase.globalWriteRun:
0386 return f'run={self.sync[0]}'
0387 if self.transition == Phase.streamBeginRun or Phase.streamEndRun == self.transition:
0388 return f'run={self.sync[0]}'
0389 if self.transition == Phase.globalBeginLumi or Phase.globalEndLumi == self.transition:
0390 return f'run={self.sync[0]} lumi={self.sync[1]}'
0391 if self.transition == Phase.globalWriteLumi:
0392 return f'run={self.sync[0]} lumi={self.sync[1]}'
0393 if self.transition == Phase.streamBeginLumi or Phase.streamEndLumi == self.transition:
0394 return f'run={self.sync[0]} lumi={self.sync[1]}'
0395 if self.transition == Phase.Event:
0396 return f'run={self.sync[0]} lumi={self.sync[1]} event={self.sync[2]}'
0397 if self.transition == Phase.esSyncEnqueue or self.transition == Phase.esSync:
0398 return f'run={self.sync[0]} lumi={self.sync[1]}'
0399 if self.transition == Phase.beginJob:
0400 return ''
0401 if self.transition == Phase.beginProcessBlock or self.transition == Phase.endProcessBlock or self.transition == Phase.writeProcessBlock or self.transition == Phase.accessInputProcessBlock:
0402 return ''
0403 if self.transition == Phase.startTracing:
0404 return ''
0405 if self.transition == Phase.construction or self.transition == Phase.destruction:
0406 return ''
0407 def textPostfix(self):
0408 return f'{transitionName(self.transition)} : id={self.index} {self.syncText()}'
0409 def text(self, context):
0410 return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0411
0412 def findMatchingTransition(sync, containers):
0413 for i in range(len(containers)):
0414 if containers[i][-1]["sync"] == sync:
0415 return i
0416
0417 for i in range(len(containers)):
0418 for t in containers[i]:
0419 if t["sync"] == sync:
0420 return i
0421
0422 print("find failed",sync, containers)
0423 return None
0424
0425 transitionsToFindMatch_ = {
0426 Phase.globalEndRun,
0427 Phase.globalEndLumi,
0428 Phase.globalWriteRun,
0429 Phase.globalWriteLumi
0430 }
0431
0432 class PreFrameworkTransitionParser (FrameworkTransitionParser):
0433 def __init__(self, payload):
0434 super().__init__(payload)
0435 def textSpecial(self):
0436 return "starting"
0437 def jsonInfo(self, syncs, temp, data):
0438 isSourceTrans = False
0439 if self.transition == Phase.startTracing:
0440 data.setStartTime(self.time)
0441 elif self.transition == Phase.globalBeginRun:
0442 syncs.setRun(self.index, self.sync[0])
0443 isSourceTrans = True
0444 elif self.transition == Phase.globalBeginLumi:
0445 syncs.setLumi(self.index, self.sync[0], self.sync[1])
0446 isSourceTrans = True
0447 elif self.transition == Phase.Event:
0448 syncs.setStream(self.index, self.sync[0], self.sync[1], self.sync[2])
0449 isSourceTrans = True
0450 elif self.transition == Phase.clearEvent:
0451 temp.insertTime("clearEvent", self.transition, self.index, self.time)
0452 elif not transitionIsGlobal(self.index):
0453 syncs.setStream(self.index, self.sync[0], self.sync[1], self.sync[2])
0454 if isSourceTrans:
0455 src = data.findLast("source", self.transition, self.index, Activity.process)
0456 if src.sync != self.index:
0457 raise RuntimeError("Framework and Source transitions do not have matching index: source {} framework {} for transition type {} at framework time {} and source time {}".format(src.sync, self.index, self.transition, self.time, src.timeRange))
0458 src.sync = syncs.get(self.transition, self.index)
0459 def jsonVisInfo(self, data):
0460 if transitionIsGlobal(self.transition):
0461 index = 0
0462 if self.transition == Phase.startTracing:
0463 data.indexedGlobal(0).append(jsonTransition(type=self.transition, id=index, sync=list(self.sync),start=0, finish=self.time ))
0464 return
0465 elif self.transition==Phase.globalBeginRun:
0466 index = self.index
0467 container = data.indexedGlobal(index)
0468
0469 last = container[-1]
0470 if last["type"]==Phase.globalBeginRun and last["isSrc"]:
0471 last["sync"]=list(self.sync)
0472 elif self.transition==Phase.globalBeginLumi:
0473 index = self.index
0474 container = data.indexedGlobal(index)
0475
0476 last = container[-1]
0477 if last["type"]==Phase.globalBeginLumi and last["isSrc"]:
0478 last["sync"]=list(self.sync)
0479 elif self.transition in transitionsToFindMatch_:
0480 index = findMatchingTransition(list(self.sync), data.allGlobals())
0481 container = data.indexedGlobal(index)
0482 else:
0483 container = data.indexedStream(self.index)
0484 if self.transition == Phase.Event:
0485
0486 last = container[-1]
0487 if last["type"]==Phase.Event and last["isSrc"]:
0488 last["sync"]=list(self.sync)
0489 index = self.index
0490 container.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0491
0492
0493
0494 class PostFrameworkTransitionParser (FrameworkTransitionParser):
0495 def __init__(self, payload):
0496 super().__init__(payload)
0497 if self.transition == Phase.clearEvent:
0498 self.alloc = AllocInfo(payload[6:])
0499 def textSpecial(self):
0500 return "finished"
0501 def jsonInfo(self, syncs, temp, data):
0502 if self.transition == Phase.clearEvent:
0503 start = temp.findTime("clearEvent", self.transition, self.index)
0504 data.insert( "clearEvent" , start, self.time, self.transition, self.index, syncs.get(Phase.Event, self.index) , Activity.process, self.alloc)
0505 def jsonVisInfo(self, data):
0506 if transitionIsGlobal(self.transition):
0507 index = findMatchingTransition(list(self.sync), data.allGlobals())
0508 container = data.indexedGlobal(index)
0509 else:
0510 container = data.indexedStream(self.index)
0511 container[-1]["finish"]=self.time*kMicroToSec
0512 if self.transition == Phase.clearEvent:
0513 self.alloc.inject(container[-1])
0514
0515
0516
0517 class SourceTransitionParser(object):
0518 def __init__(self, payload):
0519 self.transition = int(payload[0])
0520 if self.transition == Phase.getNextTransition:
0521 self.time = int(payload[1])
0522 self.index = -1
0523 return
0524 self.index = int(payload[1])
0525 self.time = int(payload[2])
0526 def indentLevel(self):
0527 if self.transition == Phase.globalBeginRun:
0528 return 1
0529 if self.transition == Phase.globalBeginLumi:
0530 return 2
0531 if self.transition == Phase.Event:
0532 return 3
0533 if self.transition == Phase.construction:
0534 return 1
0535 if self.transition == Phase.getNextTransition:
0536 return 1
0537 if self.transition == Phase.openFile:
0538 return 1
0539 return None
0540 def textPrefix(self):
0541 return textPrefix_(self.time, self.indentLevel())
0542 def textPostfix(self):
0543 return f'source during {transitionName(self.transition)} : id={self.index}'
0544 def text(self, context):
0545 return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0546
0547 class PreSourceTransitionParser(SourceTransitionParser):
0548 def __init__(self, payload, moduleCentric):
0549 self._moduleCentric = moduleCentric
0550 super().__init__(payload)
0551 def textSpecial(self):
0552 return "starting"
0553 def jsonInfo(self, syncs, temp, data):
0554 temp.insertTime("source", self.transition, self.index, self.time)
0555 def jsonVisInfo(self, data):
0556 if self.transition == Phase.getNextTransition:
0557 data._nextTrans.append(jsonTransition(type=self.transition, id=self.index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0558 if self._moduleCentric:
0559
0560 data.findOpenSlotInModGlobals(0,0).append(data._nextTrans[-1])
0561 return
0562 elif self.transition == Phase.construction:
0563 index = 0
0564 container = data.indexedGlobal(index)
0565 elif self.transition == Phase.Event:
0566 index = self.index
0567 container = data.indexedStream(index)
0568 else:
0569 index = self.index
0570 container = data.indexedGlobal(index)
0571 nextTrans = data._nextTrans
0572 if nextTrans:
0573 data._nextTrans = []
0574 for t in nextTrans:
0575 t['id']=index
0576
0577 transStartTime = t['start']
0578 inserted = False
0579 for i in range(-1, -1*len(container), -1):
0580 if transStartTime > container[i]['start']:
0581 if i == -1:
0582 container.append(t)
0583 inserted = True
0584 break
0585 else:
0586 container.insert(i+1,t)
0587 inserted = True
0588 break
0589 if not inserted:
0590 container.insert(0,t)
0591 container.append(jsonTransition(type=self.transition, id=index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0592 if self._moduleCentric:
0593 if self.transition == Phase.Event:
0594 data.findOpenSlotInModStreams(index,0).append(container[-1])
0595 else:
0596 data.findOpenSlotInModGlobals(index,0).append(container[-1])
0597
0598 class PostSourceTransitionParser(SourceTransitionParser):
0599 def __init__(self, payload, moduleCentric):
0600 super().__init__(payload)
0601
0602 if self.index == -1:
0603 self.allocInfo = AllocInfo(payload[2:])
0604 else:
0605 self.allocInfo = AllocInfo(payload[3:])
0606 self._moduleCentric = moduleCentric
0607 def textSpecial(self):
0608 return "finished"
0609 def jsonInfo(self, syncs, temp, data):
0610 start = temp.findTime("source", self.transition, self.index)
0611
0612 if self.transition in [ Phase.construction, Phase.getNextTransition, Phase.destruction, Phase.openFile]:
0613 data.insert( "source" , start, self.time, self.transition, self.index, (0,) , Activity.process, self.allocInfo)
0614 else:
0615 data.insert( "source" , start, self.time, self.transition, self.index, self.index , Activity.process, self.allocInfo)
0616 def jsonVisInfo(self, data):
0617 index = self.index
0618 if self.transition == Phase.Event:
0619 container = data.indexedStream(index)
0620 elif self.transition == Phase.getNextTransition:
0621 data._nextTrans[-1]['finish'] = self.time*kMicroToSec
0622 self.allocInfo.inject(data._nextTrans[-1])
0623 return
0624 elif self.transition == Phase.construction:
0625 pre = None
0626 for i, g in enumerate(data.allGlobals()):
0627 for t in reversed(g):
0628 if t["type"] != Phase.construction:
0629 break
0630 if t["isSrc"]:
0631 pre = t
0632 break
0633 if pre:
0634 pre["finish"]=self.time*kMicroToSec
0635 self.allocInfo.inject(pre)
0636 break
0637 return
0638 else:
0639 container = data.indexedGlobal(index)
0640
0641 container[-1]["finish"]=self.time*kMicroToSec
0642 self.allocInfo.inject(container[-1])
0643
0644 class EDModuleTransitionParser(object):
0645 def __init__(self, payload, moduleNames):
0646 self.transition = int(payload[0])
0647 self.index = int(payload[1])
0648 self.moduleID = int(payload[2])
0649 self.moduleName = moduleNames[self.moduleID]
0650 self.callID = int(payload[3])
0651 self.time = int(payload[4])
0652 def baseIndentLevel(self):
0653 return transitionIndentLevel(self.transition)
0654 def textPrefix(self, context):
0655 indent = 0
0656 context[(self.transition, self.index, self.moduleID, self.callID)] = indent+1
0657 return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0658 def textPostfix(self):
0659 return f'{self.moduleName} during {transitionName(self.transition)} : id={self.index}'
0660 def textIfTransform(self):
0661 if self.callID:
0662 return f' transform {self.callID-1}'
0663 return ''
0664 def text(self, context):
0665 return f'{self.textPrefix(context)} {self.textSpecial()}{self.textIfTransform()}: {self.textPostfix()}'
0666 def _preJsonVis(self, activity, data, mayUseTemp = False):
0667 index = self.index
0668 found = False
0669 if mayUseTemp:
0670 compare = lambda x: x['type'] == self.transition and x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and (x['act'] == Activity.temporary or x['act'] == Activity.externalWork)
0671 if transitionIsGlobal(self.transition):
0672 item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0673 else:
0674 item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0675 if slot:
0676 if item['act'] == Activity.temporary:
0677 slot.pop()
0678 else:
0679 item['finish']=self.time*kMicroToSec
0680 found = True
0681 if not found:
0682 if transitionIsGlobal(self.transition):
0683 slot = data.findOpenSlotInModGlobals(index, self.moduleID)
0684 else:
0685 slot = data.findOpenSlotInModStreams(index, self.moduleID)
0686 slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=activity, start=self.time))
0687 return slot[-1]
0688 def _postJsonVis(self, data, alloc, injectAfter = None):
0689 compare = lambda x: x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and x['type'] == self.transition
0690 index = self.index
0691 if transitionIsGlobal(self.transition):
0692 item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0693 else:
0694 item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0695 if item is None:
0696 print(f"failed to find {self.moduleID} for {self.transition} in {self.index}")
0697 else:
0698 item["finish"]=self.time*kMicroToSec
0699 alloc.inject(item)
0700 if injectAfter:
0701 slot.append(injectAfter)
0702 def _preJsonInfo(self, temp):
0703 temp.insertTime(self.moduleName, self.transition, self.index, self.time)
0704 def _postJsonInfo(self, syncs, temp, data, activity):
0705 start = temp.findTime(self.moduleName, self.transition, self.index)
0706 data.insert( self.moduleName , start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , activity, self.allocInfo)
0707
0708
0709 class PreEDModuleTransitionParser(EDModuleTransitionParser):
0710 def __init__(self, payload, names, moduleCentric):
0711 super().__init__(payload, names)
0712 self._moduleCentric = moduleCentric
0713 def textSpecial(self):
0714 return "starting action"
0715 def jsonVisInfo(self, data):
0716 return self._preJsonVis(Activity.process, data, mayUseTemp=self._moduleCentric)
0717 def jsonInfo(self, syncs, temp, data):
0718 self._preJsonInfo(temp)
0719
0720
0721 class PostEDModuleTransitionParser(EDModuleTransitionParser):
0722 def __init__(self, payload, names):
0723 super().__init__(payload, names)
0724 self.allocInfo = AllocInfo(payload[5:])
0725 def textSpecial(self):
0726 return "finished action"
0727 def jsonInfo(self, syncs, temp, data):
0728 self._postJsonInfo(syncs, temp, data, Activity.process)
0729 def jsonVisInfo(self, data):
0730 return self._postJsonVis(data, self.allocInfo)
0731
0732
0733 class PreEDModuleAcquireParser(EDModuleTransitionParser):
0734 def __init__(self, payload, names, moduleCentric):
0735 super().__init__(payload, names)
0736 self._moduleCentric = moduleCentric
0737 def textSpecial(self):
0738 return "starting acquire"
0739 def jsonVisInfo(self, data):
0740 return self._preJsonVis(Activity.acquire, data, mayUseTemp=self._moduleCentric)
0741 def jsonInfo(self, syncs, temp, data):
0742 self._preJsonInfo(temp)
0743
0744
0745 class PostEDModuleAcquireParser(EDModuleTransitionParser):
0746 def __init__(self, payload, names, moduleCentric):
0747 super().__init__(payload, names)
0748 self.allocInfo = AllocInfo(payload[5:])
0749 self._moduleCentric = moduleCentric
0750 def textSpecial(self):
0751 return "finished acquire"
0752 def jsonVisInfo(self, data):
0753 if self._moduleCentric:
0754
0755 return self._postJsonVis( data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.externalWork, start=self.time))
0756 return self._postJsonVis(data, self.allocInfo)
0757 def jsonInfo(self, syncs, temp, data):
0758 self._postJsonInfo(syncs, temp, data, Activity.acquire)
0759
0760 class PreEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0761 def __init__(self, payload, names):
0762 super().__init__(payload, names)
0763 def textSpecial(self):
0764 return "starting delayed get"
0765 def jsonVisInfo(self, data):
0766 return self._preJsonVis(Activity.delayedGet, data)
0767 def jsonInfo(self, syncs, temp, data):
0768 pass
0769
0770
0771 class PostEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0772 def __init__(self, payload, names):
0773 super().__init__(payload, names)
0774 self.allocInfo = AllocInfo(payload[5:])
0775 def textSpecial(self):
0776 return "finished delayed get"
0777 def jsonVisInfo(self, data):
0778 return self._postJsonVis(data, self.allocInfo)
0779 def jsonInfo(self, syncs, temp, data):
0780 pass
0781
0782
0783 class PreEventReadFromSourceParser(EDModuleTransitionParser):
0784 def __init__(self, payload, names):
0785 super().__init__(payload, names)
0786 def textSpecial(self):
0787 return "starting read from source"
0788 def jsonVisInfo(self, data):
0789 slot = self._preJsonVis(Activity.process, data)
0790 slot['isSrc'] = True
0791 return slot
0792 def jsonInfo(self, syncs, temp, data):
0793 temp.insertTime(self.moduleName+'source', self.transition, self.index, self.time)
0794
0795 class PostEventReadFromSourceParser(EDModuleTransitionParser):
0796 def __init__(self, payload, names):
0797 super().__init__(payload, names)
0798 self.allocInfo = AllocInfo(payload[5:])
0799 def textSpecial(self):
0800 return "finished read from source"
0801 def jsonVisInfo(self, data):
0802 return self._postJsonVis(data, self.allocInfo)
0803 def jsonInfo(self, syncs, temp, data):
0804 start = temp.findTime(self.moduleName+'source', self.transition, self.index)
0805 data.insert( "source" , start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , Activity.delayedGet, self.allocInfo)
0806
0807 class ESModuleTransitionParser(object):
0808 def __init__(self, payload, moduleNames, esModuleNames, recordNames):
0809 self.transition = int(payload[0])
0810 self.index = int(payload[1])
0811 self.moduleID = int(payload[2])
0812 self.moduleName = esModuleNames[self.moduleID]
0813 self.recordID = int(payload[3])
0814 self.recordName = recordNames[self.recordID]
0815 self.callID = int(payload[4])
0816 self.time = int(payload[5])
0817 def baseIndentLevel(self):
0818 return transitionIndentLevel(self.transition)
0819 def textPrefix(self, context):
0820 indent = 0
0821 context[(self.transition, self.index, -1*self.moduleID, self.callID)] = indent+1
0822 return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0823 def textPostfix(self):
0824 return f'esmodule {self.moduleName} in record {self.recordName} during {transitionName(self.transition)} : id={self.index}'
0825 def text(self, context):
0826 return f'{self.textPrefix(context)} {self.textSpecial()}: {self.textPostfix()}'
0827 def _preJsonVis(self, activity, data):
0828 index = self.index
0829 if transitionIsGlobal(self.transition):
0830 slot = data.findOpenSlotInModGlobals(index, -1*self.moduleID)
0831 else:
0832 slot = data.findOpenSlotInModStreams(index, -1*self.moduleID)
0833 slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=-1*self.moduleID, callID=self.callID, activity=activity, start=self.time))
0834 return slot[-1]
0835 def _postJsonVis(self, data, alloc):
0836 compare = lambda x: x['id'] == self.index and x['mod'] == -1*self.moduleID and x['call'] == self.callID
0837 index = self.index
0838 if transitionIsGlobal(self.transition):
0839 item,s = data.findLastInModGlobals(index, -1*self.moduleID, compare)
0840 else:
0841 item,s = data.findLastInModStreams(index, -1*self.moduleID, compare)
0842 if item is None:
0843 print(f"failed to find {-1*self.moduleID} for {self.transition} in {self.index}")
0844 return
0845 item["finish"]=self.time*kMicroToSec
0846 alloc.inject(item)
0847 def _preJsonInfo(self, temp):
0848 temp.insertTimeES(self.moduleName, self.transition, self.index, self.recordID, self.callID, self.time)
0849 def _postJsonInfo(self, syncs, temp, data, activity):
0850 start = temp.findTimeES(self.moduleName, self.transition, self.index, self.recordID, self.callID)
0851 data.insert( self.moduleName , start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , activity, self.allocInfo, self.recordName, self.callID)
0852
0853
0854 class PreESModuleTransitionParser(ESModuleTransitionParser):
0855 def __init__(self, payload, names, esNames, recordNames):
0856 super().__init__(payload, names, esNames, recordNames)
0857 def textSpecial(self):
0858 return "starting action"
0859 def jsonVisInfo(self, data):
0860 return self._preJsonVis(Activity.process, data)
0861 def jsonInfo(self, syncs, temp, data):
0862 self._preJsonInfo(temp)
0863
0864 class PostESModuleTransitionParser(ESModuleTransitionParser):
0865 def __init__(self, payload, names, esNames, recordNames):
0866 super().__init__(payload, names, esNames, recordNames)
0867 self.allocInfo = AllocInfo(payload[6:])
0868 def textSpecial(self):
0869 return "finished action"
0870 def jsonVisInfo(self, data):
0871 return self._postJsonVis(data,self.allocInfo)
0872 def jsonInfo(self, syncs, temp, data):
0873 self._postJsonInfo(syncs, temp, data, Activity.process)
0874
0875 class PreESModuleAcquireParser(ESModuleTransitionParser):
0876 def __init__(self, payload, names, recordNames):
0877 super().__init__(payload, names, recordNames)
0878 def textSpecial(self):
0879 return "starting acquire"
0880 def jsonVisInfo(self, data):
0881 return self._preJsonVis(Activity.acquire, data)
0882
0883 class PostESModuleAcquireParser(ESModuleTransitionParser):
0884 def __init__(self, payload, names, esNames, recordNames):
0885 super().__init__(payload, names, esNames, recordNames)
0886 self.allocInfo = AllocInfo(payload[6:])
0887 def textSpecial(self):
0888 return "finished acquire"
0889 def jsonVisInfo(self, data):
0890 return self._postJsonVis(data, self.allocInfo)
0891
0892
0893 def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, moduleCentric):
0894 if step == 'F':
0895 parser = PreFrameworkTransitionParser(payload)
0896 return parser
0897 if step == 'f':
0898 return PostFrameworkTransitionParser(payload)
0899 if step == 'S':
0900 return PreSourceTransitionParser(payload, moduleCentric)
0901 if step == 's':
0902 return PostSourceTransitionParser(payload, moduleCentric)
0903 if step == 'M':
0904 return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
0905 if step == 'm':
0906 return PostEDModuleTransitionParser(payload, moduleNames)
0907 if step == 'A':
0908 return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0909 if step == 'a':
0910 return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0911 if step == 'D':
0912 return PreEDModuleEventDelayedGetParser(payload, moduleNames)
0913 if step == 'd':
0914 return PostEDModuleEventDelayedGetParser(payload, moduleNames)
0915 if step == 'R':
0916 return PreEventReadFromSourceParser(payload, moduleNames)
0917 if step == 'r':
0918 return PostEventReadFromSourceParser(payload, moduleNames)
0919 if step == 'N':
0920 return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0921 if step == 'n':
0922 return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0923 if step == 'B':
0924 return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0925 if step == 'b':
0926 return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0927 raise LogicError("Unknown step '{}'".format(step))
0928
0929
0930 def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, moduleCentric):
0931 for rawl in f:
0932 l = rawl.strip()
0933 if not l or l[0] == '#':
0934 continue
0935 (step,payload) = tuple(l.split(None,1))
0936 payload=payload.split()
0937
0938 parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0939 if parser:
0940 yield parser
0941 return
0942
0943 class ModuleAllocCompactFileParser(object):
0944 def __init__(self,f, moduleCentric):
0945 streamBeginRun = str(Phase.streamBeginRun)
0946 numStreams = 0
0947 numStreamsFromSource = 0
0948 moduleNames = {}
0949 esModuleNames = {}
0950 recordNames = {}
0951 for rawl in f:
0952 l = rawl.strip()
0953 if l and l[0] == 'M':
0954 i = l.split(' ')
0955 if i[3] == streamBeginRun:
0956
0957 numStreams = int(i[1])+1
0958 break
0959 if numStreams == 0 and l and l[0] == 'S':
0960 s = int(l.split(' ')[1])
0961 if s > numStreamsFromSource:
0962 numStreamsFromSource = s
0963 if len(l) > 5 and l[0:2] == "#M":
0964 (id,name)=tuple(l[2:].split())
0965 moduleNames[int(id)] = name
0966 continue
0967 if len(l) > 5 and l[0:2] == "#N":
0968 (id,name)=tuple(l[2:].split())
0969 esModuleNames[int(id)] = name
0970 continue
0971 if len(l) > 5 and l[0:2] == "#R":
0972 (id,name)=tuple(l[2:].split())
0973 recordNames[int(id)] = name
0974 continue
0975
0976 self._f = f
0977 self._moduleCentric = moduleCentric
0978 if numStreams == 0:
0979 numStreams = numStreamsFromSource +2
0980 self.numStreams =numStreams
0981 self._moduleNames = moduleNames
0982 self._esModuleNames = esModuleNames
0983 self._recordNames = recordNames
0984 self.maxNameSize =0
0985 for n in moduleNames.items():
0986 self.maxNameSize = max(self.maxNameSize,len(n))
0987 for n in esModuleNames.items():
0988 self.maxNameSize = max(self.maxNameSize,len(n))
0989 self.maxNameSize = max(self.maxNameSize,len(kSourceDelayedRead))
0990 self.maxNameSize = max(self.maxNameSize, len('streamBeginLumi'))
0991
0992 def processingSteps(self):
0993 """Create a generator which can step through the file and return each processing step.
0994 Using a generator reduces the memory overhead when parsing a large file.
0995 """
0996 self._f.seek(0)
0997 return processingStepsFromFile(self._f,self._moduleNames, self._esModuleNames, self._recordNames, self._moduleCentric)
0998
0999 def textOutput( parser ):
1000 context = {}
1001 for p in parser.processingSteps():
1002 print(p.text(context))
1003
1004 class VisualizationContainers(object):
1005 def __init__(self):
1006 self._modGlobals = [[]]
1007 self._modStreams = [[]]
1008 self._globals = [[]]
1009 self._streams = [[]]
1010 self._nextTrans = []
1011 def _extendIfNeeded(self, container, index):
1012 while len(container) < index+1:
1013 container.append([])
1014 def allGlobals(self):
1015 return self._globals
1016 def indexedGlobal(self, index):
1017 self._extendIfNeeded(self._globals, index)
1018 return self._globals[index]
1019 def allStreams(self):
1020 return self._streams
1021 def indexedStream(self, index):
1022 self._extendIfNeeded(self._streams, index)
1023 return self._streams[index]
1024 def _findOpenSlot(self, index, fullContainer):
1025 self._extendIfNeeded(fullContainer, index)
1026 container = fullContainer[index]
1027
1028 foundOpenSlot = False
1029 for slot in container:
1030 if len(slot) == 0:
1031 foundOpenSlot = True
1032 break
1033 if slot[-1]["finish"] != 0:
1034 foundOpenSlot = True
1035 break
1036 if not foundOpenSlot:
1037 container.append([])
1038 slot = container[-1]
1039 return slot
1040 def findOpenSlotInModGlobals(self, index, modID):
1041 return self._findOpenSlot(index, self._modGlobals)
1042 def findOpenSlotInModStreams(self, index, modID):
1043 return self._findOpenSlot(index, self._modStreams)
1044 def _findLastIn(self, index, fullContainer, comparer):
1045 container = fullContainer[index]
1046
1047 for slot in container:
1048 if comparer(slot[-1]):
1049 return (slot[-1],slot)
1050 return (None,None)
1051 def findLastInModGlobals(self, index, modID, comparer):
1052 return self._findLastIn(index, self._modGlobals, comparer)
1053 def findLastInModStreams(self, index, modID, comparer):
1054 return self._findLastIn(index, self._modStreams, comparer)
1055
1056
1057 class ModuleCentricVisualizationContainers(object):
1058 def __init__(self):
1059 self._modules= []
1060 self._globals = [[]]
1061 self._streams = [[]]
1062 self._nextTrans = []
1063 self._moduleIDOffset = 0
1064 def _moduleID2Index(self, modID):
1065 return modID + self._moduleIDOffset
1066 def _extendIfNeeded(self, container, index):
1067 while len(container) < index+1:
1068 container.append([])
1069 def _extendModulesIfNeeded(self, container, index):
1070 while index + self._moduleIDOffset < 0:
1071 container.insert(0,[])
1072 self._moduleIDOffset +=1
1073 self._extendIfNeeded(container, self._moduleID2Index(index))
1074 def allGlobals(self):
1075 return self._globals
1076 def indexedGlobal(self, index):
1077 self._extendIfNeeded(self._globals, index)
1078 return self._globals[index]
1079 def allStreams(self):
1080 return self._streams
1081 def indexedStream(self, index):
1082 self._extendIfNeeded(self._streams, index)
1083 return self._streams[index]
1084 def _findOpenSlot(self, index, fullContainer):
1085 self._extendModulesIfNeeded(fullContainer, index)
1086 container = fullContainer[self._moduleID2Index(index)]
1087
1088 foundOpenSlot = False
1089 for slot in container:
1090 if len(slot) == 0:
1091 foundOpenSlot = True
1092 break
1093 if slot[-1]["finish"] != 0:
1094 foundOpenSlot = True
1095 break
1096 if not foundOpenSlot:
1097 container.append([])
1098 slot = container[-1]
1099 return slot
1100 def findOpenSlotInModGlobals(self, index, modID):
1101 return self._findOpenSlot(modID, self._modules)
1102 def findOpenSlotInModStreams(self, index, modID):
1103 return self._findOpenSlot(modID, self._modules)
1104 def _findLastIn(self, index, fullContainer, comparer):
1105 if not fullContainer:
1106 return (None, None)
1107 if len(fullContainer) > self._moduleID2Index(index):
1108 container = fullContainer[self._moduleID2Index(index)]
1109 else:
1110 return (None, None)
1111
1112 for slot in container:
1113 if slot is not None and comparer(slot[-1]):
1114 return (slot[-1],slot)
1115 return (None, None)
1116 def findLastInModGlobals(self, index, modID, comparer):
1117 return self._findLastIn(modID, self._modules, comparer)
1118 def findLastInModStreams(self, index, modID, comparer):
1119 return self._findLastIn(modID, self._modules, comparer)
1120
1121
1122 class ModuleCentricContainers(object):
1123 def __init__(self):
1124 self._modules= []
1125 self._globals = [[]]
1126 self._streams = [[]]
1127 self._nextTrans = []
1128 self._moduleIDOffset = 0
1129 def _moduleID2Index(self, modID):
1130 return modID + self._moduleIDOffset
1131 def _extendIfNeeded(self, container, index):
1132 while len(container) < index+1:
1133 container.append([])
1134 def _extendModulesIfNeeded(self, container, index):
1135 while index + self._moduleIDOffset < 0:
1136 container.insert(0,[])
1137 self._moduleIDOffset +=1
1138 self._extendIfNeeded(container, self._moduleID2Index(index))
1139 def allGlobals(self):
1140 return self._globals
1141 def indexedGlobal(self, index):
1142 self._extendIfNeeded(self._globals, index)
1143 return self._globals[index]
1144 def allStreams(self):
1145 return self._streams
1146 def indexedStream(self, index):
1147 self._extendIfNeeded(self._streams, index)
1148 return self._streams[index]
1149 def _findOpenSlot(self, index, fullContainer):
1150 self._extendModulesIfNeeded(fullContainer, index)
1151 container = fullContainer[self._moduleID2Index(index)]
1152
1153 foundOpenSlot = False
1154 for slot in container:
1155 if len(slot) == 0:
1156 foundOpenSlot = True
1157 break
1158 if slot[-1]["finish"] != 0:
1159 foundOpenSlot = True
1160 break
1161 if not foundOpenSlot:
1162 container.append([])
1163 slot = container[-1]
1164 return slot
1165 def findOpenSlotInModGlobals(self, index, modID):
1166 return self._findOpenSlot(modID, self._modules)
1167 def findOpenSlotInModStreams(self, index, modID):
1168 return self._findOpenSlot(modID, self._modules)
1169 def _findLastIn(self, index, fullContainer, comparer):
1170 if not fullContainer:
1171 return (None, None)
1172 if len(fullContainer) > self._moduleID2Index(index):
1173 container = fullContainer[self._moduleID2Index(index)]
1174 else:
1175 return (None, None)
1176
1177 for slot in container:
1178 if slot is not None and comparer(slot[-1]):
1179 return (slot[-1],slot)
1180 return (None, None)
1181 def findLastInModGlobals(self, index, modID, comparer):
1182 return self._findLastIn(modID, self._modules, comparer)
1183 def findLastInModStreams(self, index, modID, comparer):
1184 return self._findLastIn(modID, self._modules, comparer)
1185
1186
1187
1188 def jsonTransition(type, id, sync, start, finish, isSrc=False):
1189 return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}
1190
1191 def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
1192 return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}
1193
1194
1195 def startTime(x):
1196 return x["start"]
1197
1198 def jsonInfo(parser, temporal):
1199 sync = SyncValues()
1200
1201 temp = TempModuleTransitionInfos()
1202 if temporal:
1203 data = TemporalModuleData()
1204 else:
1205 data = ModuleCentricModuleData()
1206 for p in parser.processingSteps():
1207 if hasattr(p, "jsonInfo"):
1208 p.jsonInfo(sync,temp, data)
1209 return data
1210
1211 def sortByAttribute(parser, attribute):
1212 sync = SyncValues()
1213
1214 temp = TempModuleTransitionInfos()
1215 data = ModuleCentricModuleData()
1216 for p in parser.processingSteps():
1217 if hasattr(p, "jsonInfo"):
1218 p.jsonInfo(sync,temp, data)
1219 return data.sortModulesBy(attribute)
1220
1221 def jsonVisualizationInfo(parser):
1222 if parser._moduleCentric:
1223 data = ModuleCentricContainers()
1224 else:
1225 data = VisualizationContainers()
1226 for p in parser.processingSteps():
1227 p.jsonVisInfo( data)
1228
1229 for g in data.allGlobals():
1230 g.sort(key=startTime)
1231 final = {"transitions" : [] , "modules": [], "esModules": []}
1232 final["transitions"].append({ "name":"Global", "slots": []})
1233 globals = final["transitions"][-1]["slots"]
1234 for i, g in enumerate(data.allGlobals()):
1235 globals.append(g)
1236 if not parser._moduleCentric:
1237 if len(data._modGlobals) < i+1:
1238 break
1239 for mod in data._modGlobals[i]:
1240 globals.append(mod)
1241 for i,s in enumerate(data.allStreams()):
1242 final["transitions"].append({"name": f"Stream {i}", "slots":[]})
1243 stream = final["transitions"][-1]["slots"]
1244 stream.append(s)
1245 if not parser._moduleCentric:
1246 for mod in data._modStreams[i]:
1247 stream.append(mod)
1248 if parser._moduleCentric:
1249 sourceSlot = data._modules[data._moduleID2Index(0)]
1250 modules = []
1251 for i,m in parser._moduleNames.items():
1252 modules.append({"name": f"{m}", "slots":[]})
1253 slots = modules[-1]["slots"]
1254 foundSlots = data._modules[data._moduleID2Index(i)]
1255 time = 0
1256 for s in foundSlots:
1257 slots.append(s)
1258 for t in s:
1259 if t["act"] !=Activity.prefetch:
1260 time += t["finish"]-t["start"]
1261 modules[-1]['time']=time
1262 for i,m in parser._esModuleNames.items():
1263 modules.append({"name": f"{m}", "slots":[]})
1264 slots = modules[-1]["slots"]
1265 foundSlots = data._modules[data._moduleID2Index(-1*i)]
1266 time = 0
1267 for s in foundSlots:
1268 slots.append(s)
1269 for t in s:
1270 if t["act"] !=Activity.prefetch:
1271 time += t["finish"]-t["start"]
1272 modules[-1]['time']=time
1273 modules.sort(key= lambda x : x['time'], reverse=True)
1274 final['transitions'].append({"name": "source", "slots":sourceSlot})
1275 for m in modules:
1276 final['transitions'].append(m)
1277
1278 max = 0
1279 for k in parser._moduleNames.keys():
1280 if k > max:
1281 max = k
1282
1283 final["modules"] =['']*(max+1)
1284 final["modules"][0] = 'source'
1285 for k,v in parser._moduleNames.items():
1286 final["modules"][k]=v
1287
1288 max = 0
1289 for k in parser._esModuleNames.keys():
1290 if k > max:
1291 max = k
1292 final["esModules"] = ['']*(max+1)
1293 for k,v in parser._esModuleNames.items():
1294 final["esModules"][k] = v
1295 return final
1296
1297
1298
1299
1300 import unittest
1301
1302 class DummyFile(list):
1303 def __init__(self):
1304 super()
1305 def seek(self, i):
1306 pass
1307
1308 class TestModuleCommand(unittest.TestCase):
1309 def setUp(self):
1310 self.tracerFile = DummyFile()
1311 t = [0]
1312 def incr(t):
1313 t[0] += 1
1314 return t[0]
1315
1316 self.tracerFile.extend([
1317 '#R 1 Record',
1318 '#M 1 Module',
1319 '#N 1 ESModule',
1320 f'F {Phase.startTracing} 0 0 0 0 {incr(t)}',
1321 f'S {Phase.construction} 0 {incr(t)}',
1322 f's {Phase.construction} 0 {incr(t)} 1 1 10 0 10 10',
1323 f'M {Phase.construction} 0 1 0 {incr(t)}',
1324 f'm {Phase.construction} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1325 f'F {Phase.beginJob} 0 0 0 0 {incr(t)}',
1326 f'M {Phase.beginJob} 0 1 0 {incr(t)}',
1327 f'm {Phase.beginJob} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1328 f'f {Phase.beginJob} 0 0 0 0 {incr(t)}',
1329 f'F {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1330 f'f {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1331 f'S {Phase.getNextTransition} {incr(t)}',
1332 f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1333 f'S {Phase.globalBeginRun} 0 {incr(t)}',
1334 f's {Phase.globalBeginRun} 0 {incr(t)} 1 1 10 0 10 10',
1335 f'S {Phase.getNextTransition} {incr(t)}',
1336 f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1337 f'F {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1338 f'M {Phase.globalBeginRun} 0 1 0 {incr(t)}',
1339 f'm {Phase.globalBeginRun} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1340 f'f {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1341 f'S {Phase.getNextTransition} {incr(t)}',
1342 f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1343 f'F {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1344 f'M {Phase.streamBeginRun} 0 1 0 {incr(t)}',
1345 f'm {Phase.streamBeginRun} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1346 f'f {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1347 f'F {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1348 f'M {Phase.streamBeginRun} 1 1 0 {incr(t)}',
1349 f'm {Phase.streamBeginRun} 1 1 0 {incr(t)} 3 2 20 0 50 25',
1350 f'f {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1351 f'S {Phase.globalBeginLumi} 0 {incr(t)}',
1352 f's {Phase.globalBeginLumi} 0 {incr(t)} 1 1 10 0 10 10',
1353 f'S {Phase.getNextTransition} {incr(t)}',
1354 f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1355 f'F {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1356 f'M {Phase.globalBeginLumi} 0 1 0 {incr(t)}',
1357 f'm {Phase.globalBeginLumi} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1358 f'f {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1359 f'F {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1360 f'f {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1361 f'F {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1362 f'f {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1363 f'S {Phase.Event} 0 {incr(t)}',
1364 f's {Phase.Event} 0 {incr(t)} 1 1 10 0 10 10',
1365 f'S {Phase.getNextTransition} {incr(t)}',
1366 f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1367 f'F {Phase.Event} 0 1 1 1 {incr(t)}',
1368 f'S {Phase.Event} 1 {incr(t)}',
1369 f's {Phase.Event} 1 {incr(t)} 1 1 10 0 10 10',
1370 f'F {Phase.Event} 1 1 1 2 {incr(t)}',
1371 f'N {Phase.Event} 0 1 1 0 {incr(t)}',
1372 f'n {Phase.Event} 0 1 1 0 {incr(t)} 6 5 30 0 100 80',
1373 f'M {Phase.Event} 0 1 0 {incr(t)}',
1374 f'M {Phase.Event} 1 1 0 {incr(t)}',
1375 f'm {Phase.Event} 1 1 0 {incr(t)} 3 2 20 0 50 25',
1376 f'm {Phase.Event} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1377 f'f {Phase.Event} 0 1 1 1 {incr(t)}',
1378 f'f {Phase.Event} 1 1 1 2 {incr(t)}'])
1379
1380 None
1381 def testSyncValues(self):
1382 s = SyncValues()
1383 s.setRun(0,1)
1384 self.assertEqual(s.runFor(0), 1)
1385 s.setLumi(0,1, 1)
1386 self.assertEqual(s.lumiFor(0), (1,1))
1387 s.setStream(1, 1,1,3)
1388 self.assertEqual(s.streamFor(1), (1,1,3))
1389 def testContainers(self):
1390 c = VisualizationContainers()
1391 c.indexedGlobal(2)
1392 self.assertEqual(len(c.allGlobals()), 3)
1393 c.indexedStream(2)
1394 self.assertEqual(len(c.allStreams()), 3)
1395 slot = c.findOpenSlotInModGlobals(2, 1)
1396 self.assertEqual(len(c._modGlobals),3)
1397 self.assertEqual(len(slot),0)
1398 slot.append({"start":1, "finish":0, "id":1})
1399 def testFind(item):
1400 return item["id"]==1
1401 item,s = c.findLastInModGlobals(2, 1, testFind)
1402 self.assertEqual(item["id"],1)
1403 self.assertEqual(slot,s)
1404 slot = c.findOpenSlotInModStreams(2, 1)
1405 self.assertEqual(len(c._modStreams),3)
1406 self.assertEqual(len(slot),0)
1407 slot.append({"start":1, "finish":0, "id":1})
1408 item,s = c.findLastInModStreams(2, 1, testFind)
1409 self.assertEqual(item["id"],1)
1410 self.assertEqual(slot,s)
1411 def testJson(self):
1412 parser = ModuleAllocCompactFileParser(self.tracerFile, False)
1413 j = jsonInfo(parser, False)
1414 self.assertEqual(len(j.data()),3)
1415 self.assertEqual(len(j.data()["source"]), 10)
1416 self.assertEqual(len(j.data()["Module"]), 8)
1417 self.assertEqual(len(j.data()["ESModule"]), 1)
1418 def testJsonTemporal(self):
1419 parser = ModuleAllocCompactFileParser(self.tracerFile, True)
1420 j = jsonInfo(parser, True)
1421 self.assertEqual(len(j.data()),19)
1422 def testSortBy(self):
1423 parser = ModuleAllocCompactFileParser(self.tracerFile, True)
1424 d = sortByAttribute(parser, 'maxTemp')
1425
1426 self.assertEqual(len(d), 3)
1427 self.assertEqual(d[0][0], 'ESModule')
1428 self.assertEqual(d[1][0], 'Module')
1429 self.assertEqual(d[2][0], 'source')
1430 def testFullVisualization(self):
1431 parser = ModuleAllocCompactFileParser(self.tracerFile, False)
1432 j = jsonVisualizationInfo(parser)
1433
1434 self.assertEqual(len(j["modules"]), 2)
1435 self.assertEqual(len(j["esModules"]), 2)
1436 self.assertEqual(len(j['transitions']), 3)
1437 self.assertEqual(j['transitions'][0]['name'], "Global")
1438 self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1439 self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1440 self.assertEqual(len(j["transitions"][0]["slots"]), 2)
1441 self.assertEqual(len(j["transitions"][0]["slots"][0]), 11)
1442 self.assertEqual(len(j["transitions"][0]["slots"][1]), 4)
1443 self.assertEqual(len(j["transitions"][1]["slots"]), 2)
1444 self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1445 self.assertEqual(len(j["transitions"][1]["slots"][1]), 3)
1446 self.assertEqual(len(j["transitions"][2]["slots"]), 2)
1447 self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1448 self.assertEqual(len(j["transitions"][2]["slots"][1]), 2)
1449 def testModuleCentricVisualization(self):
1450 parser = ModuleAllocCompactFileParser(self.tracerFile, True)
1451 j = jsonVisualizationInfo(parser)
1452
1453 self.assertEqual(len(j["modules"]), 2)
1454 self.assertEqual(len(j["esModules"]), 2)
1455 self.assertEqual(len(j['transitions']), 6)
1456 self.assertEqual(j['transitions'][0]['name'], "Global")
1457 self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1458 self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1459 self.assertEqual(j['transitions'][3]['name'], "source")
1460 self.assertEqual(j['transitions'][4]['name'], "Module")
1461 self.assertEqual(j['transitions'][5]['name'], "ESModule")
1462 self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1463 self.assertEqual(len(j["transitions"][0]["slots"][0]), 11)
1464 self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1465 self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1466 self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1467 self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1468 self.assertEqual(len(j["transitions"][4]["slots"]), 2)
1469 self.assertEqual(len(j["transitions"][4]["slots"][0]), 7)
1470 self.assertEqual(len(j["transitions"][4]["slots"][1]), 1)
1471 self.assertTrue(j["transitions"][4]["slots"][1][-1]['finish'] != 0.0)
1472 self.assertEqual(len(j["transitions"][5]["slots"]), 1)
1473 self.assertEqual(len(j["transitions"][5]["slots"][0]), 1)
1474
1475 def runTests():
1476 return unittest.main(argv=sys.argv[:1])
1477
1478
1479 if __name__=="__main__":
1480 import argparse
1481 import re
1482 import sys
1483
1484
1485 parser = argparse.ArgumentParser(description='Convert a compact tracer file into human readable output.',
1486 formatter_class=argparse.RawDescriptionHelpFormatter,
1487 epilog=printHelp())
1488 parser.add_argument('filename',
1489 type=argparse.FileType('r'),
1490 help='file to process')
1491 parser.add_argument('-j', '--json',
1492 action='store_true',
1493 help='''Write output in json format.''' )
1494 parser.add_argument('-s', '--sortBy',
1495 default = '',
1496 type = str,
1497 help="sort modules by attribute. Alloed values 'nAllocs', 'nDeallocs', 'added', 'minTemp', maxTemp', and 'max1Alloc'")
1498
1499
1500
1501 parser.add_argument('-t', '--timeOrdered',
1502 action = 'store_true',
1503 help='''For --json, organize data by time instead of by module.''' )
1504 parser.add_argument('-T', '--test',
1505 action='store_true',
1506 help='''Run internal tests.''')
1507
1508 args = parser.parse_args()
1509 if args.test:
1510 runTests()
1511 else :
1512 parser = ModuleAllocCompactFileParser(args.filename, not args.timeOrdered)
1513
1514 if args.json:
1515 json.dump(jsonInfo(parser, args.timeOrdered).toSimpleDict(), sys.stdout, indent=2)
1516
1517
1518
1519
1520
1521 elif args.sortBy:
1522 print(json.dumps(sortByAttribute(parser, args.sortBy), indent=2))
1523 else:
1524 textOutput(parser)