AllocMonitor/scripts/edmModuleAllocMonitorAnalyze.py

0001 #!/usr/bin/env python3
0002 from builtins import range
0003 from itertools import groupby
0004 from operator import attrgetter,itemgetter
0005 import sys
0006 import json
0007 from collections import defaultdict
0008 #----------------------------------------------
0009 def printHelp():
0010     s = '''
0011 To Use: Add the ModuleAllocMonitor Service to the cmsRun job use something like this
0012   in the configuration:
0013
0014   process.add_(cms.Service("ModuleAllocMonitor", fileName = cms.untracked.string("moduleAlloc.log")))
0015
0016   After running the job, execute this script and pass the name of the
0017   ModuleAllocMonitor log file to the script.
0018
0019   This script will output a more human readable form of the data in the log file.'''
0020     return s
0021
0022 #these values come from moduleALloc_setupFile.cc
0023 #enum class Step : char {
0024 #  preSourceTransition = 'S',
0025 #  postSourceTransition = 's',
0026 #  preModulePrefetching = 'P',
0027 #  postModulePrefetching = 'p',
0028 #  preModuleEventAcquire = 'A',
0029 #  postModuleEventAcquire = 'a',
0030 #  preModuleTransition = 'M',
0031 #  preEventReadFromSource = 'R',
0032 #  postEventReadFromSource = 'r',
0033 #  preModuleEventDelayedGet = 'D',
0034 #  postModuleEventDelayedGet = 'd',
0035 #  postModuleTransition = 'm',
0036 #  preESModulePrefetching = 'Q',
0037 #  postESModulePrefetching = 'q',
0038 #  preESModule = 'N',
0039 #  postESModule = 'n',
0040 #  preESModuleAcquire = 'B',
0041 #  postESModuleAcquire = 'b',
0042 #  preFrameworkTransition = 'F',
0043 #  postFrameworkTransition = 'f'
0044 #};
0045
0046
0047 kMicroToSec = 0.000001
0048 #Special names
0049 kSourceFindEvent = "sourceFindEvent"
0050 kSourceDelayedRead ="sourceDelayedRead"
0051 #this value is defined in the framework itself
0052 kLargestLumiNumber = 4294967295
0053
0054 #these values must match the enum class Phase in tracer_setupFile.cc
0055 class Phase (object):
0056   destruction = -16
0057   endJob = -12
0058   endStream = -11
0059   writeProcessBlock = -10
0060   endProcessBlock = -9
0061   globalWriteRun = -7
0062   globalEndRun = -6
0063   streamEndRun = -5
0064   globalWriteLumi = -4
0065   globalEndLumi = -3
0066   streamEndLumi = -2
0067   clearEvent = -1
0068   Event = 0
0069   streamBeginLumi = 2
0070   globalBeginLumi = 3
0071   streamBeginRun = 5
0072   globalBeginRun = 6
0073   accessInputProcessBlock = 8
0074   beginProcessBlock = 9
0075   openFile = 10
0076   beginStream = 11
0077   beginJob  = 12
0078   esSync = 13
0079   esSyncEnqueue = 14
0080   getNextTransition = 15
0081   construction = 16
0082   startTracing = 17
0083
0084 #used for json output
0085 class Activity (object):
0086   prefetch = 0
0087   acquire = 1
0088   process = 2
0089   delayedGet = 3
0090   externalWork = 4
0091   temporary = 100
0092
0093 activityNames_ = { Activity.prefetch : 'prefetch',
0094                    Activity.acquire : 'acquire',
0095                    Activity.process : 'process',
0096                    Activity.delayedGet : 'delayedGet',
0097                    Activity.externalWork : 'externalWork' }
0098
0099 def activityName(activity):
0100     return activityNames_[activity]
0101
0102 transitionToNames_ = {
0103     Phase.startTracing: 'start tracing',
0104     Phase.construction: 'construction',
0105     Phase.destruction: 'destruction',
0106     Phase.beginJob: 'begin job',
0107     Phase.endJob: 'end job',
0108     Phase.beginStream: 'begin stream',
0109     Phase.endStream: 'end stream',
0110     Phase.beginProcessBlock: 'begin process block',
0111     Phase.endProcessBlock: 'end process block',
0112     Phase.accessInputProcessBlock: 'access input process block',
0113     Phase.writeProcessBlock: 'write process block',
0114     Phase.globalBeginRun: 'global begin run',
0115     Phase.globalEndRun: 'global end run',
0116     Phase.globalWriteRun: 'global write run',
0117     Phase.streamBeginRun: 'stream begin run',
0118     Phase.streamEndRun: 'stream end run',
0119     Phase.globalBeginLumi: 'global begin lumi',
0120     Phase.globalEndLumi: 'global end lumi',
0121     Phase.globalWriteLumi: 'global write lumi',
0122     Phase.streamBeginLumi: 'stream begin lumi',
0123     Phase.streamEndLumi: 'stream end lumi',
0124     Phase.esSyncEnqueue: 'EventSetup synchronization',
0125     Phase.esSync: 'EventSetup synchronization',
0126     Phase.Event: 'event',
0127     Phase.clearEvent: 'clear event',
0128     Phase.getNextTransition: 'get next transition',
0129     Phase.openFile : "open file"
0130 }
0131
0132 def transitionName(transition):
0133     return transitionToNames_[transition]
0134
0135 transitionToIndent_ = {
0136     Phase.startTracing: 0,
0137     Phase.construction: 0,
0138     Phase.destruction: 0,
0139     Phase.endJob: 0,
0140     Phase.beginJob: 0,
0141     Phase.beginStream: 0,
0142     Phase.endStream: 0,
0143     Phase.beginProcessBlock: 1,
0144     Phase.endProcessBlock: 1,
0145     Phase.accessInputProcessBlock: 1,
0146     Phase.writeProcessBlock: 1,
0147     Phase.globalBeginRun: 1,
0148     Phase.globalEndRun: 1,
0149     Phase.globalWriteRun: 1,
0150     Phase.streamBeginRun: 1,
0151     Phase.streamEndRun: 1,
0152     Phase.globalBeginLumi: 2,
0153     Phase.globalEndLumi: 2,
0154     Phase.globalWriteLumi: 2,
0155     Phase.streamBeginLumi: 2,
0156     Phase.streamEndLumi: 2,
0157     Phase.Event: 3,
0158     Phase.clearEvent: 3,
0159     Phase.esSyncEnqueue: 1,
0160     Phase.esSync: 1,
0161     Phase.getNextTransition: 1
0162 }
0163 def transitionIndentLevel(transition):
0164     return transitionToIndent_[transition]
0165
0166 globalTransitions_ = {
0167     Phase.startTracing,
0168     Phase.construction,
0169     Phase.destruction,
0170     Phase.endJob,
0171     Phase.beginJob,
0172     Phase.beginProcessBlock,
0173     Phase.endProcessBlock,
0174     Phase.accessInputProcessBlock,
0175     Phase.writeProcessBlock,
0176     Phase.globalBeginRun,
0177     Phase.globalEndRun,
0178     Phase.globalWriteRun,
0179     Phase.globalBeginLumi,
0180     Phase.globalEndLumi,
0181     Phase.globalWriteLumi,
0182     Phase.esSyncEnqueue,
0183     Phase.esSync,
0184     Phase.getNextTransition,
0185     Phase.openFile
0186 }
0187 def transitionIsGlobal(transition):
0188     return transition in globalTransitions_;
0189
0190 def textPrefix_(time, indentLevel):
0191     #using 11 spaces for time should accomodate a job that runs 24 hrs
0192     return f'{time:>11} '+"++"*indentLevel
0193
0194 class AllocInfo(object):
0195     def __init__(self,payload):
0196         self.nAllocs = int(payload[0])
0197         self.nDeallocs = int(payload[1])
0198         self.added = int(payload[2])
0199         self.minTemp = int(payload[3])
0200         self.maxTemp = int(payload[4])
0201         self.max1Alloc = int(payload[5])
0202     def inject(self, transition):
0203         transition["nAllocs"]=self.nAllocs
0204         transition["nDeallocs"]=self.nDeallocs
0205         transition["added"]=self.added
0206         transition["minTemp"]=self.minTemp
0207         transition["maxTemp"]=self.maxTemp
0208         transition["max1Alloc"]=self.max1Alloc
0209     def __repr__(self):
0210         return "{{'nAlloc': {}, 'nDealloc': {}, 'added': {}, 'minTemp': {}, 'maxTemp': {}, 'max1Alloc': {} }}".format(self.nAllocs, self.nDeallocs, self.added, self.minTemp, self.maxTemp, self.max1Alloc)
0211     def toSimpleDict(self):
0212         return {'nAlloc' : self.nAllocs, 'nDealloc' :self.nDeallocs, 'added' : self.added, 'minTemp' : self.minTemp, 'maxTemp' : self.maxTemp, 'max1Alloc' : self.max1Alloc }
0213
0214 class SyncValues(object):
0215     def __init__(self):
0216         self._runs = []
0217         self._lumis = []
0218         self._streams = []
0219     def setRun(self, index, runNumber):
0220         while len(self._runs) <= index:
0221             self._runs.append(0)
0222         self._runs[index] = runNumber
0223     def runFor(self,index):
0224         return self._runs[index]
0225     def setLumi(self, index, runNumber, lumiNumber):
0226         while len(self._lumis) <= index:
0227             self._lumis.append((0,0))
0228         self._lumis[index] = (runNumber, lumiNumber)
0229     def lumiFor(self, index):
0230         return self._lumis[index]
0231     def setStream(self, index, runNumber, lumiNumber, eventNumber):
0232         while len(self._streams) <= index:
0233             self._streams.append((0,0,0))
0234         self._streams[index] = (runNumber, lumiNumber, eventNumber)
0235     def streamFor(self, index):
0236         return self._streams[index]
0237     def get(self, transition, index):
0238         if transition == Phase.construction or transition == Phase.destruction:
0239             return ()
0240         if transition == Phase.beginJob or transition == Phase.endJob or transition == Phase.openFile:
0241             return ()
0242         if transition == Phase.globalBeginRun or transition == Phase.globalEndRun or transition == Phase.globalWriteRun:
0243             return (self.runFor(index),)
0244         if transition == Phase.globalBeginLumi or transition == Phase.globalEndLumi or transition == Phase.globalWriteLumi:
0245             return self.lumiFor(index)
0246         if transition == Phase.getNextTransition:
0247             return ()
0248         if transition == Phase.writeProcessBlock:
0249             return ()
0250         if transition == Phase.beginStream:
0251             self.setStream(index, 0,0,0)
0252             return ()
0253         if not transitionIsGlobal(transition):
0254             return self.streamFor(index)
0255         raise RuntimeError("Unknown transition {}".format(transition))
0256
0257 class TempModuleTransitionInfos(object):
0258     def __init__(self):
0259         self._times = {}
0260         self._esTimes = {}
0261     def insertTime(self, label, transition, index, time):
0262         self._times[(label, transition, index)] = time
0263     def findTime(self, label, transition, index):
0264         time = self._times[(label, transition, index)]
0265         del self._times[(label, transition, index)]
0266         return time
0267     def insertTimeES(self, label, transition, index, record, call, time):
0268         self._esTimes[(label, transition, index, record, call)] = time
0269     def findTimeES(self, label, transition, index, record, call):
0270         time = self._esTimes[(label, transition, index, record, call)]
0271         del self._esTimes[(label, transition, index, record, call)]
0272         return time
0273
0274 class ModuleData(object):
0275     def __init__(self, start, stop, transition, sync, activity, allocInfo, recordName=None, callID=None):
0276         self.timeRange = (start, stop)
0277         self.transition = transition
0278         self.sync = sync
0279         self.activity = activity
0280         self.allocInfo = allocInfo
0281         self.record = (recordName, callID)
0282     def __repr__(self):
0283         if self.record[0]:
0284             return "{{ 'timeRange': {}, 'transition': {}, 'sync' :{}, 'activity':{}, 'record': {{'name' : {}, 'callID' :{} }}, 'alloc':{} }}".format(self.timeRange, self.transition, self.sync, self.activity, self.record[0], self.record[1], self.allocInfo)
0285
0286         return "{{ 'timeRange': {}, 'transition': {}, 'sync' :{}, 'activity':{}, 'alloc':{} }}".format(self.timeRange, self.transition, self.sync, self.activity, self.allocInfo)
0287     def syncToSimpleDict(self):
0288         if len(self.sync) == 0:
0289             return self.sync
0290         if len(self.sync) == 1:
0291             return {'run' : self.sync[0]}
0292         if len(self.sync) == 2:
0293             return {'run' : self.sync[0], 'lumi' : self.sync[1] }
0294         return {'run' : self.sync[0], 'lumi' : self.sync[1], 'event' : self.sync[2] }
0295     def toSimpleDict(self) :
0296         if self.record[0]:
0297             return {'timeRange': self.timeRange, 'transition': transitionName(self.transition), 'sync' : self.syncToSimpleDict(), 'activity' : activityName(self.activity), 'record' :{'name': self.record[0], 'callID' : self.record[1] }, 'alloc' : self.allocInfo.toSimpleDict() }
0298         return {'timeRange': self.timeRange, 'transition': transitionName(self.transition), 'sync' : self.syncToSimpleDict(), 'activity': activityName(self.activity), 'alloc' : self.allocInfo.toSimpleDict() }
0299
0300
0301 class ModuleCentricModuleData(object):
0302     def __init__(self):
0303         self._data = {}
0304         self._last = {}
0305         self._startTime = None
0306     def setStartTime(self, time):
0307         self._startTime = time
0308     def insert(self, label, start, stop, transition, index, sync, activity, allocInfo, recordName=None, callID=None):
0309         if label not in self._data:
0310             self._data[label] = []
0311         self._data[label].append(ModuleData(start, stop, transition, sync, activity, allocInfo, recordName, callID))
0312         self._last[(label, transition, index, activity)] = self._data[label][-1]
0313     def findLast(self, label, transition, index, activity):
0314         return self._last[(label, transition, index, activity)]
0315     def __repr__(self):
0316         return str(self._data)
0317     def data(self):
0318         return self._data
0319     def toSimpleDict(self):
0320         dct = {'startedMonitoring': self._startTime, 'source' :[], 'clearEvent': [], 'modules' :{}}
0321         modules = dct['modules']
0322         for m,lst in self._data.items():
0323             l = None
0324             if m == 'source':
0325                 l = dct['source']
0326             elif m == 'clearEvent':
0327                 l = dct['clearEvent']
0328             else:
0329                 modules[m]=[]
0330                 l = modules[m]
0331             for d in lst:
0332                 l.append( d.toSimpleDict() )
0333         return dct
0334     def sortModulesBy(self, attribute):
0335         data = []
0336         for m, lst in self._data.items():
0337             data.append((m, max(lst, key = lambda x: getattr(x.allocInfo,attribute))) )
0338         data.sort( key = lambda x: getattr(x[1].allocInfo,attribute), reverse=True)
0339         return list(map(lambda x: (x[0], x[1].toSimpleDict()), data))
0340
0341 class TemporalModuleData(object):
0342     def __init__(self):
0343         self._data = []
0344         self._last = {}
0345         self._startTime = None
0346     def setStartTime(self, time):
0347         self._startTime = time
0348     def insert(self, label, start, stop, transition, index, sync, activity, allocInfo, recordName=None, callID=None):
0349         self._data.append((label, ModuleData(start, stop, transition, sync, activity, allocInfo, recordName, callID)))
0350         self._last[(label,transition, index, activity)] = self._data[-1]
0351     def findLast(self, label, transition,index, activity):
0352         v = self._last.get((label, transition, index, activity), None)
0353         if v:
0354             return v[1]
0355         return None
0356     def __repr__(self):
0357         return str(self._data)
0358     def data(self):
0359         return self._data
0360     def toSimpleDict(self):
0361         dct = {'startedMonitoring': self._startTime, 'measurements' :[]}
0362         measurements = dct['measurements']
0363         for d in self._data:
0364             entry = d[1].toSimpleDict()
0365             entry['label'] = d[0]
0366             measurements.append(entry)
0367         return dct
0368
0369
0370
0371
0372 class FrameworkTransitionParser (object):
0373     def __init__(self, payload):
0374         self.transition = int(payload[0])
0375         self.index = int(payload[1])
0376         self.sync = (int(payload[2]), int(payload[3]), int(payload[4]))
0377         self.time = int(payload[5])
0378     def indentLevel(self):
0379         return transitionIndentLevel(self.transition)
0380     def textPrefix(self):
0381         return textPrefix_(self.time, self.indentLevel())
0382     def syncText(self):
0383         if self.transition == Phase.globalBeginRun or Phase.globalEndRun == self.transition:
0384             return f'run={self.sync[0]}'
0385         if self.transition == Phase.globalWriteRun:
0386             return f'run={self.sync[0]}'
0387         if self.transition == Phase.streamBeginRun or Phase.streamEndRun == self.transition:
0388             return f'run={self.sync[0]}'
0389         if self.transition == Phase.globalBeginLumi or Phase.globalEndLumi == self.transition:
0390             return f'run={self.sync[0]} lumi={self.sync[1]}'
0391         if self.transition == Phase.globalWriteLumi:
0392             return f'run={self.sync[0]} lumi={self.sync[1]}'
0393         if self.transition == Phase.streamBeginLumi or Phase.streamEndLumi == self.transition:
0394             return f'run={self.sync[0]} lumi={self.sync[1]}'
0395         if self.transition == Phase.Event:
0396             return f'run={self.sync[0]} lumi={self.sync[1]} event={self.sync[2]}'
0397         if self.transition == Phase.esSyncEnqueue or self.transition == Phase.esSync:
0398             return f'run={self.sync[0]} lumi={self.sync[1]}'
0399         if self.transition == Phase.beginJob:
0400             return ''
0401         if self.transition == Phase.beginProcessBlock or self.transition == Phase.endProcessBlock or self.transition == Phase.writeProcessBlock or self.transition == Phase.accessInputProcessBlock:
0402             return ''
0403         if self.transition == Phase.startTracing:
0404             return ''
0405         if self.transition == Phase.construction or self.transition == Phase.destruction:
0406             return ''
0407     def textPostfix(self):
0408         return f'{transitionName(self.transition)} : id={self.index} {self.syncText()}'
0409     def text(self, context):
0410         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0411
0412 def findMatchingTransition(sync, containers):
0413     for i in range(len(containers)):
0414         if containers[i][-1]["sync"] == sync:
0415             return i
0416     #need more exhausting search
0417     for i in range(len(containers)):
0418         for t in containers[i]:
0419             if t["sync"] == sync:
0420                 return i
0421
0422     print("find failed",sync, containers)
0423     return None
0424
0425 transitionsToFindMatch_ = {
0426     Phase.globalEndRun,
0427     Phase.globalEndLumi,
0428     Phase.globalWriteRun,
0429     Phase.globalWriteLumi
0430 }
0431
0432 class PreFrameworkTransitionParser (FrameworkTransitionParser):
0433     def __init__(self, payload):
0434         super().__init__(payload)
0435     def textSpecial(self):
0436         return "starting"
0437     def jsonInfo(self, syncs, temp, data):
0438         isSourceTrans = False
0439         if self.transition == Phase.startTracing:
0440             data.setStartTime(self.time)
0441         elif self.transition == Phase.globalBeginRun:
0442             syncs.setRun(self.index, self.sync[0])
0443             isSourceTrans = True
0444         elif self.transition == Phase.globalBeginLumi:
0445             syncs.setLumi(self.index, self.sync[0], self.sync[1])
0446             isSourceTrans = True
0447         elif self.transition == Phase.Event:
0448             syncs.setStream(self.index, self.sync[0], self.sync[1], self.sync[2])
0449             isSourceTrans = True
0450         elif self.transition == Phase.clearEvent:
0451             temp.insertTime("clearEvent", self.transition, self.index, self.time)
0452         elif not transitionIsGlobal(self.index):
0453             syncs.setStream(self.index, self.sync[0], self.sync[1], self.sync[2])
0454         if isSourceTrans:
0455             src = data.findLast("source", self.transition, self.index, Activity.process)
0456             if src.sync != self.index:
0457                 raise RuntimeError("Framework and Source transitions do not have matching index: source {} framework {} for transition type {} at framework time {} and source time {}".format(src.sync, self.index, self.transition, self.time, src.timeRange))
0458             src.sync = syncs.get(self.transition, self.index)
0459     def jsonVisInfo(self,  data):
0460         if transitionIsGlobal(self.transition):
0461             index = 0
0462             if self.transition == Phase.startTracing:
0463                 data.indexedGlobal(0).append(jsonTransition(type=self.transition, id=index, sync=list(self.sync),start=0, finish=self.time ))
0464                 return
0465             elif self.transition==Phase.globalBeginRun:
0466                 index = self.index
0467                 container = data.indexedGlobal(index)
0468                 #find source, should be previous
0469                 last = container[-1]
0470                 if last["type"]==Phase.globalBeginRun and last["isSrc"]:
0471                     last["sync"]=list(self.sync)
0472             elif self.transition==Phase.globalBeginLumi:
0473                 index = self.index
0474                 container = data.indexedGlobal(index)
0475                 #find source, should be previous
0476                 last = container[-1]
0477                 if last["type"]==Phase.globalBeginLumi and last["isSrc"]:
0478                     last["sync"]=list(self.sync)
0479             elif self.transition in transitionsToFindMatch_:
0480                 index = findMatchingTransition(list(self.sync), data.allGlobals())
0481             container = data.indexedGlobal(index)
0482         else:
0483             container = data.indexedStream(self.index)
0484             if self.transition == Phase.Event:
0485                 #find source, should be previous
0486                 last = container[-1]
0487                 if last["type"]==Phase.Event and last["isSrc"]:
0488                     last["sync"]=list(self.sync)
0489             index = self.index
0490         container.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0491
0492
0493
0494 class PostFrameworkTransitionParser (FrameworkTransitionParser):
0495     def __init__(self, payload):
0496         super().__init__(payload)
0497         if self.transition == Phase.clearEvent:
0498             self.alloc = AllocInfo(payload[6:])
0499     def textSpecial(self):
0500         return "finished"
0501     def jsonInfo(self, syncs, temp, data):
0502         if self.transition == Phase.clearEvent:
0503             start = temp.findTime("clearEvent", self.transition, self.index)
0504             data.insert( "clearEvent" , start, self.time, self.transition, self.index, syncs.get(Phase.Event, self.index) , Activity.process, self.alloc)
0505     def jsonVisInfo(self,  data):
0506         if transitionIsGlobal(self.transition):
0507             index = findMatchingTransition(list(self.sync), data.allGlobals())
0508             container = data.indexedGlobal(index)
0509         else:
0510             container = data.indexedStream(self.index)
0511         container[-1]["finish"]=self.time*kMicroToSec
0512         if self.transition == Phase.clearEvent:
0513             self.alloc.inject(container[-1])
0514
0515
0516
0517 class SourceTransitionParser(object):
0518     def __init__(self, payload):
0519         self.transition = int(payload[0])
0520         if self.transition == Phase.getNextTransition:
0521             self.time = int(payload[1])
0522             self.index = -1
0523             return
0524         self.index = int(payload[1])
0525         self.time = int(payload[2])
0526     def indentLevel(self):
0527         if self.transition == Phase.globalBeginRun:
0528             return 1
0529         if self.transition == Phase.globalBeginLumi:
0530             return 2
0531         if self.transition == Phase.Event:
0532             return 3
0533         if self.transition == Phase.construction:
0534             return 1
0535         if self.transition == Phase.getNextTransition:
0536             return 1
0537         if self.transition == Phase.openFile:
0538             return 1
0539         return None
0540     def textPrefix(self):
0541         return textPrefix_(self.time, self.indentLevel())
0542     def textPostfix(self):
0543         return f'source during {transitionName(self.transition)} : id={self.index}'
0544     def text(self, context):
0545         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0546
0547 class PreSourceTransitionParser(SourceTransitionParser):
0548     def __init__(self, payload, moduleCentric):
0549         self._moduleCentric = moduleCentric
0550         super().__init__(payload)
0551     def textSpecial(self):
0552         return "starting"
0553     def jsonInfo(self, syncs, temp, data):
0554         temp.insertTime("source", self.transition, self.index, self.time)
0555     def jsonVisInfo(self,  data):
0556         if self.transition == Phase.getNextTransition:
0557             data._nextTrans.append(jsonTransition(type=self.transition, id=self.index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0558             if self._moduleCentric:
0559                 #this all goes to a module ID sorted container so not knowing actual index is OK
0560                 data.findOpenSlotInModGlobals(0,0).append(data._nextTrans[-1])
0561             return
0562         elif self.transition == Phase.construction:
0563             index = 0
0564             container = data.indexedGlobal(index)
0565         elif self.transition == Phase.Event:
0566             index = self.index
0567             container = data.indexedStream(index)
0568         else:
0569             index = self.index
0570             container = data.indexedGlobal(index)
0571         nextTrans = data._nextTrans
0572         if nextTrans:
0573             data._nextTrans = []
0574             for t in nextTrans:
0575                 t['id']=index
0576                 #find proper time order in the container
0577                 transStartTime = t['start']
0578                 inserted = False
0579                 for i in range(-1, -1*len(container), -1):
0580                     if transStartTime > container[i]['start']:
0581                         if i == -1:
0582                             container.append(t)
0583                             inserted = True
0584                             break
0585                         else:
0586                             container.insert(i+1,t)
0587                             inserted = True
0588                             break
0589                 if not inserted:
0590                     container.insert(0,t)
0591         container.append(jsonTransition(type=self.transition, id=index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0592         if self._moduleCentric:
0593             if self.transition == Phase.Event:
0594                 data.findOpenSlotInModStreams(index,0).append(container[-1])
0595             else:
0596                 data.findOpenSlotInModGlobals(index,0).append(container[-1])
0597
0598 class PostSourceTransitionParser(SourceTransitionParser):
0599     def __init__(self, payload, moduleCentric):
0600         super().__init__(payload)
0601         #print(payload)
0602         if self.index == -1:
0603             self.allocInfo = AllocInfo(payload[2:])
0604         else:
0605             self.allocInfo = AllocInfo(payload[3:])
0606         self._moduleCentric = moduleCentric
0607     def textSpecial(self):
0608         return "finished"
0609     def jsonInfo(self, syncs, temp, data):
0610         start = temp.findTime("source", self.transition, self.index)
0611         #we do not know the sync yet so have to wait until the framework transition
0612         if self.transition in [ Phase.construction, Phase.getNextTransition, Phase.destruction, Phase.openFile]:
0613             data.insert( "source" , start, self.time, self.transition, self.index, (0,) , Activity.process, self.allocInfo)
0614         else:
0615             data.insert( "source" , start, self.time, self.transition, self.index, self.index , Activity.process, self.allocInfo)
0616     def jsonVisInfo(self,  data):
0617         index = self.index
0618         if self.transition == Phase.Event:
0619             container = data.indexedStream(index)
0620         elif self.transition == Phase.getNextTransition:
0621             data._nextTrans[-1]['finish'] = self.time*kMicroToSec
0622             self.allocInfo.inject(data._nextTrans[-1])
0623             return
0624         elif self.transition == Phase.construction:
0625             pre = None
0626             for i, g in enumerate(data.allGlobals()):
0627                 for t in reversed(g):
0628                     if t["type"] != Phase.construction:
0629                         break
0630                     if t["isSrc"]:
0631                         pre = t
0632                         break
0633                 if pre:
0634                     pre["finish"]=self.time*kMicroToSec
0635                     self.allocInfo.inject(pre)
0636                     break
0637             return
0638         else:
0639             container = data.indexedGlobal(index)
0640
0641         container[-1]["finish"]=self.time*kMicroToSec
0642         self.allocInfo.inject(container[-1])
0643
0644 class EDModuleTransitionParser(object):
0645     def __init__(self, payload, moduleNames):
0646         self.transition = int(payload[0])
0647         self.index = int(payload[1])
0648         self.moduleID = int(payload[2])
0649         self.moduleName = moduleNames[self.moduleID]
0650         self.callID = int(payload[3])
0651         self.time = int(payload[4])
0652     def baseIndentLevel(self):
0653         return transitionIndentLevel(self.transition)
0654     def textPrefix(self, context):
0655         indent = 0
0656         context[(self.transition, self.index, self.moduleID, self.callID)] = indent+1
0657         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0658     def textPostfix(self):
0659         return f'{self.moduleName} during {transitionName(self.transition)} : id={self.index}'
0660     def textIfTransform(self):
0661         if self.callID:
0662             return f' transform {self.callID-1}'
0663         return ''
0664     def text(self, context):
0665         return f'{self.textPrefix(context)} {self.textSpecial()}{self.textIfTransform()}: {self.textPostfix()}'
0666     def _preJsonVis(self, activity,  data, mayUseTemp = False):
0667         index = self.index
0668         found = False
0669         if mayUseTemp:
0670             compare = lambda x: x['type'] == self.transition and x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and (x['act'] == Activity.temporary or x['act'] == Activity.externalWork)
0671             if transitionIsGlobal(self.transition):
0672                 item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0673             else:
0674                 item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0675             if slot:
0676                 if item['act'] == Activity.temporary:
0677                     slot.pop()
0678                 else:
0679                     item['finish']=self.time*kMicroToSec
0680                 found = True
0681         if not found:
0682             if transitionIsGlobal(self.transition):
0683                 slot = data.findOpenSlotInModGlobals(index, self.moduleID)
0684             else:
0685                 slot = data.findOpenSlotInModStreams(index, self.moduleID)
0686         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=activity, start=self.time))
0687         return slot[-1]
0688     def _postJsonVis(self,  data, alloc, injectAfter = None):
0689         compare = lambda x: x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and x['type'] == self.transition
0690         index = self.index
0691         if transitionIsGlobal(self.transition):
0692             item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0693         else:
0694             item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0695         if item is None:
0696             print(f"failed to find {self.moduleID} for {self.transition} in {self.index}")
0697         else:
0698             item["finish"]=self.time*kMicroToSec
0699             alloc.inject(item)
0700             if injectAfter:
0701                 slot.append(injectAfter)
0702     def _preJsonInfo(self, temp):
0703         temp.insertTime(self.moduleName, self.transition, self.index, self.time)
0704     def _postJsonInfo(self, syncs, temp, data, activity):
0705         start = temp.findTime(self.moduleName, self.transition, self.index)
0706         data.insert( self.moduleName , start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , activity, self.allocInfo)
0707
0708
0709 class PreEDModuleTransitionParser(EDModuleTransitionParser):
0710     def __init__(self, payload, names, moduleCentric):
0711         super().__init__(payload, names)
0712         self._moduleCentric = moduleCentric
0713     def textSpecial(self):
0714         return "starting action"
0715     def jsonVisInfo(self,  data):
0716         return self._preJsonVis(Activity.process, data, mayUseTemp=self._moduleCentric)
0717     def jsonInfo(self, syncs, temp, data):
0718         self._preJsonInfo(temp)
0719
0720
0721 class PostEDModuleTransitionParser(EDModuleTransitionParser):
0722     def __init__(self, payload, names):
0723         super().__init__(payload, names)
0724         self.allocInfo = AllocInfo(payload[5:])
0725     def textSpecial(self):
0726         return "finished action"
0727     def jsonInfo(self, syncs, temp, data):
0728         self._postJsonInfo(syncs, temp, data, Activity.process)
0729     def jsonVisInfo(self,  data):
0730         return self._postJsonVis(data, self.allocInfo)
0731
0732
0733 class PreEDModuleAcquireParser(EDModuleTransitionParser):
0734     def __init__(self, payload, names, moduleCentric):
0735         super().__init__(payload, names)
0736         self._moduleCentric = moduleCentric
0737     def textSpecial(self):
0738         return "starting acquire"
0739     def jsonVisInfo(self,  data):
0740         return self._preJsonVis(Activity.acquire, data, mayUseTemp=self._moduleCentric)
0741     def jsonInfo(self, syncs, temp, data):
0742         self._preJsonInfo(temp)
0743
0744
0745 class PostEDModuleAcquireParser(EDModuleTransitionParser):
0746     def __init__(self, payload, names, moduleCentric):
0747         super().__init__(payload, names)
0748         self.allocInfo = AllocInfo(payload[5:])
0749         self._moduleCentric = moduleCentric
0750     def textSpecial(self):
0751         return "finished acquire"
0752     def jsonVisInfo(self,  data):
0753         if self._moduleCentric:
0754             #inject an external work at end of the same slot to guarantee module run is in that slot
0755             return self._postJsonVis( data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.externalWork, start=self.time))
0756         return self._postJsonVis(data, self.allocInfo)
0757     def jsonInfo(self, syncs, temp, data):
0758         self._postJsonInfo(syncs, temp, data, Activity.acquire)
0759
0760 class PreEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0761     def __init__(self, payload, names):
0762         super().__init__(payload, names)
0763     def textSpecial(self):
0764         return "starting delayed get"
0765     def jsonVisInfo(self,  data):
0766         return self._preJsonVis(Activity.delayedGet, data)
0767     def jsonInfo(self, syncs, temp, data):
0768         pass
0769         #self._preJsonInfo(temp)
0770
0771 class PostEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0772     def __init__(self, payload, names):
0773         super().__init__(payload, names)
0774         self.allocInfo = AllocInfo(payload[5:])
0775     def textSpecial(self):
0776         return "finished delayed get"
0777     def jsonVisInfo(self,  data):
0778         return self._postJsonVis(data, self.allocInfo)
0779     def jsonInfo(self, syncs, temp, data):
0780         pass
0781         #self._postJsonInfo(syncs, temp, data, Activity.delayedGet)
0782
0783 class PreEventReadFromSourceParser(EDModuleTransitionParser):
0784     def __init__(self, payload, names):
0785         super().__init__(payload, names)
0786     def textSpecial(self):
0787         return "starting read from source"
0788     def jsonVisInfo(self,  data):
0789         slot = self._preJsonVis(Activity.process, data)
0790         slot['isSrc'] = True
0791         return slot
0792     def jsonInfo(self, syncs, temp, data):
0793         temp.insertTime(self.moduleName+'source', self.transition, self.index, self.time)
0794
0795 class PostEventReadFromSourceParser(EDModuleTransitionParser):
0796     def __init__(self, payload, names):
0797         super().__init__(payload, names)
0798         self.allocInfo = AllocInfo(payload[5:])
0799     def textSpecial(self):
0800         return "finished read from source"
0801     def jsonVisInfo(self,  data):
0802         return self._postJsonVis(data, self.allocInfo)
0803     def jsonInfo(self, syncs, temp, data):
0804         start = temp.findTime(self.moduleName+'source', self.transition, self.index)
0805         data.insert( "source" , start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , Activity.delayedGet, self.allocInfo)
0806
0807 class ESModuleTransitionParser(object):
0808     def __init__(self, payload, moduleNames, esModuleNames, recordNames):
0809         self.transition = int(payload[0])
0810         self.index = int(payload[1])
0811         self.moduleID = int(payload[2])
0812         self.moduleName = esModuleNames[self.moduleID]
0813         self.recordID = int(payload[3])
0814         self.recordName = recordNames[self.recordID]
0815         self.callID = int(payload[4])
0816         self.time = int(payload[5])
0817     def baseIndentLevel(self):
0818         return transitionIndentLevel(self.transition)
0819     def textPrefix(self, context):
0820         indent = 0
0821         context[(self.transition, self.index, -1*self.moduleID, self.callID)] = indent+1
0822         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0823     def textPostfix(self):
0824         return f'esmodule {self.moduleName} in record {self.recordName} during {transitionName(self.transition)} : id={self.index}'
0825     def text(self, context):
0826         return f'{self.textPrefix(context)} {self.textSpecial()}: {self.textPostfix()}'
0827     def _preJsonVis(self, activity,  data):
0828         index = self.index
0829         if transitionIsGlobal(self.transition):
0830             slot = data.findOpenSlotInModGlobals(index, -1*self.moduleID)
0831         else:
0832             slot = data.findOpenSlotInModStreams(index, -1*self.moduleID)
0833         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=-1*self.moduleID, callID=self.callID, activity=activity, start=self.time))
0834         return slot[-1]
0835     def _postJsonVis(self,  data, alloc):
0836         compare = lambda x: x['id'] == self.index and x['mod'] == -1*self.moduleID and x['call'] == self.callID
0837         index = self.index
0838         if transitionIsGlobal(self.transition):
0839             item,s = data.findLastInModGlobals(index, -1*self.moduleID, compare)
0840         else:
0841             item,s = data.findLastInModStreams(index, -1*self.moduleID, compare)
0842         if item is None:
0843             print(f"failed to find {-1*self.moduleID} for {self.transition} in {self.index}")
0844             return
0845         item["finish"]=self.time*kMicroToSec
0846         alloc.inject(item)
0847     def _preJsonInfo(self, temp):
0848         temp.insertTimeES(self.moduleName, self.transition, self.index, self.recordID, self.callID, self.time)
0849     def _postJsonInfo(self, syncs, temp, data, activity):
0850         start = temp.findTimeES(self.moduleName, self.transition, self.index, self.recordID, self.callID)
0851         data.insert( self.moduleName , start, self.time, self.transition, self.index, syncs.get(self.transition, self.index) , activity, self.allocInfo, self.recordName, self.callID)
0852
0853
0854 class PreESModuleTransitionParser(ESModuleTransitionParser):
0855     def __init__(self, payload, names, esNames, recordNames):
0856         super().__init__(payload, names, esNames, recordNames)
0857     def textSpecial(self):
0858         return "starting action"
0859     def jsonVisInfo(self,  data):
0860         return self._preJsonVis(Activity.process, data)
0861     def jsonInfo(self, syncs, temp, data):
0862         self._preJsonInfo(temp)
0863
0864 class PostESModuleTransitionParser(ESModuleTransitionParser):
0865     def __init__(self, payload, names, esNames, recordNames):
0866         super().__init__(payload, names, esNames, recordNames)
0867         self.allocInfo = AllocInfo(payload[6:])
0868     def textSpecial(self):
0869         return "finished action"
0870     def jsonVisInfo(self,  data):
0871         return self._postJsonVis(data,self.allocInfo)
0872     def jsonInfo(self, syncs, temp, data):
0873         self._postJsonInfo(syncs, temp, data, Activity.process)
0874
0875 class PreESModuleAcquireParser(ESModuleTransitionParser):
0876     def __init__(self, payload, names, recordNames):
0877         super().__init__(payload, names, recordNames)
0878     def textSpecial(self):
0879         return "starting acquire"
0880     def jsonVisInfo(self,  data):
0881         return self._preJsonVis(Activity.acquire, data)
0882
0883 class PostESModuleAcquireParser(ESModuleTransitionParser):
0884     def __init__(self, payload, names, esNames, recordNames):
0885         super().__init__(payload, names, esNames, recordNames)
0886         self.allocInfo = AllocInfo(payload[6:])
0887     def textSpecial(self):
0888         return "finished acquire"
0889     def jsonVisInfo(self,  data):
0890         return self._postJsonVis(data, self.allocInfo)
0891
0892
0893 def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, moduleCentric):
0894     if step == 'F':
0895         parser = PreFrameworkTransitionParser(payload)
0896         return parser
0897     if step == 'f':
0898         return PostFrameworkTransitionParser(payload)
0899     if step == 'S':
0900         return PreSourceTransitionParser(payload, moduleCentric)
0901     if step == 's':
0902         return PostSourceTransitionParser(payload, moduleCentric)
0903     if step == 'M':
0904         return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
0905     if step == 'm':
0906         return PostEDModuleTransitionParser(payload, moduleNames)
0907     if step == 'A':
0908         return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0909     if step == 'a':
0910         return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0911     if step == 'D':
0912         return PreEDModuleEventDelayedGetParser(payload, moduleNames)
0913     if step == 'd':
0914         return PostEDModuleEventDelayedGetParser(payload, moduleNames)
0915     if step == 'R':
0916         return PreEventReadFromSourceParser(payload, moduleNames)
0917     if step == 'r':
0918         return PostEventReadFromSourceParser(payload, moduleNames)
0919     if step == 'N':
0920         return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0921     if step == 'n':
0922         return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0923     if step == 'B':
0924         return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0925     if step == 'b':
0926         return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0927     raise LogicError("Unknown step '{}'".format(step))
0928
0929 #----------------------------------------------
0930 def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, moduleCentric):
0931     for rawl in f:
0932         l = rawl.strip()
0933         if not l or l[0] == '#':
0934             continue
0935         (step,payload) = tuple(l.split(None,1))
0936         payload=payload.split()
0937
0938         parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0939         if parser:
0940             yield parser
0941     return
0942
0943 class ModuleAllocCompactFileParser(object):
0944     def __init__(self,f, moduleCentric):
0945         streamBeginRun = str(Phase.streamBeginRun)
0946         numStreams = 0
0947         numStreamsFromSource = 0
0948         moduleNames = {}
0949         esModuleNames = {}
0950         recordNames = {}
0951         for rawl in f:
0952             l = rawl.strip()
0953             if l and l[0] == 'M':
0954                 i = l.split(' ')
0955                 if i[3] == streamBeginRun:
0956                     #found global begin run
0957                     numStreams = int(i[1])+1
0958                     break
0959             if numStreams == 0 and l and l[0] == 'S':
0960                 s = int(l.split(' ')[1])
0961                 if s > numStreamsFromSource:
0962                   numStreamsFromSource = s
0963             if len(l) > 5 and l[0:2] == "#M":
0964                 (id,name)=tuple(l[2:].split())
0965                 moduleNames[int(id)] = name
0966                 continue
0967             if len(l) > 5 and l[0:2] == "#N":
0968                 (id,name)=tuple(l[2:].split())
0969                 esModuleNames[int(id)] = name
0970                 continue
0971             if len(l) > 5 and l[0:2] == "#R":
0972                 (id,name)=tuple(l[2:].split())
0973                 recordNames[int(id)] = name
0974                 continue
0975
0976         self._f = f
0977         self._moduleCentric = moduleCentric
0978         if numStreams == 0:
0979           numStreams = numStreamsFromSource +2
0980         self.numStreams =numStreams
0981         self._moduleNames = moduleNames
0982         self._esModuleNames = esModuleNames
0983         self._recordNames = recordNames
0984         self.maxNameSize =0
0985         for n in moduleNames.items():
0986             self.maxNameSize = max(self.maxNameSize,len(n))
0987         for n in esModuleNames.items():
0988             self.maxNameSize = max(self.maxNameSize,len(n))
0989         self.maxNameSize = max(self.maxNameSize,len(kSourceDelayedRead))
0990         self.maxNameSize = max(self.maxNameSize, len('streamBeginLumi'))
0991
0992     def processingSteps(self):
0993         """Create a generator which can step through the file and return each processing step.
0994         Using a generator reduces the memory overhead when parsing a large file.
0995             """
0996         self._f.seek(0)
0997         return processingStepsFromFile(self._f,self._moduleNames, self._esModuleNames, self._recordNames,  self._moduleCentric)
0998
0999 def textOutput( parser ):
1000     context = {}
1001     for p in parser.processingSteps():
1002         print(p.text(context))
1003
1004 class VisualizationContainers(object):
1005     def __init__(self):
1006         self._modGlobals = [[]]
1007         self._modStreams = [[]]
1008         self._globals = [[]]
1009         self._streams = [[]]
1010         self._nextTrans = []
1011     def _extendIfNeeded(self, container, index):
1012         while len(container) < index+1:
1013             container.append([])
1014     def allGlobals(self):
1015         return self._globals
1016     def indexedGlobal(self, index):
1017         self._extendIfNeeded(self._globals, index)
1018         return self._globals[index]
1019     def allStreams(self):
1020         return self._streams
1021     def indexedStream(self, index):
1022         self._extendIfNeeded(self._streams, index)
1023         return self._streams[index]
1024     def _findOpenSlot(self, index, fullContainer):
1025         self._extendIfNeeded(fullContainer, index)
1026         container = fullContainer[index]
1027         #find open slot
1028         foundOpenSlot = False
1029         for slot in container:
1030             if len(slot) == 0:
1031                 foundOpenSlot = True
1032                 break
1033             if slot[-1]["finish"] != 0:
1034                 foundOpenSlot = True
1035                 break
1036         if not foundOpenSlot:
1037             container.append([])
1038             slot = container[-1]
1039         return slot
1040     def findOpenSlotInModGlobals(self, index, modID):
1041         return self._findOpenSlot(index, self._modGlobals)
1042     def findOpenSlotInModStreams(self, index, modID):
1043         return self._findOpenSlot(index, self._modStreams)
1044     def _findLastIn(self, index, fullContainer, comparer):
1045         container = fullContainer[index]
1046         #find slot containing the pre
1047         for slot in container:
1048             if comparer(slot[-1]):
1049                 return (slot[-1],slot)
1050         return (None,None)
1051     def findLastInModGlobals(self, index, modID, comparer):
1052         return self._findLastIn(index, self._modGlobals, comparer)
1053     def findLastInModStreams(self, index, modID, comparer):
1054         return self._findLastIn(index, self._modStreams, comparer)
1055
1056
1057 class ModuleCentricVisualizationContainers(object):
1058     def __init__(self):
1059         self._modules= []
1060         self._globals = [[]]
1061         self._streams = [[]]
1062         self._nextTrans = []
1063         self._moduleIDOffset = 0
1064     def _moduleID2Index(self, modID):
1065         return modID + self._moduleIDOffset
1066     def _extendIfNeeded(self, container, index):
1067         while len(container) < index+1:
1068             container.append([])
1069     def _extendModulesIfNeeded(self, container, index):
1070         while index + self._moduleIDOffset < 0:
1071             container.insert(0,[])
1072             self._moduleIDOffset +=1
1073         self._extendIfNeeded(container, self._moduleID2Index(index))
1074     def allGlobals(self):
1075         return self._globals
1076     def indexedGlobal(self, index):
1077         self._extendIfNeeded(self._globals, index)
1078         return self._globals[index]
1079     def allStreams(self):
1080         return self._streams
1081     def indexedStream(self, index):
1082         self._extendIfNeeded(self._streams, index)
1083         return self._streams[index]
1084     def _findOpenSlot(self, index, fullContainer):
1085         self._extendModulesIfNeeded(fullContainer, index)
1086         container = fullContainer[self._moduleID2Index(index)]
1087         #find open slot
1088         foundOpenSlot = False
1089         for slot in container:
1090             if len(slot) == 0:
1091                 foundOpenSlot = True
1092                 break
1093             if slot[-1]["finish"] != 0:
1094                 foundOpenSlot = True
1095                 break
1096         if not foundOpenSlot:
1097             container.append([])
1098             slot = container[-1]
1099         return slot
1100     def findOpenSlotInModGlobals(self, index, modID):
1101         return self._findOpenSlot(modID, self._modules)
1102     def findOpenSlotInModStreams(self, index, modID):
1103         return self._findOpenSlot(modID, self._modules)
1104     def _findLastIn(self, index, fullContainer, comparer):
1105         if not fullContainer:
1106             return (None, None)
1107         if len(fullContainer) > self._moduleID2Index(index):
1108             container = fullContainer[self._moduleID2Index(index)]
1109         else:
1110             return (None, None)
1111         #find slot containing the pre
1112         for slot in container:
1113             if slot is not None and comparer(slot[-1]):
1114                 return (slot[-1],slot)
1115         return (None, None)
1116     def findLastInModGlobals(self, index, modID, comparer):
1117         return self._findLastIn(modID, self._modules, comparer)
1118     def findLastInModStreams(self, index, modID, comparer):
1119         return self._findLastIn(modID, self._modules, comparer)
1120
1121
1122 class ModuleCentricContainers(object):
1123     def __init__(self):
1124         self._modules= []
1125         self._globals = [[]]
1126         self._streams = [[]]
1127         self._nextTrans = []
1128         self._moduleIDOffset = 0
1129     def _moduleID2Index(self, modID):
1130         return modID + self._moduleIDOffset
1131     def _extendIfNeeded(self, container, index):
1132         while len(container) < index+1:
1133             container.append([])
1134     def _extendModulesIfNeeded(self, container, index):
1135         while index + self._moduleIDOffset < 0:
1136             container.insert(0,[])
1137             self._moduleIDOffset +=1
1138         self._extendIfNeeded(container, self._moduleID2Index(index))
1139     def allGlobals(self):
1140         return self._globals
1141     def indexedGlobal(self, index):
1142         self._extendIfNeeded(self._globals, index)
1143         return self._globals[index]
1144     def allStreams(self):
1145         return self._streams
1146     def indexedStream(self, index):
1147         self._extendIfNeeded(self._streams, index)
1148         return self._streams[index]
1149     def _findOpenSlot(self, index, fullContainer):
1150         self._extendModulesIfNeeded(fullContainer, index)
1151         container = fullContainer[self._moduleID2Index(index)]
1152         #find open slot
1153         foundOpenSlot = False
1154         for slot in container:
1155             if len(slot) == 0:
1156                 foundOpenSlot = True
1157                 break
1158             if slot[-1]["finish"] != 0:
1159                 foundOpenSlot = True
1160                 break
1161         if not foundOpenSlot:
1162             container.append([])
1163             slot = container[-1]
1164         return slot
1165     def findOpenSlotInModGlobals(self, index, modID):
1166         return self._findOpenSlot(modID, self._modules)
1167     def findOpenSlotInModStreams(self, index, modID):
1168         return self._findOpenSlot(modID, self._modules)
1169     def _findLastIn(self, index, fullContainer, comparer):
1170         if not fullContainer:
1171             return (None, None)
1172         if len(fullContainer) > self._moduleID2Index(index):
1173             container = fullContainer[self._moduleID2Index(index)]
1174         else:
1175             return (None, None)
1176         #find slot containing the pre
1177         for slot in container:
1178             if slot is not None and comparer(slot[-1]):
1179                 return (slot[-1],slot)
1180         return (None, None)
1181     def findLastInModGlobals(self, index, modID, comparer):
1182         return self._findLastIn(modID, self._modules, comparer)
1183     def findLastInModStreams(self, index, modID, comparer):
1184         return self._findLastIn(modID, self._modules, comparer)
1185
1186
1187
1188 def jsonTransition(type, id, sync, start, finish, isSrc=False):
1189     return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}
1190
1191 def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
1192     return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}
1193
1194
1195 def startTime(x):
1196     return x["start"]
1197
1198 def jsonInfo(parser, temporal):
1199     sync = SyncValues()
1200     #used to keep track of outstanding module transitions
1201     temp = TempModuleTransitionInfos()
1202     if temporal:
1203         data = TemporalModuleData()
1204     else:
1205         data = ModuleCentricModuleData()
1206     for p in parser.processingSteps():
1207         if hasattr(p, "jsonInfo"):
1208             p.jsonInfo(sync,temp, data)
1209     return data
1210
1211 def sortByAttribute(parser, attribute):
1212     sync = SyncValues()
1213     #used to keep track of outstanding module transitions
1214     temp = TempModuleTransitionInfos()
1215     data = ModuleCentricModuleData()
1216     for p in parser.processingSteps():
1217         if hasattr(p, "jsonInfo"):
1218             p.jsonInfo(sync,temp, data)
1219     return data.sortModulesBy(attribute)
1220
1221 def jsonVisualizationInfo(parser):
1222     if parser._moduleCentric:
1223         data = ModuleCentricContainers()
1224     else:
1225         data = VisualizationContainers()
1226     for p in parser.processingSteps():
1227         p.jsonVisInfo( data)
1228     #make sure everything is sorted
1229     for g in data.allGlobals():
1230         g.sort(key=startTime)
1231     final = {"transitions" : [] , "modules": [], "esModules": []}
1232     final["transitions"].append({ "name":"Global", "slots": []})
1233     globals = final["transitions"][-1]["slots"]
1234     for i, g in enumerate(data.allGlobals()):
1235         globals.append(g)
1236         if not parser._moduleCentric:
1237             if len(data._modGlobals) < i+1:
1238                 break
1239             for mod in data._modGlobals[i]:
1240                 globals.append(mod)
1241     for i,s in enumerate(data.allStreams()):
1242         final["transitions"].append({"name": f"Stream {i}", "slots":[]})
1243         stream = final["transitions"][-1]["slots"]
1244         stream.append(s)
1245         if not parser._moduleCentric:
1246             for mod in data._modStreams[i]:
1247                 stream.append(mod)
1248     if parser._moduleCentric:
1249         sourceSlot = data._modules[data._moduleID2Index(0)]
1250         modules = []
1251         for i,m in parser._moduleNames.items():
1252             modules.append({"name": f"{m}", "slots":[]})
1253             slots = modules[-1]["slots"]
1254             foundSlots = data._modules[data._moduleID2Index(i)]
1255             time = 0
1256             for s in foundSlots:
1257                 slots.append(s)
1258                 for t in s:
1259                     if t["act"] !=Activity.prefetch:
1260                         time += t["finish"]-t["start"]
1261             modules[-1]['time']=time
1262         for i,m in parser._esModuleNames.items():
1263             modules.append({"name": f"{m}", "slots":[]})
1264             slots = modules[-1]["slots"]
1265             foundSlots = data._modules[data._moduleID2Index(-1*i)]
1266             time = 0
1267             for s in foundSlots:
1268                 slots.append(s)
1269                 for t in s:
1270                     if t["act"] !=Activity.prefetch:
1271                         time += t["finish"]-t["start"]
1272             modules[-1]['time']=time
1273         modules.sort(key= lambda x : x['time'], reverse=True)
1274         final['transitions'].append({"name": "source", "slots":sourceSlot})
1275         for m in modules:
1276             final['transitions'].append(m)
1277
1278     max = 0
1279     for k in parser._moduleNames.keys():
1280         if k > max:
1281             max = k
1282
1283     final["modules"] =['']*(max+1)
1284     final["modules"][0] = 'source'
1285     for k,v in parser._moduleNames.items():
1286         final["modules"][k]=v
1287
1288     max = 0
1289     for k in parser._esModuleNames.keys():
1290         if k > max:
1291             max = k
1292     final["esModules"] = ['']*(max+1)
1293     for k,v in parser._esModuleNames.items():
1294         final["esModules"][k] = v
1295     return final
1296
1297
1298
1299 #=======================================
1300 import unittest
1301
1302 class DummyFile(list):
1303     def __init__(self):
1304         super()
1305     def seek(self, i):
1306         pass
1307
1308 class TestModuleCommand(unittest.TestCase):
1309     def setUp(self):
1310         self.tracerFile = DummyFile()
1311         t = [0]
1312         def incr(t):
1313             t[0] += 1
1314             return t[0]
1315
1316         self.tracerFile.extend([
1317             '#R 1 Record',
1318             '#M 1 Module',
1319             '#N 1 ESModule',
1320              f'F {Phase.startTracing} 0 0 0 0 {incr(t)}',
1321              f'S {Phase.construction} 0 {incr(t)}',
1322              f's {Phase.construction} 0 {incr(t)} 1 1 10 0 10 10',
1323              f'M {Phase.construction} 0 1 0 {incr(t)}',
1324              f'm {Phase.construction} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1325              f'F {Phase.beginJob} 0 0 0 0 {incr(t)}',
1326              f'M {Phase.beginJob} 0 1 0 {incr(t)}',
1327              f'm {Phase.beginJob} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1328              f'f {Phase.beginJob} 0 0 0 0 {incr(t)}',
1329              f'F {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1330              f'f {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1331              f'S {Phase.getNextTransition} {incr(t)}',
1332              f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1333              f'S {Phase.globalBeginRun} 0 {incr(t)}',
1334              f's {Phase.globalBeginRun} 0 {incr(t)} 1 1 10 0 10 10',
1335              f'S {Phase.getNextTransition} {incr(t)}',
1336              f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1337              f'F {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1338              f'M {Phase.globalBeginRun} 0 1 0 {incr(t)}',
1339              f'm {Phase.globalBeginRun} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1340              f'f {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1341              f'S {Phase.getNextTransition} {incr(t)}',
1342              f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1343              f'F {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1344              f'M {Phase.streamBeginRun} 0 1 0 {incr(t)}',
1345              f'm {Phase.streamBeginRun} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1346              f'f {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1347              f'F {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1348              f'M {Phase.streamBeginRun} 1 1 0 {incr(t)}',
1349              f'm {Phase.streamBeginRun} 1 1 0 {incr(t)} 3 2 20 0 50 25',
1350              f'f {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1351              f'S {Phase.globalBeginLumi} 0 {incr(t)}',
1352              f's {Phase.globalBeginLumi} 0 {incr(t)} 1 1 10 0 10 10',
1353              f'S {Phase.getNextTransition} {incr(t)}',
1354              f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1355              f'F {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1356              f'M {Phase.globalBeginLumi} 0 1 0 {incr(t)}',
1357              f'm {Phase.globalBeginLumi} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1358              f'f {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1359              f'F {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1360              f'f {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1361              f'F {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1362              f'f {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1363              f'S {Phase.Event} 0 {incr(t)}',
1364              f's {Phase.Event} 0 {incr(t)} 1 1 10 0 10 10',
1365              f'S {Phase.getNextTransition} {incr(t)}',
1366              f's {Phase.getNextTransition} {incr(t)} 1 1 10 0 10 10',
1367              f'F {Phase.Event} 0 1 1 1 {incr(t)}',
1368              f'S {Phase.Event} 1 {incr(t)}',
1369              f's {Phase.Event} 1 {incr(t)} 1 1 10 0 10 10',
1370              f'F {Phase.Event} 1 1 1 2 {incr(t)}',
1371              f'N {Phase.Event} 0 1 1 0 {incr(t)}',
1372              f'n {Phase.Event} 0 1 1 0 {incr(t)} 6 5 30 0 100 80',
1373              f'M {Phase.Event} 0 1 0 {incr(t)}',
1374              f'M {Phase.Event} 1 1 0 {incr(t)}',
1375              f'm {Phase.Event} 1 1 0 {incr(t)} 3 2 20 0 50 25',
1376              f'm {Phase.Event} 0 1 0 {incr(t)} 3 2 20 0 50 25',
1377              f'f {Phase.Event} 0 1 1 1 {incr(t)}',
1378              f'f {Phase.Event} 1 1 1 2 {incr(t)}'])
1379
1380         None
1381     def testSyncValues(self):
1382         s = SyncValues()
1383         s.setRun(0,1)
1384         self.assertEqual(s.runFor(0), 1)
1385         s.setLumi(0,1, 1)
1386         self.assertEqual(s.lumiFor(0), (1,1))
1387         s.setStream(1, 1,1,3)
1388         self.assertEqual(s.streamFor(1), (1,1,3))
1389     def testContainers(self):
1390         c = VisualizationContainers()
1391         c.indexedGlobal(2)
1392         self.assertEqual(len(c.allGlobals()), 3)
1393         c.indexedStream(2)
1394         self.assertEqual(len(c.allStreams()), 3)
1395         slot = c.findOpenSlotInModGlobals(2, 1)
1396         self.assertEqual(len(c._modGlobals),3)
1397         self.assertEqual(len(slot),0)
1398         slot.append({"start":1, "finish":0, "id":1})
1399         def testFind(item):
1400             return item["id"]==1
1401         item,s = c.findLastInModGlobals(2, 1, testFind)
1402         self.assertEqual(item["id"],1)
1403         self.assertEqual(slot,s)
1404         slot = c.findOpenSlotInModStreams(2, 1)
1405         self.assertEqual(len(c._modStreams),3)
1406         self.assertEqual(len(slot),0)
1407         slot.append({"start":1, "finish":0, "id":1})
1408         item,s = c.findLastInModStreams(2, 1, testFind)
1409         self.assertEqual(item["id"],1)
1410         self.assertEqual(slot,s)
1411     def testJson(self):
1412         parser = ModuleAllocCompactFileParser(self.tracerFile, False)
1413         j = jsonInfo(parser, False)
1414         self.assertEqual(len(j.data()),3)
1415         self.assertEqual(len(j.data()["source"]), 10)
1416         self.assertEqual(len(j.data()["Module"]), 8)
1417         self.assertEqual(len(j.data()["ESModule"]), 1)
1418     def testJsonTemporal(self):
1419         parser = ModuleAllocCompactFileParser(self.tracerFile, True)
1420         j = jsonInfo(parser, True)
1421         self.assertEqual(len(j.data()),19)
1422     def testSortBy(self):
1423         parser = ModuleAllocCompactFileParser(self.tracerFile, True)
1424         d = sortByAttribute(parser, 'maxTemp')
1425         #print(d)
1426         self.assertEqual(len(d), 3)
1427         self.assertEqual(d[0][0], 'ESModule')
1428         self.assertEqual(d[1][0], 'Module')
1429         self.assertEqual(d[2][0], 'source')
1430     def testFullVisualization(self):
1431         parser = ModuleAllocCompactFileParser(self.tracerFile, False)
1432         j = jsonVisualizationInfo(parser)
1433         #print(j)
1434         self.assertEqual(len(j["modules"]), 2)
1435         self.assertEqual(len(j["esModules"]), 2)
1436         self.assertEqual(len(j['transitions']), 3)
1437         self.assertEqual(j['transitions'][0]['name'], "Global")
1438         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1439         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1440         self.assertEqual(len(j["transitions"][0]["slots"]), 2)
1441         self.assertEqual(len(j["transitions"][0]["slots"][0]), 11)
1442         self.assertEqual(len(j["transitions"][0]["slots"][1]), 4)
1443         self.assertEqual(len(j["transitions"][1]["slots"]), 2)
1444         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1445         self.assertEqual(len(j["transitions"][1]["slots"][1]), 3)
1446         self.assertEqual(len(j["transitions"][2]["slots"]), 2)
1447         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1448         self.assertEqual(len(j["transitions"][2]["slots"][1]), 2)
1449     def testModuleCentricVisualization(self):
1450         parser = ModuleAllocCompactFileParser(self.tracerFile, True)
1451         j = jsonVisualizationInfo(parser)
1452         #print(j)
1453         self.assertEqual(len(j["modules"]), 2)
1454         self.assertEqual(len(j["esModules"]), 2)
1455         self.assertEqual(len(j['transitions']), 6)
1456         self.assertEqual(j['transitions'][0]['name'], "Global")
1457         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1458         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1459         self.assertEqual(j['transitions'][3]['name'], "source")
1460         self.assertEqual(j['transitions'][4]['name'], "Module")
1461         self.assertEqual(j['transitions'][5]['name'], "ESModule")
1462         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1463         self.assertEqual(len(j["transitions"][0]["slots"][0]), 11)
1464         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1465         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1466         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1467         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1468         self.assertEqual(len(j["transitions"][4]["slots"]), 2)
1469         self.assertEqual(len(j["transitions"][4]["slots"][0]), 7)
1470         self.assertEqual(len(j["transitions"][4]["slots"][1]), 1)
1471         self.assertTrue(j["transitions"][4]["slots"][1][-1]['finish'] != 0.0)
1472         self.assertEqual(len(j["transitions"][5]["slots"]), 1)
1473         self.assertEqual(len(j["transitions"][5]["slots"][0]), 1)
1474
1475 def runTests():
1476     return unittest.main(argv=sys.argv[:1])
1477
1478 #=======================================
1479 if __name__=="__main__":
1480     import argparse
1481     import re
1482     import sys
1483
1484     # Program options
1485     parser = argparse.ArgumentParser(description='Convert a compact tracer file into human readable output.',
1486                                      formatter_class=argparse.RawDescriptionHelpFormatter,
1487                                      epilog=printHelp())
1488     parser.add_argument('filename',
1489                         type=argparse.FileType('r'), # open file
1490                         help='file to process')
1491     parser.add_argument('-j', '--json',
1492                         action='store_true',
1493                         help='''Write output in json format.''' )
1494     parser.add_argument('-s', '--sortBy',
1495                         default = '',
1496                         type = str,
1497                         help="sort modules by attribute. Alloed values 'nAllocs', 'nDeallocs', 'added', 'minTemp', maxTemp', and 'max1Alloc'")
1498 #    parser.add_argument('-w', '--web',
1499 #                        action='store_true',
1500 #                        help='''Writes data.js file that can be used with the web based inspector. To use, copy directory ${CMSSW_RELEASE_BASE}/src/FWCore/Services/template/web to a web accessible area and move data.js into that directory.''')
1501     parser.add_argument('-t', '--timeOrdered',
1502                         action = 'store_true',
1503                         help='''For --json, organize data by time instead of by module.''' )
1504     parser.add_argument('-T', '--test',
1505                         action='store_true',
1506                         help='''Run internal tests.''')
1507
1508     args = parser.parse_args()
1509     if args.test:
1510         runTests()
1511     else :
1512         parser = ModuleAllocCompactFileParser(args.filename, not args.timeOrdered)
1513 #        if args.json or args.web:
1514         if args.json:
1515             json.dump(jsonInfo(parser, args.timeOrdered).toSimpleDict(), sys.stdout, indent=2)
1516 #           if args.web:
1517 #                j ='export const data = ' + j
1518 #                f=open('data.js', 'w')
1519 #                f.write(j)
1520 #                f.close()
1521         elif args.sortBy:
1522             print(json.dumps(sortByAttribute(parser, args.sortBy), indent=2))
1523         else:
1524             textOutput(parser)