Services/scripts/edmTracerCompactLogViewer.py

0001 #!/usr/bin/env python3
0002 from __future__ import print_function
0003 from builtins import range
0004 from itertools import groupby
0005 from operator import attrgetter,itemgetter
0006 import sys
0007 import json
0008 from collections import defaultdict
0009 #----------------------------------------------
0010 def printHelp():
0011     s = '''
0012 To Use: Add the Tracer Service to the cmsRun job use something like this
0013   in the configuration:
0014
0015   process.add_(cms.Service("Tracer", fileName = cms.untracked.string("tracer.log")))
0016
0017   After running the job, execute this script and pass the name of the
0018   Tracer log file to the script.
0019
0020   This script will output a more human readable form of the data in the Tracer log file.'''
0021     return s
0022
0023 #these values come from tracer_setupFile.cc
0024 #enum class Step : char {
0025 #  preSourceTransition = 'S',
0026 #  postSourceTransition = 's',
0027 #  preModulePrefetching = 'P',
0028 #  postModulePrefetching = 'p',
0029 #  preModuleEventAcquire = 'A',
0030 #  postModuleEventAcquire = 'a',
0031 #  preModuleTransition = 'M',
0032 #  preEventReadFromSource = 'R',
0033 #  postEventReadFromSource = 'r',
0034 #  preModuleEventDelayedGet = 'D',
0035 #  postModuleEventDelayedGet = 'd',
0036 #  postModuleTransition = 'm',
0037 #  preESModulePrefetching = 'Q',
0038 #  postESModulePrefetching = 'q',
0039 #  preESModule = 'N',
0040 #  postESModule = 'n',
0041 #  preESModuleAcquire = 'B',
0042 #  postESModuleAcquire = 'b',
0043 #  preFrameworkTransition = 'F',
0044 #  postFrameworkTransition = 'f'
0045 #};
0046
0047
0048 kMicroToSec = 0.000001
0049 #Special names
0050 kSourceFindEvent = "sourceFindEvent"
0051 kSourceDelayedRead ="sourceDelayedRead"
0052 #this value is defined in the framework itself
0053 kLargestLumiNumber = 4294967295
0054
0055 #these values must match the enum class Phase in tracer_setupFile.cc
0056 class Phase (object):
0057   destruction = -16
0058   endJob = -12
0059   endStream = -11
0060   writeProcessBlock = -10
0061   endProcessBlock = -9
0062   globalWriteRun = -7
0063   globalEndRun = -6
0064   streamEndRun = -5
0065   globalWriteLumi = -4
0066   globalEndLumi = -3
0067   streamEndLumi = -2
0068   clearEvent = -1
0069   Event = 0
0070   streamBeginLumi = 2
0071   globalBeginLumi = 3
0072   streamBeginRun = 5
0073   globalBeginRun = 6
0074   accessInputProcessBlock = 8
0075   beginProcessBlock = 9
0076   openFile = 10
0077   beginStream = 11
0078   beginJob  = 12
0079   esSync = 13
0080   esSyncEnqueue = 14
0081   getNextTransition = 15
0082   construction = 16
0083   startTracing = 17
0084
0085 #used for json output
0086 class Activity (object):
0087   prefetch = 0
0088   acquire = 1
0089   process = 2
0090   delayedGet = 3
0091   externalWork = 4
0092   temporary = 100
0093
0094 transitionToNames_ = {
0095     Phase.startTracing: 'start tracing',
0096     Phase.construction: 'construction',
0097     Phase.destruction: 'destruction',
0098     Phase.beginJob: 'begin job',
0099     Phase.endJob: 'end job',
0100     Phase.beginStream: 'begin stream',
0101     Phase.endStream: 'end stream',
0102     Phase.beginProcessBlock: 'begin process block',
0103     Phase.endProcessBlock: 'end process block',
0104     Phase.accessInputProcessBlock: 'access input process block',
0105     Phase.writeProcessBlock: 'write process block',
0106     Phase.globalBeginRun: 'global begin run',
0107     Phase.globalEndRun: 'global end run',
0108     Phase.globalWriteRun: 'global write run',
0109     Phase.streamBeginRun: 'stream begin run',
0110     Phase.streamEndRun: 'stream end run',
0111     Phase.globalBeginLumi: 'global begin lumi',
0112     Phase.globalEndLumi: 'global end lumi',
0113     Phase.globalWriteLumi: 'global write lumi',
0114     Phase.streamBeginLumi: 'stream begin lumi',
0115     Phase.streamEndLumi: 'stream end lumi',
0116     Phase.esSyncEnqueue: 'EventSetup synchronization',
0117     Phase.esSync: 'EventSetup synchronization',
0118     Phase.Event: 'event',
0119     Phase.clearEvent: 'clear event',
0120     Phase.getNextTransition: 'get next transition'
0121 }
0122
0123 def transitionName(transition):
0124     return transitionToNames_[transition]
0125
0126 transitionToIndent_ = {
0127     Phase.startTracing: 0,
0128     Phase.construction: 0,
0129     Phase.destruction: 0,
0130     Phase.endJob: 0,
0131     Phase.beginJob: 0,
0132     Phase.beginStream: 0,
0133     Phase.endStream: 0,
0134     Phase.beginProcessBlock: 1,
0135     Phase.endProcessBlock: 1,
0136     Phase.accessInputProcessBlock: 1,
0137     Phase.writeProcessBlock: 1,
0138     Phase.globalBeginRun: 1,
0139     Phase.globalEndRun: 1,
0140     Phase.globalWriteRun: 1,
0141     Phase.streamBeginRun: 1,
0142     Phase.streamEndRun: 1,
0143     Phase.globalBeginLumi: 2,
0144     Phase.globalEndLumi: 2,
0145     Phase.globalWriteLumi: 2,
0146     Phase.streamBeginLumi: 2,
0147     Phase.streamEndLumi: 2,
0148     Phase.Event: 3,
0149     Phase.clearEvent: 3,
0150     Phase.esSyncEnqueue: 1,
0151     Phase.esSync: 1,
0152     Phase.getNextTransition: 1
0153 }
0154 def transitionIndentLevel(transition):
0155     return transitionToIndent_[transition]
0156
0157 globalTransitions_ = {
0158     Phase.startTracing,
0159     Phase.construction,
0160     Phase.destruction,
0161     Phase.endJob,
0162     Phase.beginJob,
0163     Phase.beginProcessBlock,
0164     Phase.endProcessBlock,
0165     Phase.accessInputProcessBlock,
0166     Phase.writeProcessBlock,
0167     Phase.globalBeginRun,
0168     Phase.globalEndRun,
0169     Phase.globalWriteRun,
0170     Phase.globalBeginLumi,
0171     Phase.globalEndLumi,
0172     Phase.globalWriteLumi,
0173     Phase.esSyncEnqueue,
0174     Phase.esSync,
0175     Phase.getNextTransition
0176 }
0177 def transitionIsGlobal(transition):
0178     return transition in globalTransitions_;
0179
0180 def textPrefix_(time, indentLevel):
0181     #using 11 spaces for time should accomodate a job that runs 24 hrs
0182     return f'{time:>11} '+"++"*indentLevel
0183
0184 class FrameworkTransitionParser (object):
0185     def __init__(self, payload):
0186         self.transition = int(payload[0])
0187         self.index = int(payload[1])
0188         self.sync = (int(payload[2]), int(payload[3]), int(payload[4]))
0189         self.time = int(payload[5])
0190     def indentLevel(self):
0191         return transitionIndentLevel(self.transition)
0192     def textPrefix(self):
0193         return textPrefix_(self.time, self.indentLevel())
0194     def syncText(self):
0195         if self.transition == Phase.globalBeginRun or Phase.globalEndRun == self.transition:
0196             return f'run={self.sync[0]}'
0197         if self.transition == Phase.globalWriteRun:
0198             return f'run={self.sync[0]}'
0199         if self.transition == Phase.streamBeginRun or Phase.streamEndRun == self.transition:
0200             return f'run={self.sync[0]}'
0201         if self.transition == Phase.globalBeginLumi or Phase.globalEndLumi == self.transition:
0202             return f'run={self.sync[0]} lumi={self.sync[1]}'
0203         if self.transition == Phase.globalWriteLumi:
0204             return f'run={self.sync[0]} lumi={self.sync[1]}'
0205         if self.transition == Phase.streamBeginLumi or Phase.streamEndLumi == self.transition:
0206             return f'run={self.sync[0]} lumi={self.sync[1]}'
0207         if self.transition == Phase.Event:
0208             return f'run={self.sync[0]} lumi={self.sync[1]} event={self.sync[2]}'
0209         if self.transition == Phase.esSyncEnqueue or self.transition == Phase.esSync:
0210             return f'run={self.sync[0]} lumi={self.sync[1]}'
0211         if self.transition == Phase.beginJob:
0212             return ''
0213         if self.transition == Phase.beginProcessBlock or self.transition == Phase.endProcessBlock or self.transition == Phase.writeProcessBlock or self.transition == Phase.accessInputProcessBlock:
0214             return ''
0215         if self.transition == Phase.startTracing:
0216             return ''
0217         if self.transition == Phase.construction or self.transition == Phase.destruction:
0218             return ''
0219     def textPostfix(self):
0220         return f'{transitionName(self.transition)} : id={self.index} {self.syncText()}'
0221     def text(self, context):
0222         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0223
0224 def findMatchingTransition(sync, containers):
0225     for i in range(len(containers)):
0226         if containers[i][-1]["sync"] == sync:
0227             return i
0228     #need more exhausting search
0229     for i in range(len(containers)):
0230         for t in containers[i]:
0231             if t["sync"] == sync:
0232                 return i
0233
0234     print("find failed",sync, containers)
0235     return None
0236
0237 def popQueuedTransitions(sync, container):
0238     results = []
0239     for i in range(len(container)):
0240         if sync == container[i]["sync"]:
0241             results.append(container[i])
0242             results.append(container[i+1])
0243             del container[i]
0244             del container[i]
0245             break
0246     return results
0247
0248 transitionsToFindMatch_ = {
0249     Phase.globalEndRun,
0250     Phase.globalEndLumi,
0251     Phase.globalWriteRun,
0252     Phase.globalWriteLumi
0253 }
0254
0255 class PreFrameworkTransitionParser (FrameworkTransitionParser):
0256     def __init__(self, payload):
0257         super().__init__(payload)
0258     def textSpecial(self):
0259         return "starting"
0260     def jsonInfo(self, counter, data):
0261         if transitionIsGlobal(self.transition):
0262             index = 0
0263             if self.transition == Phase.startTracing:
0264                 data.indexedGlobal(0).append(jsonTransition(type=self.transition, id=index, sync=list(self.sync),start=0, finish=self.time ))
0265                 return
0266             elif self.transition == Phase.esSync:
0267                 if self.sync[1] == kLargestLumiNumber:
0268                     #at end run transition
0269                     index = findMatchingTransition(list(self.sync), data.allGlobals())
0270                     container = data.indexedGlobal(index)
0271                     container[-1]["finish"] = self.time*kMicroToSec
0272                 else:
0273                     data._queued[-1]["finish"] = self.time*kMicroToSec
0274                     data._queued.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0275                     return
0276             elif self.transition==Phase.globalBeginRun:
0277                 index = self.index
0278                 #find associated es queued items
0279                 queued = data._queued
0280                 q = popQueuedTransitions(list(self.sync), queued)
0281                 container = data.indexedGlobal(index)
0282                 #find source, should be previous
0283                 last = container[-1]
0284                 if last["type"]==Phase.globalBeginRun and last["isSrc"]:
0285                     last["sync"]=list(self.sync)
0286                 container.append(q[0])
0287                 container.append(q[1])
0288             elif self.transition==Phase.globalBeginLumi:
0289                 index = self.index
0290                 #find associated es queued items
0291                 queued = data._queued
0292                 q = popQueuedTransitions(list(self.sync), queued)
0293                 container = data.indexedGlobal(index)
0294                 #find source, should be previous
0295                 last = container[-1]
0296                 if last["type"]==Phase.globalBeginLumi and last["isSrc"]:
0297                     last["sync"]=list(self.sync)
0298                 container.append(q[0])
0299                 container.append(q[1])
0300             elif self.transition in transitionsToFindMatch_:
0301                 index = findMatchingTransition(list(self.sync), data.allGlobals())
0302             container = data.indexedGlobal(index)
0303         else:
0304             container = data.indexedStream(self.index)
0305             if self.transition == Phase.Event:
0306                 #find source, should be previous
0307                 last = container[-1]
0308                 if last["type"]==Phase.Event and last["isSrc"]:
0309                     last["sync"]=list(self.sync)
0310             index = self.index
0311         container.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0312
0313
0314 class PostFrameworkTransitionParser (FrameworkTransitionParser):
0315     def __init__(self, payload):
0316         super().__init__(payload)
0317     def textSpecial(self):
0318         return "finished"
0319     def jsonInfo(self, counter, data):
0320         if transitionIsGlobal(self.transition):
0321             if self.transition == Phase.esSync and self.sync[1] != kLargestLumiNumber:
0322                 data._queued[-1]['finish']=self.time*kMicroToSec
0323                 return
0324             index = findMatchingTransition(list(self.sync), data.allGlobals())
0325             container = data.indexedGlobal(index)
0326         else:
0327             container = data.indexedStream(self.index)
0328         container[-1]["finish"]=self.time*kMicroToSec
0329
0330
0331 class QueuingFrameworkTransitionParser (FrameworkTransitionParser):
0332     def __init__(self, payload):
0333         super().__init__(payload)
0334     def textSpecial(self):
0335         return "queuing"
0336     def jsonInfo(self, counter, data):
0337         index = -1
0338         if self.sync[1] == kLargestLumiNumber:
0339             #find the mtching open run
0340             index = findMatchingTransition([self.sync[0],0,0], data.allGlobals())
0341             data.indexedGlobal(index).append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0342         else:
0343             data._queued.append(jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0344
0345 class SourceTransitionParser(object):
0346     def __init__(self, payload):
0347         self.transition = int(payload[0])
0348         if self.transition == Phase.getNextTransition:
0349             self.time = int(payload[1])
0350             self.index = -1
0351             return
0352         self.index = int(payload[1])
0353         self.time = int(payload[2])
0354     def indentLevel(self):
0355         if self.transition == Phase.globalBeginRun:
0356             return 1
0357         if self.transition == Phase.globalBeginLumi:
0358             return 2
0359         if self.transition == Phase.Event:
0360             return 3
0361         if self.transition == Phase.construction:
0362             return 1
0363         if self.transition == Phase.getNextTransition:
0364             return 1
0365         return None
0366     def textPrefix(self):
0367         return textPrefix_(self.time, self.indentLevel())
0368     def textPostfix(self):
0369         return f'source during {transitionName(self.transition)} : id={self.index}'
0370     def text(self, context):
0371         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0372
0373 class PreSourceTransitionParser(SourceTransitionParser):
0374     def __init__(self, payload, moduleCentric):
0375         self._moduleCentric = moduleCentric
0376         super().__init__(payload)
0377     def textSpecial(self):
0378         return "starting"
0379     def jsonInfo(self, counter, data):
0380         if self.transition == Phase.getNextTransition:
0381             data._nextTrans.append(jsonTransition(type=self.transition, id=self.index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0382             if self._moduleCentric:
0383                 #this all goes to a module ID sorted container so not knowing actual index is OK
0384                 data.findOpenSlotInModGlobals(0,0).append(data._nextTrans[-1])
0385             return
0386         elif self.transition == Phase.construction:
0387             index = counter.start()
0388             container = data.indexedGlobal(index)
0389         elif self.transition == Phase.Event:
0390             index = self.index
0391             container = data.indexedStream(index)
0392         else:
0393             index = self.index
0394             container = data.indexedGlobal(index)
0395         nextTrans = data._nextTrans
0396         if nextTrans:
0397             data._nextTrans = []
0398             for t in nextTrans:
0399                 t['id']=index
0400                 #find proper time order in the container
0401                 transStartTime = t['start']
0402                 inserted = False
0403                 for i in range(-1, -1*len(container), -1):
0404                     if transStartTime > container[i]['start']:
0405                         if i == -1:
0406                             container.append(t)
0407                             inserted = True
0408                             break
0409                         else:
0410                             container.insert(i+1,t)
0411                             inserted = True
0412                             break
0413                 if not inserted:
0414                     container.insert(0,t)
0415         container.append(jsonTransition(type=self.transition, id=index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0416         if self._moduleCentric:
0417             if self.transition == Phase.Event:
0418                 data.findOpenSlotInModStreams(index,0).append(container[-1])
0419             else:
0420                 data.findOpenSlotInModGlobals(index,0).append(container[-1])
0421
0422 class PostSourceTransitionParser(SourceTransitionParser):
0423     def __init__(self, payload, moduleCentric):
0424         super().__init__(payload)
0425         self._moduleCentric = moduleCentric
0426     def textSpecial(self):
0427         return "finished"
0428     def jsonInfo(self, counter, data):
0429         index = self.index
0430         if self.transition == Phase.Event:
0431             container = data.indexedStream(index)
0432         elif self.transition == Phase.getNextTransition:
0433             data._nextTrans[-1]['finish'] = self.time*kMicroToSec
0434             return
0435         elif self.transition == Phase.construction:
0436             pre = None
0437             for i, g in enumerate(data.allGlobals()):
0438                 for t in reversed(g):
0439                     if t["type"] != Phase.construction:
0440                         break
0441                     if t["isSrc"]:
0442                         pre = t
0443                         break
0444                 if pre:
0445                     pre["finish"]=self.time*kMicroToSec
0446                     break
0447             counter.finish(i)
0448             return
0449         else:
0450             container = data.indexedGlobal(index)
0451
0452         container[-1]["finish"]=self.time*kMicroToSec
0453
0454 class EDModuleTransitionParser(object):
0455     def __init__(self, payload, moduleNames):
0456         self.transition = int(payload[0])
0457         self.index = int(payload[1])
0458         self.moduleID = int(payload[2])
0459         self.moduleName = moduleNames[self.moduleID]
0460         self.callID = int(payload[3])
0461         self.requestingModuleID = int(payload[4])
0462         self.requestingCallID = int(payload[5])
0463         self.requestingModuleName = None
0464         if self.requestingModuleID != 0:
0465             self.requestingModuleName = moduleNames[self.requestingModuleID]
0466         self.time = int(payload[6])
0467     def baseIndentLevel(self):
0468         return transitionIndentLevel(self.transition)
0469     def textPrefix(self, context):
0470         indent = 0
0471         if self.requestingModuleID != 0:
0472             indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
0473         context[(self.transition, self.index, self.moduleID, self.callID)] = indent+1
0474         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0475     def textPostfix(self):
0476         return f'{self.moduleName} during {transitionName(self.transition)} : id={self.index}'
0477     def textIfTransform(self):
0478         if self.callID:
0479             return f' transform {self.callID-1}'
0480         return ''
0481     def text(self, context):
0482         return f'{self.textPrefix(context)} {self.textSpecial()}{self.textIfTransform()}: {self.textPostfix()}'
0483     def _preJson(self, activity, counter, data, mayUseTemp = False):
0484         index = self.index
0485         found = False
0486         if mayUseTemp:
0487             compare = lambda x: x['type'] == self.transition and x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and (x['act'] == Activity.temporary or x['act'] == Activity.externalWork)
0488             if transitionIsGlobal(self.transition):
0489                 item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0490             else:
0491                 item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0492             if slot:
0493                 if item['act'] == Activity.temporary:
0494                     slot.pop()
0495                 else:
0496                     item['finish']=self.time*kMicroToSec
0497                 found = True
0498         if not found:
0499             if transitionIsGlobal(self.transition):
0500                 slot = data.findOpenSlotInModGlobals(index, self.moduleID)
0501             else:
0502                 slot = data.findOpenSlotInModStreams(index, self.moduleID)
0503         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=activity, start=self.time))
0504         return slot[-1]
0505     def _postJson(self, counter, data, injectAfter = None):
0506         compare = lambda x: x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and x['type'] == self.transition
0507         index = self.index
0508         if transitionIsGlobal(self.transition):
0509             item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0510         else:
0511             item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0512         if item is None:
0513             print(f"failed to find {self.moduleID} for {self.transition} in {self.index}")
0514         else:
0515             item["finish"]=self.time*kMicroToSec
0516             if injectAfter:
0517                 slot.append(injectAfter)
0518
0519 class PreEDModuleTransitionParser(EDModuleTransitionParser):
0520     def __init__(self, payload, names, moduleCentric):
0521         super().__init__(payload, names)
0522         self._moduleCentric = moduleCentric
0523     def textSpecial(self):
0524         return "starting action"
0525     def jsonInfo(self, counter, data):
0526         return self._preJson(Activity.process, counter,data, mayUseTemp=self._moduleCentric)
0527
0528 class PostEDModuleTransitionParser(EDModuleTransitionParser):
0529     def __init__(self, payload, names):
0530         super().__init__(payload, names)
0531     def textSpecial(self):
0532         return "finished action"
0533     def jsonInfo(self, counter, data):
0534         return self._postJson(counter,data)
0535
0536 class PreEDModulePrefetchingParser(EDModuleTransitionParser):
0537     def __init__(self, payload, names, moduleCentric):
0538         super().__init__(payload, names)
0539         self._moduleCentric = moduleCentric
0540     def textSpecial(self):
0541         return "starting prefetch"
0542     def jsonInfo(self, counter, data):
0543         #the total time in prefetching isn't useful, but seeing the start is
0544         entry = self._preJson(Activity.prefetch, counter,data)
0545         if self._moduleCentric:
0546             return entry
0547         kPrefetchLength = 2*kMicroToSec
0548         entry["finish"]=entry["start"]+kPrefetchLength
0549         return entry
0550
0551
0552 class PostEDModulePrefetchingParser(EDModuleTransitionParser):
0553     def __init__(self, payload, names, moduleCentric):
0554         super().__init__(payload, names)
0555         self._moduleCentric = moduleCentric
0556     def textSpecial(self):
0557         return "finished prefetch"
0558     def jsonInfo(self, counter, data):
0559         if self._moduleCentric:
0560             #inject a dummy at end of the same slot to guarantee module run is in that slot
0561             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.temporary, start=self.time))
0562         pass
0563
0564 class PreEDModuleAcquireParser(EDModuleTransitionParser):
0565     def __init__(self, payload, names, moduleCentric):
0566         super().__init__(payload, names)
0567         self._moduleCentric = moduleCentric
0568     def textSpecial(self):
0569         return "starting acquire"
0570     def jsonInfo(self, counter, data):
0571         return self._preJson(Activity.acquire, counter,data, mayUseTemp=self._moduleCentric)
0572
0573 class PostEDModuleAcquireParser(EDModuleTransitionParser):
0574     def __init__(self, payload, names, moduleCentric):
0575         super().__init__(payload, names)
0576         self._moduleCentric = moduleCentric
0577     def textSpecial(self):
0578         return "finished acquire"
0579     def jsonInfo(self, counter, data):
0580         if self._moduleCentric:
0581             #inject an external work at end of the same slot to guarantee module run is in that slot
0582             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.externalWork, start=self.time))
0583         return self._postJson(counter,data)
0584
0585 class PreEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0586     def __init__(self, payload, names):
0587         super().__init__(payload, names)
0588     def textSpecial(self):
0589         return "starting delayed get"
0590     def jsonInfo(self, counter, data):
0591         return self._preJson(Activity.delayedGet, counter,data)
0592
0593 class PostEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0594     def __init__(self, payload, names):
0595         super().__init__(payload, names)
0596     def textSpecial(self):
0597         return "finished delayed get"
0598     def jsonInfo(self, counter, data):
0599         return self._postJson(counter,data)
0600
0601 class PreEventReadFromSourceParser(EDModuleTransitionParser):
0602     def __init__(self, payload, names):
0603         super().__init__(payload, names)
0604     def textSpecial(self):
0605         return "starting read from source"
0606     def jsonInfo(self, counter, data):
0607         slot = self._preJson(Activity.process, counter,data)
0608         slot['isSrc'] = True
0609         return slot
0610
0611 class PostEventReadFromSourceParser(EDModuleTransitionParser):
0612     def __init__(self, payload, names):
0613         super().__init__(payload, names)
0614     def textSpecial(self):
0615         return "finished read from source"
0616     def jsonInfo(self, counter, data):
0617         return self._postJson(counter,data)
0618
0619 class ESModuleTransitionParser(object):
0620     def __init__(self, payload, moduleNames, esModuleNames, recordNames):
0621         self.transition = int(payload[0])
0622         self.index = int(payload[1])
0623         self.moduleID = int(payload[2])
0624         self.moduleName = esModuleNames[self.moduleID]
0625         self.recordID = int(payload[3])
0626         self.recordName = recordNames[self.recordID]
0627         self.callID = int(payload[4])
0628         self.requestingModuleID = int(payload[5])
0629         self.requestingCallID = int(payload[6])
0630         self.requestingModuleName = None
0631         if self.requestingModuleID < 0 :
0632             self.requestingModuleName = esModuleNames[-1*self.requestingModuleID]
0633         else:
0634             self.requestingModuleName = moduleNames[self.requestingModuleID]
0635         self.time = int(payload[7])
0636     def baseIndentLevel(self):
0637         return transitionIndentLevel(self.transition)
0638     def textPrefix(self, context):
0639         indent = 0
0640         indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
0641         context[(self.transition, self.index, -1*self.moduleID, self.callID)] = indent+1
0642         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0643     def textPostfix(self):
0644         return f'esmodule {self.moduleName} in record {self.recordName} during {transitionName(self.transition)} : id={self.index}'
0645     def text(self, context):
0646         return f'{self.textPrefix(context)} {self.textSpecial()}: {self.textPostfix()}'
0647     def _preJson(self, activity, counter, data):
0648         index = self.index
0649         if transitionIsGlobal(self.transition):
0650             slot = data.findOpenSlotInModGlobals(index, -1*self.moduleID)
0651         else:
0652             slot = data.findOpenSlotInModStreams(index, -1*self.moduleID)
0653         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=-1*self.moduleID, callID=self.callID, activity=activity, start=self.time))
0654         return slot[-1]
0655     def _postJson(self, counter, data):
0656         compare = lambda x: x['id'] == self.index and x['mod'] == -1*self.moduleID and x['call'] == self.callID
0657         index = self.index
0658         if transitionIsGlobal(self.transition):
0659             item,s = data.findLastInModGlobals(index, -1*self.moduleID, compare)
0660         else:
0661             item,s = data.findLastInModStreams(index, -1*self.moduleID, compare)
0662         if item is None:
0663             print(f"failed to find {-1*self.moduleID} for {self.transition} in {self.index}")
0664             return
0665         item["finish"]=self.time*kMicroToSec
0666
0667
0668 class PreESModuleTransitionParser(ESModuleTransitionParser):
0669     def __init__(self, payload, names, esNames, recordNames):
0670         super().__init__(payload, names, esNames, recordNames)
0671     def textSpecial(self):
0672         return "starting action"
0673     def jsonInfo(self, counter, data):
0674         return self._preJson(Activity.process, counter,data)
0675
0676 class PostESModuleTransitionParser(ESModuleTransitionParser):
0677     def __init__(self, payload, names, esNames, recordNames):
0678         super().__init__(payload, names, esNames, recordNames)
0679     def textSpecial(self):
0680         return "finished action"
0681     def jsonInfo(self, counter, data):
0682         return self._postJson(counter,data)
0683
0684 class PreESModulePrefetchingParser(ESModuleTransitionParser):
0685     def __init__(self, payload, names, esNames, recordNames, moduleCentric):
0686         super().__init__(payload, names, esNames, recordNames)
0687         self._moduleCentric = moduleCentric
0688     def textSpecial(self):
0689         return "starting prefetch"
0690     def jsonInfo(self, counter, data):
0691         entry = self._preJson(Activity.prefetch, counter,data)
0692         if not self._moduleCentric:
0693             entry["finish"] = entry["start"]+2*kMicroToSec;
0694         return entry
0695
0696 class PostESModulePrefetchingParser(ESModuleTransitionParser):
0697     def __init__(self, payload, names, esNames, recordNames, moduleCentric):
0698         super().__init__(payload, names, esNames, recordNames)
0699         self._moduleCentric = moduleCentric
0700     def textSpecial(self):
0701         return "finished prefetch"
0702     def jsonInfo(self, counter, data):
0703         if self._moduleCentric:
0704             return self._postJson(counter, data)
0705         pass
0706
0707 class PreESModuleAcquireParser(ESModuleTransitionParser):
0708     def __init__(self, payload, names, recordNames):
0709         super().__init__(payload, names, recordNames)
0710     def textSpecial(self):
0711         return "starting acquire"
0712     def jsonInfo(self, counter, data):
0713         return self._preJson(Activity.acquire, counter,data)
0714
0715 class PostESModuleAcquireParser(ESModuleTransitionParser):
0716     def __init__(self, payload, names, esNames, recordNames):
0717         super().__init__(payload, names, esNames, recordNames)
0718     def textSpecial(self):
0719         return "finished acquire"
0720     def jsonInfo(self, counter, data):
0721         return self._postJson(counter,data)
0722
0723
0724 def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
0725     if step == 'F':
0726         parser = PreFrameworkTransitionParser(payload)
0727         if parser.transition == Phase.esSyncEnqueue:
0728             return QueuingFrameworkTransitionParser(payload)
0729         return parser
0730     if step == 'f':
0731         return PostFrameworkTransitionParser(payload)
0732     if step == 'S':
0733         return PreSourceTransitionParser(payload, moduleCentric)
0734     if step == 's':
0735         return PostSourceTransitionParser(payload, moduleCentric)
0736     if frameworkOnly:
0737         return None
0738     if step == 'M':
0739         return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
0740     if step == 'm':
0741         return PostEDModuleTransitionParser(payload, moduleNames)
0742     if step == 'P':
0743         return PreEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
0744     if step == 'p':
0745         return PostEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
0746     if step == 'A':
0747         return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0748     if step == 'a':
0749         return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0750     if step == 'D':
0751         return PreEDModuleEventDelayedGetParser(payload, moduleNames)
0752     if step == 'd':
0753         return PostEDModuleEventDelayedGetParser(payload, moduleNames)
0754     if step == 'R':
0755         return PreEventReadFromSourceParser(payload, moduleNames)
0756     if step == 'r':
0757         return PostEventReadFromSourceParser(payload, moduleNames)
0758     if step == 'N':
0759         return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0760     if step == 'n':
0761         return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0762     if step == 'Q':
0763         return PreESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0764     if step == 'q':
0765         return PostESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0766     if step == 'B':
0767         return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0768     if step == 'b':
0769         return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0770
0771
0772 #----------------------------------------------
0773 def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
0774     for rawl in f:
0775         l = rawl.strip()
0776         if not l or l[0] == '#':
0777             continue
0778         (step,payload) = tuple(l.split(None,1))
0779         payload=payload.split()
0780
0781         parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
0782         if parser:
0783             yield parser
0784     return
0785
0786 class TracerCompactFileParser(object):
0787     def __init__(self,f, frameworkOnly, moduleCentric):
0788         streamBeginRun = str(Phase.streamBeginRun)
0789         numStreams = 0
0790         numStreamsFromSource = 0
0791         moduleNames = {}
0792         esModuleNames = {}
0793         recordNames = {}
0794         for rawl in f:
0795             l = rawl.strip()
0796             if l and l[0] == 'M':
0797                 i = l.split(' ')
0798                 if i[3] == streamBeginRun:
0799                     #found global begin run
0800                     numStreams = int(i[1])+1
0801                     break
0802             if numStreams == 0 and l and l[0] == 'S':
0803                 s = int(l.split(' ')[1])
0804                 if s > numStreamsFromSource:
0805                   numStreamsFromSource = s
0806             if len(l) > 5 and l[0:2] == "#M":
0807                 (id,name)=tuple(l[2:].split())
0808                 moduleNames[int(id)] = name
0809                 continue
0810             if len(l) > 5 and l[0:2] == "#N":
0811                 (id,name)=tuple(l[2:].split())
0812                 esModuleNames[int(id)] = name
0813                 continue
0814             if len(l) > 5 and l[0:2] == "#R":
0815                 (id,name)=tuple(l[2:].split())
0816                 recordNames[int(id)] = name
0817                 continue
0818
0819         self._f = f
0820         self._frameworkOnly = frameworkOnly
0821         self._moduleCentric = moduleCentric
0822         if numStreams == 0:
0823           numStreams = numStreamsFromSource +2
0824         self.numStreams =numStreams
0825         self._moduleNames = moduleNames
0826         self._esModuleNames = esModuleNames
0827         self._recordNames = recordNames
0828         self.maxNameSize =0
0829         for n in moduleNames.items():
0830             self.maxNameSize = max(self.maxNameSize,len(n))
0831         for n in esModuleNames.items():
0832             self.maxNameSize = max(self.maxNameSize,len(n))
0833         self.maxNameSize = max(self.maxNameSize,len(kSourceDelayedRead))
0834         self.maxNameSize = max(self.maxNameSize, len('streamBeginLumi'))
0835
0836     def processingSteps(self):
0837         """Create a generator which can step through the file and return each processing step.
0838         Using a generator reduces the memory overhead when parsing a large file.
0839             """
0840         self._f.seek(0)
0841         return processingStepsFromFile(self._f,self._moduleNames, self._esModuleNames, self._recordNames, self._frameworkOnly, self._moduleCentric)
0842
0843 def textOutput( parser ):
0844     context = {}
0845     for p in parser.processingSteps():
0846         print(p.text(context))
0847
0848 class Counter(object):
0849     def __init__(self):
0850         self.activeSlots = [False]
0851     def start(self):
0852         if 0 != self.activeSlots.count(False):
0853             index = self.activeSlots.index(False)
0854             self.activeSlots[index]=True
0855             return index
0856         index = len(self.activeSlots)
0857         self.activeSlots.append(True)
0858         return  index
0859     def finish(self, index):
0860         self.activeSlots[index] = False
0861
0862 class Containers(object):
0863     def __init__(self):
0864         self._modGlobals = [[]]
0865         self._modStreams = [[]]
0866         self._globals = [[]]
0867         self._streams = [[]]
0868         self._queued = []
0869         self._nextTrans = []
0870     def _extendIfNeeded(self, container, index):
0871         while len(container) < index+1:
0872             container.append([])
0873     def allGlobals(self):
0874         return self._globals
0875     def indexedGlobal(self, index):
0876         self._extendIfNeeded(self._globals, index)
0877         return self._globals[index]
0878     def allStreams(self):
0879         return self._streams
0880     def indexedStream(self, index):
0881         self._extendIfNeeded(self._streams, index)
0882         return self._streams[index]
0883     def _findOpenSlot(self, index, fullContainer):
0884         self._extendIfNeeded(fullContainer, index)
0885         container = fullContainer[index]
0886         #find open slot
0887         foundOpenSlot = False
0888         for slot in container:
0889             if len(slot) == 0:
0890                 foundOpenSlot = True
0891                 break
0892             if slot[-1]["finish"] != 0:
0893                 foundOpenSlot = True
0894                 break
0895         if not foundOpenSlot:
0896             container.append([])
0897             slot = container[-1]
0898         return slot
0899     def findOpenSlotInModGlobals(self, index, modID):
0900         return self._findOpenSlot(index, self._modGlobals)
0901     def findOpenSlotInModStreams(self, index, modID):
0902         return self._findOpenSlot(index, self._modStreams)
0903     def _findLastIn(self, index, fullContainer, comparer):
0904         container = fullContainer[index]
0905         #find slot containing the pre
0906         for slot in container:
0907             if comparer(slot[-1]):
0908                 return (slot[-1],slot)
0909         return (None,None)
0910     def findLastInModGlobals(self, index, modID, comparer):
0911         return self._findLastIn(index, self._modGlobals, comparer)
0912     def findLastInModStreams(self, index, modID, comparer):
0913         return self._findLastIn(index, self._modStreams, comparer)
0914
0915
0916 class ModuleCentricContainers(object):
0917     def __init__(self):
0918         self._modules= []
0919         self._globals = [[]]
0920         self._streams = [[]]
0921         self._queued = []
0922         self._nextTrans = []
0923         self._moduleIDOffset = 0
0924     def _moduleID2Index(self, modID):
0925         return modID + self._moduleIDOffset
0926     def _extendIfNeeded(self, container, index):
0927         while len(container) < index+1:
0928             container.append([])
0929     def _extendModulesIfNeeded(self, container, index):
0930         while index + self._moduleIDOffset < 0:
0931             container.insert(0,[])
0932             self._moduleIDOffset +=1
0933         self._extendIfNeeded(container, self._moduleID2Index(index))
0934     def allGlobals(self):
0935         return self._globals
0936     def indexedGlobal(self, index):
0937         self._extendIfNeeded(self._globals, index)
0938         return self._globals[index]
0939     def allStreams(self):
0940         return self._streams
0941     def indexedStream(self, index):
0942         self._extendIfNeeded(self._streams, index)
0943         return self._streams[index]
0944     def _findOpenSlot(self, index, fullContainer):
0945         self._extendModulesIfNeeded(fullContainer, index)
0946         container = fullContainer[self._moduleID2Index(index)]
0947         #find open slot
0948         foundOpenSlot = False
0949         for slot in container:
0950             if len(slot) == 0:
0951                 foundOpenSlot = True
0952                 break
0953             if slot[-1]["finish"] != 0:
0954                 foundOpenSlot = True
0955                 break
0956         if not foundOpenSlot:
0957             container.append([])
0958             slot = container[-1]
0959         return slot
0960     def findOpenSlotInModGlobals(self, index, modID):
0961         return self._findOpenSlot(modID, self._modules)
0962     def findOpenSlotInModStreams(self, index, modID):
0963         return self._findOpenSlot(modID, self._modules)
0964     def _findLastIn(self, index, fullContainer, comparer):
0965         if not fullContainer:
0966             return (None, None)
0967         if len(fullContainer) > self._moduleID2Index(index):
0968             container = fullContainer[self._moduleID2Index(index)]
0969         else:
0970             return (None, None)
0971         #find slot containing the pre
0972         for slot in container:
0973             if slot is not None and comparer(slot[-1]):
0974                 return (slot[-1],slot)
0975         return (None, None)
0976     def findLastInModGlobals(self, index, modID, comparer):
0977         return self._findLastIn(modID, self._modules, comparer)
0978     def findLastInModStreams(self, index, modID, comparer):
0979         return self._findLastIn(modID, self._modules, comparer)
0980
0981
0982
0983 def jsonTransition(type, id, sync, start, finish, isSrc=False):
0984     return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}
0985
0986 def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
0987     return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}
0988
0989 def startTime(x):
0990     return x["start"]
0991 def jsonInfo(parser):
0992     counter = Counter()
0993     if parser._moduleCentric:
0994         data = ModuleCentricContainers()
0995     else:
0996         data = Containers()
0997     for p in parser.processingSteps():
0998         p.jsonInfo(counter, data)
0999     #make sure everything is sorted
1000     for g in data.allGlobals():
1001         g.sort(key=startTime)
1002     final = {"transitions" : [] , "modules": [], "esModules": []}
1003     final["transitions"].append({ "name":"Global", "slots": []})
1004     globals = final["transitions"][-1]["slots"]
1005     for i, g in enumerate(data.allGlobals()):
1006         globals.append(g)
1007         if not parser._moduleCentric and not parser._frameworkOnly:
1008             if len(data._modGlobals) < i+1:
1009                 break
1010             for mod in data._modGlobals[i]:
1011                 globals.append(mod)
1012     for i,s in enumerate(data.allStreams()):
1013         final["transitions"].append({"name": f"Stream {i}", "slots":[]})
1014         stream = final["transitions"][-1]["slots"]
1015         stream.append(s)
1016         if not parser._moduleCentric and not parser._frameworkOnly:
1017             for mod in data._modStreams[i]:
1018                 stream.append(mod)
1019     if parser._moduleCentric:
1020         sourceSlot = data._modules[data._moduleID2Index(0)]
1021         modules = []
1022         for i,m in parser._moduleNames.items():
1023             modules.append({"name": f"{m}", "slots":[]})
1024             slots = modules[-1]["slots"]
1025             foundSlots = data._modules[data._moduleID2Index(i)]
1026             time = 0
1027             for s in foundSlots:
1028                 slots.append(s)
1029                 for t in s:
1030                     if t["act"] !=Activity.prefetch:
1031                         time += t["finish"]-t["start"]
1032             modules[-1]['time']=time
1033         for i,m in parser._esModuleNames.items():
1034             modules.append({"name": f"{m}", "slots":[]})
1035             slots = modules[-1]["slots"]
1036             foundSlots = data._modules[data._moduleID2Index(-1*i)]
1037             time = 0
1038             for s in foundSlots:
1039                 slots.append(s)
1040                 for t in s:
1041                     if t["act"] !=Activity.prefetch:
1042                         time += t["finish"]-t["start"]
1043             modules[-1]['time']=time
1044         modules.sort(key= lambda x : x['time'], reverse=True)
1045         final['transitions'].append({"name": "source", "slots":sourceSlot})
1046         for m in modules:
1047             final['transitions'].append(m)
1048
1049     if not parser._frameworkOnly:
1050         max = 0
1051         for k in parser._moduleNames.keys():
1052             if k > max:
1053                 max = k
1054
1055         final["modules"] =['']*(max+1)
1056         final["modules"][0] = 'source'
1057         for k,v in parser._moduleNames.items():
1058             final["modules"][k]=v
1059
1060         max = 0
1061         for k in parser._esModuleNames.keys():
1062             if k > max:
1063                 max = k
1064         final["esModules"] = ['']*(max+1)
1065         for k,v in parser._esModuleNames.items():
1066             final["esModules"][k] = v
1067     return final
1068
1069 #=======================================
1070 import unittest
1071
1072 class DummyFile(list):
1073     def __init__(self):
1074         super()
1075     def seek(self, i):
1076         pass
1077
1078 class TestModuleCommand(unittest.TestCase):
1079     def setUp(self):
1080         self.tracerFile = DummyFile()
1081         t = [0]
1082         def incr(t):
1083             t[0] += 1
1084             return t[0]
1085
1086         self.tracerFile.extend([
1087             '#R 1 Record',
1088             '#M 1 Module',
1089             '#N 1 ESModule',
1090              f'F {Phase.startTracing} 0 0 0 0 {incr(t)}',
1091              f'S {Phase.construction} 0 {incr(t)}',
1092              f's {Phase.construction} 0 {incr(t)}3',
1093              f'M {Phase.construction} 0 1 0 0 0 {incr(t)}',
1094              f'm {Phase.construction} 0 1 0 0 0 {incr(t)}',
1095              f'F {Phase.beginJob} 0 0 0 0 {incr(t)}',
1096              f'M {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
1097              f'm {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
1098              f'f {Phase.beginJob} 0 0 0 0 {incr(t)}',
1099              f'F {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1100              f'f {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1101              f'S {Phase.getNextTransition} {incr(t)}',
1102              f's {Phase.getNextTransition} {incr(t)}',
1103              f'F {Phase.esSyncEnqueue} -1 1 0 0 {incr(t)}',
1104              f'F {Phase.esSync} -1 1 0 0 {incr(t)}',
1105              f'f {Phase.esSync} -1 1 0 0 {incr(t)}',
1106              f'S {Phase.globalBeginRun} 0 {incr(t)}',
1107              f's {Phase.globalBeginRun} 0 {incr(t)}',
1108              f'S {Phase.getNextTransition} {incr(t)}',
1109              f's {Phase.getNextTransition} {incr(t)}',
1110              f'F {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1111              f'P {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1112              f'p {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1113              f'M {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1114              f'm {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1115              f'f {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1116              f'F {Phase.esSyncEnqueue} -1 1 1 0 {incr(t)}',
1117              f'F {Phase.esSync} -1 1 1 0 {incr(t)}',
1118              f'f {Phase.esSync} -1 1 1 0 {incr(t)}',
1119              f'S {Phase.getNextTransition} {incr(t)}',
1120              f's {Phase.getNextTransition} {incr(t)}',
1121              f'F {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1122              f'M {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
1123              f'm {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
1124              f'f {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1125              f'F {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1126              f'M {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
1127              f'm {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
1128              f'f {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1129              f'S {Phase.globalBeginLumi} 0 {incr(t)}',
1130              f's {Phase.globalBeginLumi} 0 {incr(t)}',
1131              f'S {Phase.getNextTransition} {incr(t)}',
1132              f's {Phase.getNextTransition} {incr(t)}',
1133              f'F {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1134              f'P {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1135              f'p {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1136              f'M {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1137              f'm {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1138              f'f {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1139              f'F {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1140              f'f {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1141              f'F {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1142              f'f {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1143              f'S {Phase.Event} 0 {incr(t)}',
1144              f's {Phase.Event} 0 {incr(t)}',
1145              f'S {Phase.getNextTransition} {incr(t)}',
1146              f's {Phase.getNextTransition} {incr(t)}',
1147              f'F {Phase.Event} 0 1 1 1 {incr(t)}',
1148              f'S {Phase.Event} 1 {incr(t)}',
1149              f's {Phase.Event} 1 {incr(t)}',
1150              f'F {Phase.Event} 1 1 1 2 {incr(t)}',
1151              f'P {Phase.Event} 0 1 0 0 0 {incr(t)}',
1152              f'p {Phase.Event} 0 1 0 0 0 {incr(t)}',
1153              f'Q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1154              f'q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1155              f'N {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1156              f'n {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1157              f'P {Phase.Event} 1 1 0 0 0 {incr(t)}',
1158              f'p {Phase.Event} 1 1 0 0 0 {incr(t)}',
1159              f'M {Phase.Event} 0 1 0 0 0 {incr(t)}',
1160              f'M {Phase.Event} 1 1 0 0 0 {incr(t)}',
1161              f'm {Phase.Event} 1 1 0 0 0 {incr(t)}',
1162              f'm {Phase.Event} 0 1 0 0 0 {incr(t)}',
1163              f'f {Phase.Event} 0 1 1 1 {incr(t)}',
1164              f'f {Phase.Event} 1 1 1 2 {incr(t)}'])
1165
1166         None
1167     def testContainers(self):
1168         c = Containers()
1169         c.indexedGlobal(2)
1170         self.assertEqual(len(c.allGlobals()), 3)
1171         c.indexedStream(2)
1172         self.assertEqual(len(c.allStreams()), 3)
1173         slot = c.findOpenSlotInModGlobals(2, 1)
1174         self.assertEqual(len(c._modGlobals),3)
1175         self.assertEqual(len(slot),0)
1176         slot.append({"start":1, "finish":0, "id":1})
1177         def testFind(item):
1178             return item["id"]==1
1179         item,s = c.findLastInModGlobals(2, 1, testFind)
1180         self.assertEqual(item["id"],1)
1181         self.assertEqual(slot,s)
1182         slot = c.findOpenSlotInModStreams(2, 1)
1183         self.assertEqual(len(c._modStreams),3)
1184         self.assertEqual(len(slot),0)
1185         slot.append({"start":1, "finish":0, "id":1})
1186         item,s = c.findLastInModStreams(2, 1, testFind)
1187         self.assertEqual(item["id"],1)
1188         self.assertEqual(slot,s)
1189     def testFrameworkOnly(self):
1190         parser = TracerCompactFileParser(self.tracerFile, True, False)
1191         j = jsonInfo(parser)
1192         #print(j)
1193         self.assertEqual(len(j["modules"]), 0)
1194         self.assertEqual(len(j["esModules"]), 0)
1195         self.assertEqual(len(j['transitions']), 3)
1196         self.assertEqual(j['transitions'][0]['name'], "Global")
1197         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1198         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1199         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1200         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1201         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1202         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1203         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1204         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1205     def testFull(self):
1206         parser = TracerCompactFileParser(self.tracerFile, False, False)
1207         j = jsonInfo(parser)
1208         #print(j)
1209         self.assertEqual(len(j["modules"]), 2)
1210         self.assertEqual(len(j["esModules"]), 2)
1211         self.assertEqual(len(j['transitions']), 3)
1212         self.assertEqual(j['transitions'][0]['name'], "Global")
1213         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1214         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1215         self.assertEqual(len(j["transitions"][0]["slots"]), 2)
1216         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1217         self.assertEqual(len(j["transitions"][0]["slots"][1]), 6)
1218         self.assertEqual(len(j["transitions"][1]["slots"]), 2)
1219         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1220         self.assertEqual(len(j["transitions"][1]["slots"][1]), 5)
1221         self.assertEqual(len(j["transitions"][2]["slots"]), 2)
1222         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1223         self.assertEqual(len(j["transitions"][2]["slots"][1]), 3)
1224     def testModuleCentric(self):
1225         parser = TracerCompactFileParser(self.tracerFile, False, True)
1226         j = jsonInfo(parser)
1227         #print(j)
1228         self.assertEqual(len(j["modules"]), 2)
1229         self.assertEqual(len(j["esModules"]), 2)
1230         self.assertEqual(len(j['transitions']), 6)
1231         self.assertEqual(j['transitions'][0]['name'], "Global")
1232         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1233         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1234         self.assertEqual(j['transitions'][3]['name'], "source")
1235         self.assertEqual(j['transitions'][4]['name'], "Module")
1236         self.assertEqual(j['transitions'][5]['name'], "ESModule")
1237         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1238         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1239         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1240         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1241         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1242         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1243         self.assertEqual(len(j["transitions"][4]["slots"]), 2)
1244         self.assertEqual(len(j["transitions"][4]["slots"][0]), 10)
1245         self.assertEqual(len(j["transitions"][4]["slots"][1]), 2)
1246         self.assertTrue(j["transitions"][4]["slots"][1][-1]['finish'] != 0.0)
1247         self.assertEqual(len(j["transitions"][5]["slots"]), 1)
1248         self.assertEqual(len(j["transitions"][5]["slots"][0]), 2)
1249
1250 def runTests():
1251     return unittest.main(argv=sys.argv[:1])
1252
1253 #=======================================
1254 if __name__=="__main__":
1255     import argparse
1256     import re
1257     import sys
1258
1259     # Program options
1260     parser = argparse.ArgumentParser(description='Convert a compact tracer file into human readable output.',
1261                                      formatter_class=argparse.RawDescriptionHelpFormatter,
1262                                      epilog=printHelp())
1263     parser.add_argument('filename',
1264                         type=argparse.FileType('r'), # open file
1265                         help='file to process')
1266     parser.add_argument('-f', '--frameworkOnly',
1267                         action='store_true',
1268                         help='''Output only the framework transitions, excluding the individual module transitions.''')
1269     parser.add_argument('-j', '--json',
1270                         action='store_true',
1271                         help='''Write output in json format.''' )
1272     parser.add_argument('-w', '--web',
1273                         action='store_true',
1274                         help='''Writes data.js file that can be used with the web based inspector. To use, copy directory ${CMSSW_RELEASE_BASE}/src/FWCore/Services/template/web to a web accessible area and move data.js into that directory.''')
1275     parser.add_argument('-m', '--module_centric',
1276                         action = 'store_true',
1277                         help='''For --json or --web, organize data by module instead of by global/stream.''' )
1278     parser.add_argument('-t', '--test',
1279                         action='store_true',
1280                         help='''Run internal tests.''')
1281
1282     args = parser.parse_args()
1283     if args.test:
1284         runTests()
1285     else :
1286         parser = TracerCompactFileParser(args.filename, args.frameworkOnly, args.module_centric)
1287         if args.json or args.web:
1288             j = json.dumps(jsonInfo(parser))
1289             if args.json:
1290                 print(j)
1291             if args.web:
1292                 f=open('data.json', 'w')
1293                 f.write(j)
1294                 f.close()
1295         else:
1296             textOutput(parser)