Services/scripts/edmTracerCompactLogViewer.py

0001 #!/usr/bin/env python3
0002 from builtins import range
0003 from itertools import groupby
0004 from operator import attrgetter,itemgetter
0005 import sys
0006 import json
0007 from collections import defaultdict
0008 #----------------------------------------------
0009 def printHelp():
0010     s = '''
0011 To Use: Add the Tracer Service to the cmsRun job use something like this
0012   in the configuration:
0013
0014   process.add_(cms.Service("Tracer", fileName = cms.untracked.string("tracer.log")))
0015
0016   After running the job, execute this script and pass the name of the
0017   Tracer log file to the script.
0018
0019   This script will output a more human readable form of the data in the Tracer log file.'''
0020     return s
0021
0022 #these values come from tracer_setupFile.cc
0023 #enum class Step : char {
0024 #  preSourceTransition = 'S',
0025 #  postSourceTransition = 's',
0026 #  preModulePrefetching = 'P',
0027 #  postModulePrefetching = 'p',
0028 #  preModuleEventAcquire = 'A',
0029 #  postModuleEventAcquire = 'a',
0030 #  preModuleTransition = 'M',
0031 #  preEventReadFromSource = 'R',
0032 #  postEventReadFromSource = 'r',
0033 #  preModuleEventDelayedGet = 'D',
0034 #  postModuleEventDelayedGet = 'd',
0035 #  postModuleTransition = 'm',
0036 #  preESModulePrefetching = 'Q',
0037 #  postESModulePrefetching = 'q',
0038 #  preESModule = 'N',
0039 #  postESModule = 'n',
0040 #  preESModuleAcquire = 'B',
0041 #  postESModuleAcquire = 'b',
0042 #  preFrameworkTransition = 'F',
0043 #  postFrameworkTransition = 'f'
0044 #};
0045
0046
0047 kMicroToSec = 0.000001
0048 #Special names
0049 kSourceFindEvent = "sourceFindEvent"
0050 kSourceDelayedRead ="sourceDelayedRead"
0051 #this value is defined in the framework itself
0052 kLargestLumiNumber = 4294967295
0053
0054 #these values must match the enum class Phase in tracer_setupFile.cc
0055 class Phase (object):
0056   destruction = -16
0057   endJob = -12
0058   endStream = -11
0059   writeProcessBlock = -10
0060   endProcessBlock = -9
0061   globalWriteRun = -7
0062   globalEndRun = -6
0063   streamEndRun = -5
0064   globalWriteLumi = -4
0065   globalEndLumi = -3
0066   streamEndLumi = -2
0067   clearEvent = -1
0068   Event = 0
0069   streamBeginLumi = 2
0070   globalBeginLumi = 3
0071   streamBeginRun = 5
0072   globalBeginRun = 6
0073   accessInputProcessBlock = 8
0074   beginProcessBlock = 9
0075   openFile = 10
0076   beginStream = 11
0077   beginJob  = 12
0078   esSync = 13
0079   esSyncEnqueue = 14
0080   getNextTransition = 15
0081   construction = 16
0082   startTracing = 17
0083
0084 #used for json output
0085 class Activity (object):
0086   prefetch = 0
0087   acquire = 1
0088   process = 2
0089   delayedGet = 3
0090   externalWork = 4
0091   temporary = 100
0092
0093 transitionToNames_ = {
0094     Phase.startTracing: 'start tracing',
0095     Phase.construction: 'construction',
0096     Phase.destruction: 'destruction',
0097     Phase.beginJob: 'begin job',
0098     Phase.endJob: 'end job',
0099     Phase.beginStream: 'begin stream',
0100     Phase.endStream: 'end stream',
0101     Phase.beginProcessBlock: 'begin process block',
0102     Phase.endProcessBlock: 'end process block',
0103     Phase.accessInputProcessBlock: 'access input process block',
0104     Phase.writeProcessBlock: 'write process block',
0105     Phase.globalBeginRun: 'global begin run',
0106     Phase.globalEndRun: 'global end run',
0107     Phase.globalWriteRun: 'global write run',
0108     Phase.streamBeginRun: 'stream begin run',
0109     Phase.streamEndRun: 'stream end run',
0110     Phase.globalBeginLumi: 'global begin lumi',
0111     Phase.globalEndLumi: 'global end lumi',
0112     Phase.globalWriteLumi: 'global write lumi',
0113     Phase.streamBeginLumi: 'stream begin lumi',
0114     Phase.streamEndLumi: 'stream end lumi',
0115     Phase.esSyncEnqueue: 'EventSetup synchronization',
0116     Phase.esSync: 'EventSetup synchronization',
0117     Phase.Event: 'event',
0118     Phase.clearEvent: 'clear event',
0119     Phase.getNextTransition: 'get next transition'
0120 }
0121
0122 def transitionName(transition):
0123     return transitionToNames_[transition]
0124
0125 transitionToIndent_ = {
0126     Phase.startTracing: 0,
0127     Phase.construction: 0,
0128     Phase.destruction: 0,
0129     Phase.endJob: 0,
0130     Phase.beginJob: 0,
0131     Phase.beginStream: 0,
0132     Phase.endStream: 0,
0133     Phase.beginProcessBlock: 1,
0134     Phase.endProcessBlock: 1,
0135     Phase.accessInputProcessBlock: 1,
0136     Phase.writeProcessBlock: 1,
0137     Phase.globalBeginRun: 1,
0138     Phase.globalEndRun: 1,
0139     Phase.globalWriteRun: 1,
0140     Phase.streamBeginRun: 1,
0141     Phase.streamEndRun: 1,
0142     Phase.globalBeginLumi: 2,
0143     Phase.globalEndLumi: 2,
0144     Phase.globalWriteLumi: 2,
0145     Phase.streamBeginLumi: 2,
0146     Phase.streamEndLumi: 2,
0147     Phase.Event: 3,
0148     Phase.clearEvent: 3,
0149     Phase.esSyncEnqueue: 1,
0150     Phase.esSync: 1,
0151     Phase.getNextTransition: 1
0152 }
0153 def transitionIndentLevel(transition):
0154     return transitionToIndent_[transition]
0155
0156 globalTransitions_ = {
0157     Phase.startTracing,
0158     Phase.construction,
0159     Phase.destruction,
0160     Phase.endJob,
0161     Phase.beginJob,
0162     Phase.beginProcessBlock,
0163     Phase.endProcessBlock,
0164     Phase.accessInputProcessBlock,
0165     Phase.writeProcessBlock,
0166     Phase.globalBeginRun,
0167     Phase.globalEndRun,
0168     Phase.globalWriteRun,
0169     Phase.globalBeginLumi,
0170     Phase.globalEndLumi,
0171     Phase.globalWriteLumi,
0172     Phase.esSyncEnqueue,
0173     Phase.esSync,
0174     Phase.getNextTransition
0175 }
0176 def transitionIsGlobal(transition):
0177     return transition in globalTransitions_;
0178
0179 def textPrefix_(time, indentLevel):
0180     #using 11 spaces for time should accomodate a job that runs 24 hrs
0181     return f'{time:>11} '+"++"*indentLevel
0182
0183 class FrameworkTransitionParser (object):
0184     def __init__(self, payload):
0185         self.transition = int(payload[0])
0186         self.index = int(payload[1])
0187         self.sync = (int(payload[2]), int(payload[3]), int(payload[4]))
0188         self.time = int(payload[5])
0189     def indentLevel(self):
0190         return transitionIndentLevel(self.transition)
0191     def textPrefix(self):
0192         return textPrefix_(self.time, self.indentLevel())
0193     def syncText(self):
0194         if self.transition == Phase.globalBeginRun or Phase.globalEndRun == self.transition:
0195             return f'run={self.sync[0]}'
0196         if self.transition == Phase.globalWriteRun:
0197             return f'run={self.sync[0]}'
0198         if self.transition == Phase.streamBeginRun or Phase.streamEndRun == self.transition:
0199             return f'run={self.sync[0]}'
0200         if self.transition == Phase.globalBeginLumi or Phase.globalEndLumi == self.transition:
0201             return f'run={self.sync[0]} lumi={self.sync[1]}'
0202         if self.transition == Phase.globalWriteLumi:
0203             return f'run={self.sync[0]} lumi={self.sync[1]}'
0204         if self.transition == Phase.streamBeginLumi or Phase.streamEndLumi == self.transition:
0205             return f'run={self.sync[0]} lumi={self.sync[1]}'
0206         if self.transition == Phase.Event:
0207             return f'run={self.sync[0]} lumi={self.sync[1]} event={self.sync[2]}'
0208         if self.transition == Phase.esSyncEnqueue or self.transition == Phase.esSync:
0209             return f'run={self.sync[0]} lumi={self.sync[1]}'
0210         if self.transition == Phase.beginJob:
0211             return ''
0212         if self.transition == Phase.beginProcessBlock or self.transition == Phase.endProcessBlock or self.transition == Phase.writeProcessBlock or self.transition == Phase.accessInputProcessBlock:
0213             return ''
0214         if self.transition == Phase.startTracing:
0215             return ''
0216         if self.transition == Phase.construction or self.transition == Phase.destruction:
0217             return ''
0218     def textPostfix(self):
0219         return f'{transitionName(self.transition)} : id={self.index} {self.syncText()}'
0220     def text(self, context):
0221         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0222
0223 def findMatchingTransition(sync, containers):
0224     for i in range(len(containers)):
0225         if containers[i][-1]["sync"] == sync:
0226             return i
0227     #need more exhausting search
0228     for i in range(len(containers)):
0229         for t in containers[i]:
0230             if t["sync"] == sync:
0231                 return i
0232
0233     print("find failed",sync, containers)
0234     return None
0235
0236 def popQueuedTransitions(sync, container):
0237     results = []
0238     for i in range(len(container)):
0239         if sync == container[i]["sync"]:
0240             results.append(container[i])
0241             results.append(container[i+1])
0242             del container[i]
0243             del container[i]
0244             break
0245     return results
0246
0247 transitionsToFindMatch_ = {
0248     Phase.globalEndRun,
0249     Phase.globalEndLumi,
0250     Phase.globalWriteRun,
0251     Phase.globalWriteLumi
0252 }
0253
0254 class PreFrameworkTransitionParser (FrameworkTransitionParser):
0255     def __init__(self, payload):
0256         super().__init__(payload)
0257     def textSpecial(self):
0258         return "starting"
0259     def jsonInfo(self, counter, data):
0260         if transitionIsGlobal(self.transition):
0261             index = 0
0262             if self.transition == Phase.startTracing:
0263                 data.indexedGlobal(0).append(jsonTransition(type=self.transition, id=index, sync=list(self.sync),start=0, finish=self.time ))
0264                 return
0265             elif self.transition == Phase.esSync:
0266                 if self.sync[1] == kLargestLumiNumber:
0267                     #at end run transition
0268                     index = findMatchingTransition(list(self.sync), data.allGlobals())
0269                     container = data.indexedGlobal(index)
0270                     container[-1]["finish"] = self.time*kMicroToSec
0271                 else:
0272                     data._queued[-1]["finish"] = self.time*kMicroToSec
0273                     data._queued.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0274                     return
0275             elif self.transition==Phase.globalBeginRun:
0276                 index = self.index
0277                 #find associated es queued items
0278                 queued = data._queued
0279                 q = popQueuedTransitions(list(self.sync), queued)
0280                 container = data.indexedGlobal(index)
0281                 #find source, should be previous
0282                 last = container[-1]
0283                 if last["type"]==Phase.globalBeginRun and last["isSrc"]:
0284                     last["sync"]=list(self.sync)
0285                 container.append(q[0])
0286                 container.append(q[1])
0287             elif self.transition==Phase.globalBeginLumi:
0288                 index = self.index
0289                 #find associated es queued items
0290                 queued = data._queued
0291                 q = popQueuedTransitions(list(self.sync), queued)
0292                 container = data.indexedGlobal(index)
0293                 #find source, should be previous
0294                 last = container[-1]
0295                 if last["type"]==Phase.globalBeginLumi and last["isSrc"]:
0296                     last["sync"]=list(self.sync)
0297                 container.append(q[0])
0298                 container.append(q[1])
0299             elif self.transition in transitionsToFindMatch_:
0300                 index = findMatchingTransition(list(self.sync), data.allGlobals())
0301             container = data.indexedGlobal(index)
0302         else:
0303             container = data.indexedStream(self.index)
0304             if self.transition == Phase.Event:
0305                 #find source, should be previous
0306                 last = container[-1]
0307                 if last["type"]==Phase.Event and last["isSrc"]:
0308                     last["sync"]=list(self.sync)
0309             index = self.index
0310         container.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0311
0312
0313 class PostFrameworkTransitionParser (FrameworkTransitionParser):
0314     def __init__(self, payload):
0315         super().__init__(payload)
0316     def textSpecial(self):
0317         return "finished"
0318     def jsonInfo(self, counter, data):
0319         if transitionIsGlobal(self.transition):
0320             if self.transition == Phase.esSync and self.sync[1] != kLargestLumiNumber:
0321                 data._queued[-1]['finish']=self.time*kMicroToSec
0322                 return
0323             index = findMatchingTransition(list(self.sync), data.allGlobals())
0324             container = data.indexedGlobal(index)
0325         else:
0326             container = data.indexedStream(self.index)
0327         container[-1]["finish"]=self.time*kMicroToSec
0328
0329
0330 class QueuingFrameworkTransitionParser (FrameworkTransitionParser):
0331     def __init__(self, payload):
0332         super().__init__(payload)
0333     def textSpecial(self):
0334         return "queuing"
0335     def jsonInfo(self, counter, data):
0336         index = -1
0337         if self.sync[1] == kLargestLumiNumber:
0338             #find the mtching open run
0339             index = findMatchingTransition([self.sync[0],0,0], data.allGlobals())
0340             data.indexedGlobal(index).append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0341         else:
0342             data._queued.append(jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0343
0344 class SourceTransitionParser(object):
0345     def __init__(self, payload):
0346         self.transition = int(payload[0])
0347         if self.transition == Phase.getNextTransition:
0348             self.time = int(payload[1])
0349             self.index = -1
0350             return
0351         self.index = int(payload[1])
0352         self.time = int(payload[2])
0353     def indentLevel(self):
0354         if self.transition == Phase.globalBeginRun:
0355             return 1
0356         if self.transition == Phase.globalBeginLumi:
0357             return 2
0358         if self.transition == Phase.Event:
0359             return 3
0360         if self.transition == Phase.construction:
0361             return 1
0362         if self.transition == Phase.getNextTransition:
0363             return 1
0364         return None
0365     def textPrefix(self):
0366         return textPrefix_(self.time, self.indentLevel())
0367     def textPostfix(self):
0368         return f'source during {transitionName(self.transition)} : id={self.index}'
0369     def text(self, context):
0370         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0371
0372 class PreSourceTransitionParser(SourceTransitionParser):
0373     def __init__(self, payload, moduleCentric):
0374         self._moduleCentric = moduleCentric
0375         super().__init__(payload)
0376     def textSpecial(self):
0377         return "starting"
0378     def jsonInfo(self, counter, data):
0379         if self.transition == Phase.getNextTransition:
0380             data._nextTrans.append(jsonTransition(type=self.transition, id=self.index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0381             if self._moduleCentric:
0382                 #this all goes to a module ID sorted container so not knowing actual index is OK
0383                 data.findOpenSlotInModGlobals(0,0).append(data._nextTrans[-1])
0384             return
0385         elif self.transition == Phase.construction:
0386             index = counter.start()
0387             container = data.indexedGlobal(index)
0388         elif self.transition == Phase.Event:
0389             index = self.index
0390             container = data.indexedStream(index)
0391         else:
0392             index = self.index
0393             container = data.indexedGlobal(index)
0394         nextTrans = data._nextTrans
0395         if nextTrans:
0396             data._nextTrans = []
0397             for t in nextTrans:
0398                 t['id']=index
0399                 #find proper time order in the container
0400                 transStartTime = t['start']
0401                 inserted = False
0402                 for i in range(-1, -1*len(container), -1):
0403                     if transStartTime > container[i]['start']:
0404                         if i == -1:
0405                             container.append(t)
0406                             inserted = True
0407                             break
0408                         else:
0409                             container.insert(i+1,t)
0410                             inserted = True
0411                             break
0412                 if not inserted:
0413                     container.insert(0,t)
0414         container.append(jsonTransition(type=self.transition, id=index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0415         if self._moduleCentric:
0416             if self.transition == Phase.Event:
0417                 data.findOpenSlotInModStreams(index,0).append(container[-1])
0418             else:
0419                 data.findOpenSlotInModGlobals(index,0).append(container[-1])
0420
0421 class PostSourceTransitionParser(SourceTransitionParser):
0422     def __init__(self, payload, moduleCentric):
0423         super().__init__(payload)
0424         self._moduleCentric = moduleCentric
0425     def textSpecial(self):
0426         return "finished"
0427     def jsonInfo(self, counter, data):
0428         index = self.index
0429         if self.transition == Phase.Event:
0430             container = data.indexedStream(index)
0431         elif self.transition == Phase.getNextTransition:
0432             data._nextTrans[-1]['finish'] = self.time*kMicroToSec
0433             return
0434         elif self.transition == Phase.construction:
0435             pre = None
0436             for i, g in enumerate(data.allGlobals()):
0437                 for t in reversed(g):
0438                     if t["type"] != Phase.construction:
0439                         break
0440                     if t["isSrc"]:
0441                         pre = t
0442                         break
0443                 if pre:
0444                     pre["finish"]=self.time*kMicroToSec
0445                     break
0446             counter.finish(i)
0447             return
0448         else:
0449             container = data.indexedGlobal(index)
0450
0451         container[-1]["finish"]=self.time*kMicroToSec
0452
0453 class EDModuleTransitionParser(object):
0454     def __init__(self, payload, moduleNames):
0455         self.transition = int(payload[0])
0456         self.index = int(payload[1])
0457         self.moduleID = int(payload[2])
0458         self.moduleName = moduleNames[self.moduleID]
0459         self.callID = int(payload[3])
0460         self.requestingModuleID = int(payload[4])
0461         self.requestingCallID = int(payload[5])
0462         self.requestingModuleName = None
0463         if self.requestingModuleID != 0:
0464             self.requestingModuleName = moduleNames[self.requestingModuleID]
0465         self.time = int(payload[6])
0466     def baseIndentLevel(self):
0467         return transitionIndentLevel(self.transition)
0468     def textPrefix(self, context):
0469         indent = 0
0470         if self.requestingModuleID != 0:
0471             indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
0472         context[(self.transition, self.index, self.moduleID, self.callID)] = indent+1
0473         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0474     def textPostfix(self):
0475         return f'{self.moduleName} during {transitionName(self.transition)} : id={self.index}'
0476     def textIfTransform(self):
0477         if self.callID:
0478             return f' transform {self.callID-1}'
0479         return ''
0480     def text(self, context):
0481         return f'{self.textPrefix(context)} {self.textSpecial()}{self.textIfTransform()}: {self.textPostfix()}'
0482     def _preJson(self, activity, counter, data, mayUseTemp = False, isSrc = False):
0483         index = self.index
0484         found = False
0485         if mayUseTemp:
0486             compare = lambda x: x['type'] == self.transition and x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and ((x.get('isSrc') != None) == isSrc ) and (x['act'] == Activity.temporary or x['act'] == Activity.externalWork)
0487             if transitionIsGlobal(self.transition):
0488                 item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0489             else:
0490                 item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0491             if slot:
0492                 if item['act'] == Activity.temporary:
0493                     slot.pop()
0494                 else:
0495                     item['finish']=self.time*kMicroToSec
0496                 found = True
0497         if not found:
0498             if transitionIsGlobal(self.transition):
0499                 slot = data.findOpenSlotInModGlobals(index, self.moduleID)
0500             else:
0501                 slot = data.findOpenSlotInModStreams(index, self.moduleID)
0502         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=activity, start=self.time))
0503         return slot[-1]
0504     def _postJson(self, counter, data, injectAfter = None, isSrc = False):
0505         compare = lambda x: x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and x['type'] == self.transition and ((x.get('isSrc') != None) == isSrc )
0506         index = self.index
0507         if transitionIsGlobal(self.transition):
0508             item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0509         else:
0510             item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0511         if item is None:
0512             print(f"failed to find {self.moduleID} for {self.transition} in {self.index}")
0513         else:
0514             item["finish"]=self.time*kMicroToSec
0515             if injectAfter:
0516                 slot.append(injectAfter)
0517
0518 class PreEDModuleTransitionParser(EDModuleTransitionParser):
0519     def __init__(self, payload, names, moduleCentric):
0520         super().__init__(payload, names)
0521         self._moduleCentric = moduleCentric
0522     def textSpecial(self):
0523         return "starting action"
0524     def jsonInfo(self, counter, data):
0525         return self._preJson(Activity.process, counter,data, mayUseTemp=self._moduleCentric)
0526
0527 class PostEDModuleTransitionParser(EDModuleTransitionParser):
0528     def __init__(self, payload, names):
0529         super().__init__(payload, names)
0530     def textSpecial(self):
0531         return "finished action"
0532     def jsonInfo(self, counter, data):
0533         return self._postJson(counter,data)
0534
0535 class PreEDModulePrefetchingParser(EDModuleTransitionParser):
0536     def __init__(self, payload, names, moduleCentric):
0537         super().__init__(payload, names)
0538         self._moduleCentric = moduleCentric
0539     def textSpecial(self):
0540         return "starting prefetch"
0541     def jsonInfo(self, counter, data):
0542         #the total time in prefetching isn't useful, but seeing the start is
0543         entry = self._preJson(Activity.prefetch, counter,data)
0544         if self._moduleCentric:
0545             return entry
0546         kPrefetchLength = 2*kMicroToSec
0547         entry["finish"]=entry["start"]+kPrefetchLength
0548         return entry
0549
0550
0551 class PostEDModulePrefetchingParser(EDModuleTransitionParser):
0552     def __init__(self, payload, names, moduleCentric):
0553         super().__init__(payload, names)
0554         self._moduleCentric = moduleCentric
0555     def textSpecial(self):
0556         return "finished prefetch"
0557     def jsonInfo(self, counter, data):
0558         if self._moduleCentric:
0559             #inject a dummy at end of the same slot to guarantee module run is in that slot
0560             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.temporary, start=self.time))
0561         pass
0562
0563 class PreEDModuleAcquireParser(EDModuleTransitionParser):
0564     def __init__(self, payload, names, moduleCentric):
0565         super().__init__(payload, names)
0566         self._moduleCentric = moduleCentric
0567     def textSpecial(self):
0568         return "starting acquire"
0569     def jsonInfo(self, counter, data):
0570         return self._preJson(Activity.acquire, counter,data, mayUseTemp=self._moduleCentric)
0571
0572 class PostEDModuleAcquireParser(EDModuleTransitionParser):
0573     def __init__(self, payload, names, moduleCentric):
0574         super().__init__(payload, names)
0575         self._moduleCentric = moduleCentric
0576     def textSpecial(self):
0577         return "finished acquire"
0578     def jsonInfo(self, counter, data):
0579         if self._moduleCentric:
0580             #inject an external work at end of the same slot to guarantee module run is in that slot
0581             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.externalWork, start=self.time))
0582         return self._postJson(counter,data)
0583
0584 class PreEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0585     def __init__(self, payload, names, moduleCentric):
0586         super().__init__(payload, names)
0587         self._moduleCentric = moduleCentric
0588     def textSpecial(self):
0589         return "starting delayed get"
0590     def jsonInfo(self, counter, data):
0591         #a pre delayed get is effectively a post transition as the system assumes one state per module per transition instance
0592         if self._moduleCentric:
0593             #inject a dummy at end of the same slot to guarantee module run is in that slot
0594             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.temporary, start=self.time))
0595         return self._postJson(counter,data)
0596         #return self._preJson(Activity.delayedGet, counter,data)
0597
0598 class PostEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0599     def __init__(self, payload, names, moduleCentric):
0600         super().__init__(payload, names)
0601         self._moduleCentric = moduleCentric
0602     def textSpecial(self):
0603         return "finished delayed get"
0604     def jsonInfo(self, counter, data):
0605         return self._preJson(Activity.process, counter,data, mayUseTemp=self._moduleCentric)
0606         #return self._postJson(counter,data)
0607
0608 class PreEventReadFromSourceParser(EDModuleTransitionParser):
0609     def __init__(self, payload, names, moduleCentric):
0610         super().__init__(payload, names)
0611         self._moduleCentric = moduleCentric
0612     def textSpecial(self):
0613         return "starting read from source"
0614     def jsonInfo(self, counter, data):
0615         slot = self._preJson(Activity.process, counter,data, mayUseTemp=self._moduleCentric, isSrc=True)
0616         slot['isSrc'] = True
0617         return slot
0618
0619 class PostEventReadFromSourceParser(EDModuleTransitionParser):
0620     def __init__(self, payload, names):
0621         super().__init__(payload, names)
0622     def textSpecial(self):
0623         return "finished read from source"
0624     def jsonInfo(self, counter, data):
0625         return self._postJson(counter,data, isSrc=True)
0626
0627 class ESModuleTransitionParser(object):
0628     def __init__(self, payload, moduleNames, esModuleNames, recordNames):
0629         self.transition = int(payload[0])
0630         self.index = int(payload[1])
0631         self.moduleID = int(payload[2])
0632         self.moduleName = esModuleNames[self.moduleID]
0633         self.recordID = int(payload[3])
0634         self.recordName = recordNames[self.recordID]
0635         self.callID = int(payload[4])
0636         self.requestingModuleID = int(payload[5])
0637         self.requestingCallID = int(payload[6])
0638         self.requestingModuleName = None
0639         if self.requestingModuleID < 0 :
0640             self.requestingModuleName = esModuleNames[-1*self.requestingModuleID]
0641         else:
0642             self.requestingModuleName = moduleNames[self.requestingModuleID]
0643         self.time = int(payload[7])
0644     def baseIndentLevel(self):
0645         return transitionIndentLevel(self.transition)
0646     def textPrefix(self, context):
0647         indent = 0
0648         indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
0649         context[(self.transition, self.index, -1*self.moduleID, self.callID)] = indent+1
0650         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0651     def textPostfix(self):
0652         return f'esmodule {self.moduleName} in record {self.recordName} during {transitionName(self.transition)} : id={self.index}'
0653     def text(self, context):
0654         return f'{self.textPrefix(context)} {self.textSpecial()}: {self.textPostfix()}'
0655     def _preJson(self, activity, counter, data):
0656         index = self.index
0657         if transitionIsGlobal(self.transition):
0658             slot = data.findOpenSlotInModGlobals(index, -1*self.moduleID)
0659         else:
0660             slot = data.findOpenSlotInModStreams(index, -1*self.moduleID)
0661         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=-1*self.moduleID, callID=self.callID, activity=activity, start=self.time))
0662         return slot[-1]
0663     def _postJson(self, counter, data):
0664         compare = lambda x: x['id'] == self.index and x['mod'] == -1*self.moduleID and x['call'] == self.callID
0665         index = self.index
0666         if transitionIsGlobal(self.transition):
0667             item,s = data.findLastInModGlobals(index, -1*self.moduleID, compare)
0668         else:
0669             item,s = data.findLastInModStreams(index, -1*self.moduleID, compare)
0670         if item is None:
0671             print(f"failed to find {-1*self.moduleID} for {self.transition} in {self.index}")
0672             return
0673         item["finish"]=self.time*kMicroToSec
0674
0675
0676 class PreESModuleTransitionParser(ESModuleTransitionParser):
0677     def __init__(self, payload, names, esNames, recordNames):
0678         super().__init__(payload, names, esNames, recordNames)
0679     def textSpecial(self):
0680         return "starting action"
0681     def jsonInfo(self, counter, data):
0682         return self._preJson(Activity.process, counter,data)
0683
0684 class PostESModuleTransitionParser(ESModuleTransitionParser):
0685     def __init__(self, payload, names, esNames, recordNames):
0686         super().__init__(payload, names, esNames, recordNames)
0687     def textSpecial(self):
0688         return "finished action"
0689     def jsonInfo(self, counter, data):
0690         return self._postJson(counter,data)
0691
0692 class PreESModulePrefetchingParser(ESModuleTransitionParser):
0693     def __init__(self, payload, names, esNames, recordNames, moduleCentric):
0694         super().__init__(payload, names, esNames, recordNames)
0695         self._moduleCentric = moduleCentric
0696     def textSpecial(self):
0697         return "starting prefetch"
0698     def jsonInfo(self, counter, data):
0699         entry = self._preJson(Activity.prefetch, counter,data)
0700         if not self._moduleCentric:
0701             entry["finish"] = entry["start"]+2*kMicroToSec;
0702         return entry
0703
0704 class PostESModulePrefetchingParser(ESModuleTransitionParser):
0705     def __init__(self, payload, names, esNames, recordNames, moduleCentric):
0706         super().__init__(payload, names, esNames, recordNames)
0707         self._moduleCentric = moduleCentric
0708     def textSpecial(self):
0709         return "finished prefetch"
0710     def jsonInfo(self, counter, data):
0711         if self._moduleCentric:
0712             return self._postJson(counter, data)
0713         pass
0714
0715 class PreESModuleAcquireParser(ESModuleTransitionParser):
0716     def __init__(self, payload, names, recordNames):
0717         super().__init__(payload, names, recordNames)
0718     def textSpecial(self):
0719         return "starting acquire"
0720     def jsonInfo(self, counter, data):
0721         return self._preJson(Activity.acquire, counter,data)
0722
0723 class PostESModuleAcquireParser(ESModuleTransitionParser):
0724     def __init__(self, payload, names, esNames, recordNames):
0725         super().__init__(payload, names, esNames, recordNames)
0726     def textSpecial(self):
0727         return "finished acquire"
0728     def jsonInfo(self, counter, data):
0729         return self._postJson(counter,data)
0730
0731
0732 def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
0733     if step == 'F':
0734         parser = PreFrameworkTransitionParser(payload)
0735         if parser.transition == Phase.esSyncEnqueue:
0736             return QueuingFrameworkTransitionParser(payload)
0737         return parser
0738     if step == 'f':
0739         return PostFrameworkTransitionParser(payload)
0740     if step == 'S':
0741         return PreSourceTransitionParser(payload, moduleCentric)
0742     if step == 's':
0743         return PostSourceTransitionParser(payload, moduleCentric)
0744     if frameworkOnly:
0745         return None
0746     if step == 'M':
0747         return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
0748     if step == 'm':
0749         return PostEDModuleTransitionParser(payload, moduleNames)
0750     if step == 'P':
0751         return PreEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
0752     if step == 'p':
0753         return PostEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
0754     if step == 'A':
0755         return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0756     if step == 'a':
0757         return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0758     if step == 'D':
0759         return PreEDModuleEventDelayedGetParser(payload, moduleNames, moduleCentric)
0760     if step == 'd':
0761         return PostEDModuleEventDelayedGetParser(payload, moduleNames, moduleCentric)
0762     if step == 'R':
0763         return PreEventReadFromSourceParser(payload, moduleNames, moduleCentric)
0764     if step == 'r':
0765         return PostEventReadFromSourceParser(payload, moduleNames)
0766     if step == 'N':
0767         return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0768     if step == 'n':
0769         return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0770     if step == 'Q':
0771         return PreESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0772     if step == 'q':
0773         return PostESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0774     if step == 'B':
0775         return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0776     if step == 'b':
0777         return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0778
0779
0780 #----------------------------------------------
0781 def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
0782     for rawl in f:
0783         l = rawl.strip()
0784         if not l or l[0] == '#':
0785             continue
0786         (step,payload) = tuple(l.split(None,1))
0787         payload=payload.split()
0788
0789         parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
0790         if parser:
0791             yield parser
0792     return
0793
0794 class TracerCompactFileParser(object):
0795     def __init__(self,f, frameworkOnly, moduleCentric):
0796         streamBeginRun = str(Phase.streamBeginRun)
0797         numStreams = 0
0798         numStreamsFromSource = 0
0799         moduleNames = {}
0800         esModuleNames = {}
0801         recordNames = {}
0802         for rawl in f:
0803             l = rawl.strip()
0804             if l and l[0] == 'M':
0805                 i = l.split(' ')
0806                 if i[3] == streamBeginRun:
0807                     #found global begin run
0808                     numStreams = int(i[1])+1
0809                     break
0810             if numStreams == 0 and l and l[0] == 'S':
0811                 s = int(l.split(' ')[1])
0812                 if s > numStreamsFromSource:
0813                   numStreamsFromSource = s
0814             if len(l) > 5 and l[0:2] == "#M":
0815                 (id,name)=tuple(l[2:].split())
0816                 moduleNames[int(id)] = name
0817                 continue
0818             if len(l) > 5 and l[0:2] == "#N":
0819                 (id,name)=tuple(l[2:].split())
0820                 esModuleNames[int(id)] = name
0821                 continue
0822             if len(l) > 5 and l[0:2] == "#R":
0823                 (id,name)=tuple(l[2:].split())
0824                 recordNames[int(id)] = name
0825                 continue
0826
0827         self._f = f
0828         self._frameworkOnly = frameworkOnly
0829         self._moduleCentric = moduleCentric
0830         if numStreams == 0:
0831           numStreams = numStreamsFromSource +2
0832         self.numStreams =numStreams
0833         self._moduleNames = moduleNames
0834         self._esModuleNames = esModuleNames
0835         self._recordNames = recordNames
0836         self.maxNameSize =0
0837         for n in moduleNames.items():
0838             self.maxNameSize = max(self.maxNameSize,len(n))
0839         for n in esModuleNames.items():
0840             self.maxNameSize = max(self.maxNameSize,len(n))
0841         self.maxNameSize = max(self.maxNameSize,len(kSourceDelayedRead))
0842         self.maxNameSize = max(self.maxNameSize, len('streamBeginLumi'))
0843
0844     def processingSteps(self):
0845         """Create a generator which can step through the file and return each processing step.
0846         Using a generator reduces the memory overhead when parsing a large file.
0847             """
0848         self._f.seek(0)
0849         return processingStepsFromFile(self._f,self._moduleNames, self._esModuleNames, self._recordNames, self._frameworkOnly, self._moduleCentric)
0850
0851 def textOutput( parser ):
0852     context = {}
0853     for p in parser.processingSteps():
0854         print(p.text(context))
0855
0856 class Counter(object):
0857     def __init__(self):
0858         self.activeSlots = [False]
0859     def start(self):
0860         if 0 != self.activeSlots.count(False):
0861             index = self.activeSlots.index(False)
0862             self.activeSlots[index]=True
0863             return index
0864         index = len(self.activeSlots)
0865         self.activeSlots.append(True)
0866         return  index
0867     def finish(self, index):
0868         self.activeSlots[index] = False
0869
0870 class Containers(object):
0871     def __init__(self):
0872         self._modGlobals = [[]]
0873         self._modStreams = [[]]
0874         self._globals = [[]]
0875         self._streams = [[]]
0876         self._queued = []
0877         self._nextTrans = []
0878     def _extendIfNeeded(self, container, index):
0879         while len(container) < index+1:
0880             container.append([])
0881     def allGlobals(self):
0882         return self._globals
0883     def indexedGlobal(self, index):
0884         self._extendIfNeeded(self._globals, index)
0885         return self._globals[index]
0886     def allStreams(self):
0887         return self._streams
0888     def indexedStream(self, index):
0889         self._extendIfNeeded(self._streams, index)
0890         return self._streams[index]
0891     def _findOpenSlot(self, index, fullContainer):
0892         self._extendIfNeeded(fullContainer, index)
0893         container = fullContainer[index]
0894         #find open slot
0895         foundOpenSlot = False
0896         for slot in container:
0897             if len(slot) == 0:
0898                 foundOpenSlot = True
0899                 break
0900             if slot[-1]["finish"] != 0:
0901                 foundOpenSlot = True
0902                 break
0903         if not foundOpenSlot:
0904             container.append([])
0905             slot = container[-1]
0906         return slot
0907     def findOpenSlotInModGlobals(self, index, modID):
0908         return self._findOpenSlot(index, self._modGlobals)
0909     def findOpenSlotInModStreams(self, index, modID):
0910         return self._findOpenSlot(index, self._modStreams)
0911     def _findLastIn(self, index, fullContainer, comparer):
0912         container = fullContainer[index]
0913         #find slot containing the pre
0914         for slot in container:
0915             if comparer(slot[-1]):
0916                 return (slot[-1],slot)
0917         return (None,None)
0918     def findLastInModGlobals(self, index, modID, comparer):
0919         return self._findLastIn(index, self._modGlobals, comparer)
0920     def findLastInModStreams(self, index, modID, comparer):
0921         return self._findLastIn(index, self._modStreams, comparer)
0922
0923
0924 class ModuleCentricContainers(object):
0925     def __init__(self):
0926         self._modules= []
0927         self._globals = [[]]
0928         self._streams = [[]]
0929         self._queued = []
0930         self._nextTrans = []
0931         self._moduleIDOffset = 0
0932     def _moduleID2Index(self, modID):
0933         return modID + self._moduleIDOffset
0934     def _extendIfNeeded(self, container, index):
0935         while len(container) < index+1:
0936             container.append([])
0937     def _extendModulesIfNeeded(self, container, index):
0938         while index + self._moduleIDOffset < 0:
0939             container.insert(0,[])
0940             self._moduleIDOffset +=1
0941         self._extendIfNeeded(container, self._moduleID2Index(index))
0942     def allGlobals(self):
0943         return self._globals
0944     def indexedGlobal(self, index):
0945         self._extendIfNeeded(self._globals, index)
0946         return self._globals[index]
0947     def allStreams(self):
0948         return self._streams
0949     def indexedStream(self, index):
0950         self._extendIfNeeded(self._streams, index)
0951         return self._streams[index]
0952     def _findOpenSlot(self, index, fullContainer):
0953         self._extendModulesIfNeeded(fullContainer, index)
0954         container = fullContainer[self._moduleID2Index(index)]
0955         #find open slot
0956         foundOpenSlot = False
0957         for slot in container:
0958             if len(slot) == 0:
0959                 foundOpenSlot = True
0960                 break
0961             if slot[-1]["finish"] != 0:
0962                 foundOpenSlot = True
0963                 break
0964         if not foundOpenSlot:
0965             container.append([])
0966             slot = container[-1]
0967         return slot
0968     def findOpenSlotInModGlobals(self, index, modID):
0969         return self._findOpenSlot(modID, self._modules)
0970     def findOpenSlotInModStreams(self, index, modID):
0971         return self._findOpenSlot(modID, self._modules)
0972     def _findLastIn(self, index, fullContainer, comparer):
0973         if not fullContainer:
0974             return (None, None)
0975         if len(fullContainer) > self._moduleID2Index(index):
0976             container = fullContainer[self._moduleID2Index(index)]
0977         else:
0978             return (None, None)
0979         #find slot containing the pre
0980         for slot in container:
0981             if slot is not None and comparer(slot[-1]):
0982                 return (slot[-1],slot)
0983         return (None, None)
0984     def findLastInModGlobals(self, index, modID, comparer):
0985         return self._findLastIn(modID, self._modules, comparer)
0986     def findLastInModStreams(self, index, modID, comparer):
0987         return self._findLastIn(modID, self._modules, comparer)
0988
0989
0990
0991 def jsonTransition(type, id, sync, start, finish, isSrc=False):
0992     return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}
0993
0994 def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
0995     return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}
0996
0997 def startTime(x):
0998     return x["start"]
0999 def jsonInfo(parser):
1000     counter = Counter()
1001     if parser._moduleCentric:
1002         data = ModuleCentricContainers()
1003     else:
1004         data = Containers()
1005     for p in parser.processingSteps():
1006         p.jsonInfo(counter, data)
1007     #make sure everything is sorted
1008     for g in data.allGlobals():
1009         g.sort(key=startTime)
1010     final = {"transitions" : [] , "modules": [], "esModules": []}
1011     final["transitions"].append({ "name":"Global", "slots": []})
1012     globals = final["transitions"][-1]["slots"]
1013     for i, g in enumerate(data.allGlobals()):
1014         globals.append(g)
1015         if not parser._moduleCentric and not parser._frameworkOnly:
1016             if len(data._modGlobals) < i+1:
1017                 break
1018             for mod in data._modGlobals[i]:
1019                 globals.append(mod)
1020     for i,s in enumerate(data.allStreams()):
1021         final["transitions"].append({"name": f"Stream {i}", "slots":[]})
1022         stream = final["transitions"][-1]["slots"]
1023         stream.append(s)
1024         if not parser._moduleCentric and not parser._frameworkOnly:
1025             for mod in data._modStreams[i]:
1026                 stream.append(mod)
1027     if parser._moduleCentric:
1028         sourceSlot = data._modules[data._moduleID2Index(0)]
1029         modules = []
1030         for i,m in parser._moduleNames.items():
1031             modules.append({"name": f"{m}", "slots":[]})
1032             slots = modules[-1]["slots"]
1033             foundSlots = data._modules[data._moduleID2Index(i)]
1034             time = 0
1035             for s in foundSlots:
1036                 slots.append(s)
1037                 for t in s:
1038                     if t["act"] !=Activity.prefetch:
1039                         time += t["finish"]-t["start"]
1040             modules[-1]['time']=time
1041         for i,m in parser._esModuleNames.items():
1042             modules.append({"name": f"{m}", "slots":[]})
1043             slots = modules[-1]["slots"]
1044             foundSlots = data._modules[data._moduleID2Index(-1*i)]
1045             time = 0
1046             for s in foundSlots:
1047                 slots.append(s)
1048                 for t in s:
1049                     if t["act"] !=Activity.prefetch:
1050                         time += t["finish"]-t["start"]
1051             modules[-1]['time']=time
1052         modules.sort(key= lambda x : x['time'], reverse=True)
1053         final['transitions'].append({"name": "source", "slots":sourceSlot})
1054         for m in modules:
1055             final['transitions'].append(m)
1056
1057     if not parser._frameworkOnly:
1058         max = 0
1059         for k in parser._moduleNames.keys():
1060             if k > max:
1061                 max = k
1062
1063         final["modules"] =['']*(max+1)
1064         final["modules"][0] = 'source'
1065         for k,v in parser._moduleNames.items():
1066             final["modules"][k]=v
1067
1068         max = 0
1069         for k in parser._esModuleNames.keys():
1070             if k > max:
1071                 max = k
1072         final["esModules"] = ['']*(max+1)
1073         for k,v in parser._esModuleNames.items():
1074             final["esModules"][k] = v
1075     return final
1076
1077 #=======================================
1078 import unittest
1079
1080 class DummyFile(list):
1081     def __init__(self):
1082         super()
1083     def seek(self, i):
1084         pass
1085
1086 class TestModuleCommand(unittest.TestCase):
1087     def setUp(self):
1088         self.tracerFile = DummyFile()
1089         t = [0]
1090         def incr(t):
1091             t[0] += 1
1092             return t[0]
1093
1094         self.tracerFile.extend([
1095             '#R 1 Record',
1096             '#M 1 Module',
1097             '#N 1 ESModule',
1098              f'F {Phase.startTracing} 0 0 0 0 {incr(t)}',
1099              f'S {Phase.construction} 0 {incr(t)}',
1100              f's {Phase.construction} 0 {incr(t)}3',
1101              f'M {Phase.construction} 0 1 0 0 0 {incr(t)}',
1102              f'm {Phase.construction} 0 1 0 0 0 {incr(t)}',
1103              f'F {Phase.beginJob} 0 0 0 0 {incr(t)}',
1104              f'M {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
1105              f'm {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
1106              f'f {Phase.beginJob} 0 0 0 0 {incr(t)}',
1107              f'F {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1108              f'f {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1109              f'S {Phase.getNextTransition} {incr(t)}',
1110              f's {Phase.getNextTransition} {incr(t)}',
1111              f'F {Phase.esSyncEnqueue} -1 1 0 0 {incr(t)}',
1112              f'F {Phase.esSync} -1 1 0 0 {incr(t)}',
1113              f'f {Phase.esSync} -1 1 0 0 {incr(t)}',
1114              f'S {Phase.globalBeginRun} 0 {incr(t)}',
1115              f's {Phase.globalBeginRun} 0 {incr(t)}',
1116              f'S {Phase.getNextTransition} {incr(t)}',
1117              f's {Phase.getNextTransition} {incr(t)}',
1118              f'F {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1119              f'P {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1120              f'p {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1121              f'M {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1122              f'm {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1123              f'f {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1124              f'F {Phase.esSyncEnqueue} -1 1 1 0 {incr(t)}',
1125              f'F {Phase.esSync} -1 1 1 0 {incr(t)}',
1126              f'f {Phase.esSync} -1 1 1 0 {incr(t)}',
1127              f'S {Phase.getNextTransition} {incr(t)}',
1128              f's {Phase.getNextTransition} {incr(t)}',
1129              f'F {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1130              f'M {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
1131              f'm {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
1132              f'f {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1133              f'F {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1134              f'M {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
1135              f'm {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
1136              f'f {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1137              f'S {Phase.globalBeginLumi} 0 {incr(t)}',
1138              f's {Phase.globalBeginLumi} 0 {incr(t)}',
1139              f'S {Phase.getNextTransition} {incr(t)}',
1140              f's {Phase.getNextTransition} {incr(t)}',
1141              f'F {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1142              f'P {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1143              f'p {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1144              f'M {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1145              f'm {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1146              f'f {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1147              f'F {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1148              f'f {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1149              f'F {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1150              f'f {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1151              f'S {Phase.Event} 0 {incr(t)}',
1152              f's {Phase.Event} 0 {incr(t)}',
1153              f'S {Phase.getNextTransition} {incr(t)}',
1154              f's {Phase.getNextTransition} {incr(t)}',
1155              f'F {Phase.Event} 0 1 1 1 {incr(t)}',
1156              f'S {Phase.Event} 1 {incr(t)}',
1157              f's {Phase.Event} 1 {incr(t)}',
1158              f'F {Phase.Event} 1 1 1 2 {incr(t)}',
1159              f'P {Phase.Event} 0 1 0 0 0 {incr(t)}',
1160              f'p {Phase.Event} 0 1 0 0 0 {incr(t)}',
1161              f'Q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1162              f'q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1163              f'N {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1164              f'n {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1165              f'P {Phase.Event} 1 1 0 0 0 {incr(t)}',
1166              f'p {Phase.Event} 1 1 0 0 0 {incr(t)}',
1167              f'M {Phase.Event} 0 1 0 0 0 {incr(t)}',
1168              f'M {Phase.Event} 1 1 0 0 0 {incr(t)}',
1169              f'm {Phase.Event} 1 1 0 0 0 {incr(t)}',
1170              f'm {Phase.Event} 0 1 0 0 0 {incr(t)}',
1171              f'f {Phase.Event} 0 1 1 1 {incr(t)}',
1172              f'f {Phase.Event} 1 1 1 2 {incr(t)}'])
1173
1174         None
1175     def testContainers(self):
1176         c = Containers()
1177         c.indexedGlobal(2)
1178         self.assertEqual(len(c.allGlobals()), 3)
1179         c.indexedStream(2)
1180         self.assertEqual(len(c.allStreams()), 3)
1181         slot = c.findOpenSlotInModGlobals(2, 1)
1182         self.assertEqual(len(c._modGlobals),3)
1183         self.assertEqual(len(slot),0)
1184         slot.append({"start":1, "finish":0, "id":1})
1185         def testFind(item):
1186             return item["id"]==1
1187         item,s = c.findLastInModGlobals(2, 1, testFind)
1188         self.assertEqual(item["id"],1)
1189         self.assertEqual(slot,s)
1190         slot = c.findOpenSlotInModStreams(2, 1)
1191         self.assertEqual(len(c._modStreams),3)
1192         self.assertEqual(len(slot),0)
1193         slot.append({"start":1, "finish":0, "id":1})
1194         item,s = c.findLastInModStreams(2, 1, testFind)
1195         self.assertEqual(item["id"],1)
1196         self.assertEqual(slot,s)
1197     def testFrameworkOnly(self):
1198         parser = TracerCompactFileParser(self.tracerFile, True, False)
1199         j = jsonInfo(parser)
1200         #print(j)
1201         self.assertEqual(len(j["modules"]), 0)
1202         self.assertEqual(len(j["esModules"]), 0)
1203         self.assertEqual(len(j['transitions']), 3)
1204         self.assertEqual(j['transitions'][0]['name'], "Global")
1205         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1206         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1207         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1208         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1209         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1210         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1211         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1212         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1213     def testFull(self):
1214         parser = TracerCompactFileParser(self.tracerFile, False, False)
1215         j = jsonInfo(parser)
1216         #print(j)
1217         self.assertEqual(len(j["modules"]), 2)
1218         self.assertEqual(len(j["esModules"]), 2)
1219         self.assertEqual(len(j['transitions']), 3)
1220         self.assertEqual(j['transitions'][0]['name'], "Global")
1221         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1222         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1223         self.assertEqual(len(j["transitions"][0]["slots"]), 2)
1224         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1225         self.assertEqual(len(j["transitions"][0]["slots"][1]), 6)
1226         self.assertEqual(len(j["transitions"][1]["slots"]), 2)
1227         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1228         self.assertEqual(len(j["transitions"][1]["slots"][1]), 5)
1229         self.assertEqual(len(j["transitions"][2]["slots"]), 2)
1230         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1231         self.assertEqual(len(j["transitions"][2]["slots"][1]), 3)
1232     def testModuleCentric(self):
1233         parser = TracerCompactFileParser(self.tracerFile, False, True)
1234         j = jsonInfo(parser)
1235         #print(j)
1236         self.assertEqual(len(j["modules"]), 2)
1237         self.assertEqual(len(j["esModules"]), 2)
1238         self.assertEqual(len(j['transitions']), 6)
1239         self.assertEqual(j['transitions'][0]['name'], "Global")
1240         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1241         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1242         self.assertEqual(j['transitions'][3]['name'], "source")
1243         self.assertEqual(j['transitions'][4]['name'], "Module")
1244         self.assertEqual(j['transitions'][5]['name'], "ESModule")
1245         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1246         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1247         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1248         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1249         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1250         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1251         self.assertEqual(len(j["transitions"][4]["slots"]), 2)
1252         self.assertEqual(len(j["transitions"][4]["slots"][0]), 10)
1253         self.assertEqual(len(j["transitions"][4]["slots"][1]), 2)
1254         self.assertTrue(j["transitions"][4]["slots"][1][-1]['finish'] != 0.0)
1255         self.assertEqual(len(j["transitions"][5]["slots"]), 1)
1256         self.assertEqual(len(j["transitions"][5]["slots"][0]), 2)
1257
1258 def runTests():
1259     return unittest.main(argv=sys.argv[:1])
1260
1261 #=======================================
1262 if __name__=="__main__":
1263     import argparse
1264     import re
1265     import sys
1266
1267     # Program options
1268     parser = argparse.ArgumentParser(description='Convert a compact tracer file into human readable output.',
1269                                      formatter_class=argparse.RawDescriptionHelpFormatter,
1270                                      epilog=printHelp())
1271     parser.add_argument('filename',
1272                         type=argparse.FileType('r'), # open file
1273                         help='file to process')
1274     parser.add_argument('-f', '--frameworkOnly',
1275                         action='store_true',
1276                         help='''Output only the framework transitions, excluding the individual module transitions.''')
1277     parser.add_argument('-j', '--json',
1278                         action='store_true',
1279                         help='''Write output in json format.''' )
1280     parser.add_argument('-w', '--web',
1281                         action='store_true',
1282                         help='''Writes data.js file that can be used with the web based inspector. To use, copy directory ${CMSSW_RELEASE_BASE}/src/FWCore/Services/template/web to a web accessible area and move data.js into that directory.''')
1283     parser.add_argument('-m', '--module_centric',
1284                         action = 'store_true',
1285                         help='''For --json or --web, organize data by module instead of by global/stream.''' )
1286     parser.add_argument('-t', '--test',
1287                         action='store_true',
1288                         help='''Run internal tests.''')
1289
1290     args = parser.parse_args()
1291     if args.test:
1292         runTests()
1293     else :
1294         parser = TracerCompactFileParser(args.filename, args.frameworkOnly, args.module_centric)
1295         if args.json or args.web:
1296             j = json.dumps(jsonInfo(parser))
1297             if args.json:
1298                 print(j)
1299             if args.web:
1300                 f=open('data.json', 'w')
1301                 f.write(j)
1302                 f.close()
1303         else:
1304             textOutput(parser)