Services/scripts/edmTracerCompactLogViewer.py

0001 #!/usr/bin/env python3
0002 from builtins import range
0003 from itertools import groupby
0004 from operator import attrgetter,itemgetter
0005 import sys
0006 import json
0007 from collections import defaultdict
0008 #----------------------------------------------
0009 def printHelp():
0010     s = '''
0011 To Use: Add the Tracer Service to the cmsRun job use something like this
0012   in the configuration:
0013
0014   process.add_(cms.Service("Tracer", fileName = cms.untracked.string("tracer.log")))
0015
0016   After running the job, execute this script and pass the name of the
0017   Tracer log file to the script.
0018
0019   This script will output a more human readable form of the data in the Tracer log file.'''
0020     return s
0021
0022 #these values come from tracer_setupFile.cc
0023 #enum class Step : char {
0024 #  preSourceTransition = 'S',
0025 #  postSourceTransition = 's',
0026 #  preModulePrefetching = 'P',
0027 #  postModulePrefetching = 'p',
0028 #  preModuleEventAcquire = 'A',
0029 #  postModuleEventAcquire = 'a',
0030 #  preModuleTransition = 'M',
0031 #  preEventReadFromSource = 'R',
0032 #  postEventReadFromSource = 'r',
0033 #  preModuleEventDelayedGet = 'D',
0034 #  postModuleEventDelayedGet = 'd',
0035 #  postModuleTransition = 'm',
0036 #  preESModulePrefetching = 'Q',
0037 #  postESModulePrefetching = 'q',
0038 #  preESModule = 'N',
0039 #  postESModule = 'n',
0040 #  preESModuleAcquire = 'B',
0041 #  postESModuleAcquire = 'b',
0042 #  preFrameworkTransition = 'F',
0043 #  postFrameworkTransition = 'f'
0044 #};
0045
0046
0047 kMicroToSec = 0.000001
0048 #Special names
0049 kSourceFindEvent = "sourceFindEvent"
0050 kSourceDelayedRead ="sourceDelayedRead"
0051 #this value is defined in the framework itself
0052 kLargestLumiNumber = 4294967295
0053
0054 #these values must match the enum class Phase in tracer_setupFile.cc
0055 class Phase (object):
0056   destruction = -16
0057   endJob = -12
0058   endStream = -11
0059   writeProcessBlock = -10
0060   endProcessBlock = -9
0061   globalWriteRun = -7
0062   globalEndRun = -6
0063   streamEndRun = -5
0064   globalWriteLumi = -4
0065   globalEndLumi = -3
0066   streamEndLumi = -2
0067   clearEvent = -1
0068   Event = 0
0069   streamBeginLumi = 2
0070   globalBeginLumi = 3
0071   streamBeginRun = 5
0072   globalBeginRun = 6
0073   accessInputProcessBlock = 8
0074   beginProcessBlock = 9
0075   openFile = 10
0076   beginStream = 11
0077   beginJob  = 12
0078   esSync = 13
0079   esSyncEnqueue = 14
0080   getNextTransition = 15
0081   construction = 16
0082   startTracing = 17
0083
0084 #used for json output
0085 class Activity (object):
0086   prefetch = 0
0087   acquire = 1
0088   process = 2
0089   delayedGet = 3
0090   externalWork = 4
0091   temporary = 100
0092
0093 transitionToNames_ = {
0094     Phase.startTracing: 'start tracing',
0095     Phase.construction: 'construction',
0096     Phase.destruction: 'destruction',
0097     Phase.beginJob: 'begin job',
0098     Phase.endJob: 'end job',
0099     Phase.beginStream: 'begin stream',
0100     Phase.endStream: 'end stream',
0101     Phase.beginProcessBlock: 'begin process block',
0102     Phase.endProcessBlock: 'end process block',
0103     Phase.accessInputProcessBlock: 'access input process block',
0104     Phase.writeProcessBlock: 'write process block',
0105     Phase.globalBeginRun: 'global begin run',
0106     Phase.globalEndRun: 'global end run',
0107     Phase.globalWriteRun: 'global write run',
0108     Phase.streamBeginRun: 'stream begin run',
0109     Phase.streamEndRun: 'stream end run',
0110     Phase.globalBeginLumi: 'global begin lumi',
0111     Phase.globalEndLumi: 'global end lumi',
0112     Phase.globalWriteLumi: 'global write lumi',
0113     Phase.streamBeginLumi: 'stream begin lumi',
0114     Phase.streamEndLumi: 'stream end lumi',
0115     Phase.esSyncEnqueue: 'EventSetup synchronization',
0116     Phase.esSync: 'EventSetup synchronization',
0117     Phase.Event: 'event',
0118     Phase.clearEvent: 'clear event',
0119     Phase.getNextTransition: 'get next transition'
0120 }
0121
0122 def transitionName(transition):
0123     return transitionToNames_[transition]
0124
0125 transitionToIndent_ = {
0126     Phase.startTracing: 0,
0127     Phase.construction: 0,
0128     Phase.destruction: 0,
0129     Phase.endJob: 0,
0130     Phase.beginJob: 0,
0131     Phase.beginStream: 0,
0132     Phase.endStream: 0,
0133     Phase.beginProcessBlock: 1,
0134     Phase.endProcessBlock: 1,
0135     Phase.accessInputProcessBlock: 1,
0136     Phase.writeProcessBlock: 1,
0137     Phase.globalBeginRun: 1,
0138     Phase.globalEndRun: 1,
0139     Phase.globalWriteRun: 1,
0140     Phase.streamBeginRun: 1,
0141     Phase.streamEndRun: 1,
0142     Phase.globalBeginLumi: 2,
0143     Phase.globalEndLumi: 2,
0144     Phase.globalWriteLumi: 2,
0145     Phase.streamBeginLumi: 2,
0146     Phase.streamEndLumi: 2,
0147     Phase.Event: 3,
0148     Phase.clearEvent: 3,
0149     Phase.esSyncEnqueue: 1,
0150     Phase.esSync: 1,
0151     Phase.getNextTransition: 1
0152 }
0153 def transitionIndentLevel(transition):
0154     return transitionToIndent_[transition]
0155
0156 globalTransitions_ = {
0157     Phase.startTracing,
0158     Phase.construction,
0159     Phase.destruction,
0160     Phase.endJob,
0161     Phase.beginJob,
0162     Phase.beginProcessBlock,
0163     Phase.endProcessBlock,
0164     Phase.accessInputProcessBlock,
0165     Phase.writeProcessBlock,
0166     Phase.globalBeginRun,
0167     Phase.globalEndRun,
0168     Phase.globalWriteRun,
0169     Phase.globalBeginLumi,
0170     Phase.globalEndLumi,
0171     Phase.globalWriteLumi,
0172     Phase.esSyncEnqueue,
0173     Phase.esSync,
0174     Phase.getNextTransition
0175 }
0176 def transitionIsGlobal(transition):
0177     return transition in globalTransitions_;
0178
0179 def textPrefix_(time, indentLevel):
0180     #using 11 spaces for time should accomodate a job that runs 24 hrs
0181     return f'{time:>11} '+"++"*indentLevel
0182
0183 class FrameworkTransitionParser (object):
0184     def __init__(self, payload):
0185         self.transition = int(payload[0])
0186         self.index = int(payload[1])
0187         self.sync = (int(payload[2]), int(payload[3]), int(payload[4]))
0188         self.time = int(payload[5])
0189     def indentLevel(self):
0190         return transitionIndentLevel(self.transition)
0191     def textPrefix(self):
0192         return textPrefix_(self.time, self.indentLevel())
0193     def syncText(self):
0194         if self.transition == Phase.globalBeginRun or Phase.globalEndRun == self.transition:
0195             return f'run={self.sync[0]}'
0196         if self.transition == Phase.globalWriteRun:
0197             return f'run={self.sync[0]}'
0198         if self.transition == Phase.streamBeginRun or Phase.streamEndRun == self.transition:
0199             return f'run={self.sync[0]}'
0200         if self.transition == Phase.globalBeginLumi or Phase.globalEndLumi == self.transition:
0201             return f'run={self.sync[0]} lumi={self.sync[1]}'
0202         if self.transition == Phase.globalWriteLumi:
0203             return f'run={self.sync[0]} lumi={self.sync[1]}'
0204         if self.transition == Phase.streamBeginLumi or Phase.streamEndLumi == self.transition:
0205             return f'run={self.sync[0]} lumi={self.sync[1]}'
0206         if self.transition == Phase.Event:
0207             return f'run={self.sync[0]} lumi={self.sync[1]} event={self.sync[2]}'
0208         if self.transition == Phase.esSyncEnqueue or self.transition == Phase.esSync:
0209             return f'run={self.sync[0]} lumi={self.sync[1]}'
0210         if self.transition == Phase.beginJob:
0211             return ''
0212         if self.transition == Phase.beginProcessBlock or self.transition == Phase.endProcessBlock or self.transition == Phase.writeProcessBlock or self.transition == Phase.accessInputProcessBlock:
0213             return ''
0214         if self.transition == Phase.startTracing:
0215             return ''
0216         if self.transition == Phase.construction or self.transition == Phase.destruction:
0217             return ''
0218     def textPostfix(self):
0219         return f'{transitionName(self.transition)} : id={self.index} {self.syncText()}'
0220     def text(self, context):
0221         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0222
0223 def findMatchingTransition(sync, containers):
0224     for i in range(len(containers)):
0225         if containers[i][-1]["sync"] == sync:
0226             return i
0227     #need more exhausting search
0228     for i in range(len(containers)):
0229         for t in containers[i]:
0230             if t["sync"] == sync:
0231                 return i
0232
0233     print("find failed",sync, containers)
0234     return None
0235
0236 def popQueuedTransitions(sync, container):
0237     results = []
0238     for i in range(len(container)):
0239         if sync == container[i]["sync"]:
0240             results.append(container[i])
0241             results.append(container[i+1])
0242             del container[i]
0243             del container[i]
0244             break
0245     return results
0246
0247 transitionsToFindMatch_ = {
0248     Phase.globalEndRun,
0249     Phase.globalEndLumi,
0250     Phase.globalWriteRun,
0251     Phase.globalWriteLumi
0252 }
0253
0254 class PreFrameworkTransitionParser (FrameworkTransitionParser):
0255     def __init__(self, payload):
0256         super().__init__(payload)
0257     def textSpecial(self):
0258         return "starting"
0259     def jsonInfo(self, counter, data):
0260         if transitionIsGlobal(self.transition):
0261             index = 0
0262             if self.transition == Phase.startTracing:
0263                 data.indexedGlobal(0).append(jsonTransition(type=self.transition, id=index, sync=list(self.sync),start=0, finish=self.time ))
0264                 return
0265             elif self.transition == Phase.esSync:
0266                 if self.sync[1] == kLargestLumiNumber:
0267                     #at end run transition
0268                     index = findMatchingTransition(list(self.sync), data.allGlobals())
0269                     container = data.indexedGlobal(index)
0270                     container[-1]["finish"] = self.time*kMicroToSec
0271                 else:
0272                     data._queued[-1]["finish"] = self.time*kMicroToSec
0273                     data._queued.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0274                     return
0275             elif self.transition==Phase.globalBeginRun:
0276                 index = self.index
0277                 #find associated es queued items
0278                 queued = data._queued
0279                 q = popQueuedTransitions(list(self.sync), queued)
0280                 container = data.indexedGlobal(index)
0281                 #find source, should be previous
0282                 last = container[-1]
0283                 if last["type"]==Phase.globalBeginRun and last["isSrc"]:
0284                     last["sync"]=list(self.sync)
0285                 container.append(q[0])
0286                 container.append(q[1])
0287             elif self.transition==Phase.globalBeginLumi:
0288                 index = self.index
0289                 #find associated es queued items
0290                 queued = data._queued
0291                 q = popQueuedTransitions(list(self.sync), queued)
0292                 container = data.indexedGlobal(index)
0293                 #find source, should be previous
0294                 last = container[-1]
0295                 if last["type"]==Phase.globalBeginLumi and last["isSrc"]:
0296                     last["sync"]=list(self.sync)
0297                 container.append(q[0])
0298                 container.append(q[1])
0299             elif self.transition in transitionsToFindMatch_:
0300                 index = findMatchingTransition(list(self.sync), data.allGlobals())
0301             container = data.indexedGlobal(index)
0302         else:
0303             container = data.indexedStream(self.index)
0304             if self.transition == Phase.Event:
0305                 #find source, should be previous
0306                 last = container[-1]
0307                 if last["type"]==Phase.Event and last["isSrc"]:
0308                     last["sync"]=list(self.sync)
0309             index = self.index
0310         container.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0311
0312
0313 class PostFrameworkTransitionParser (FrameworkTransitionParser):
0314     def __init__(self, payload):
0315         super().__init__(payload)
0316     def textSpecial(self):
0317         return "finished"
0318     def jsonInfo(self, counter, data):
0319         if transitionIsGlobal(self.transition):
0320             if self.transition == Phase.esSync and self.sync[1] != kLargestLumiNumber:
0321                 data._queued[-1]['finish']=self.time*kMicroToSec
0322                 return
0323             index = findMatchingTransition(list(self.sync), data.allGlobals())
0324             container = data.indexedGlobal(index)
0325         else:
0326             container = data.indexedStream(self.index)
0327         container[-1]["finish"]=self.time*kMicroToSec
0328
0329
0330 class QueuingFrameworkTransitionParser (FrameworkTransitionParser):
0331     def __init__(self, payload):
0332         super().__init__(payload)
0333     def textSpecial(self):
0334         return "queuing"
0335     def jsonInfo(self, counter, data):
0336         index = -1
0337         if self.sync[1] == kLargestLumiNumber:
0338             #find the mtching open run
0339             index = findMatchingTransition([self.sync[0],0,0], data.allGlobals())
0340             data.indexedGlobal(index).append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0341         else:
0342             data._queued.append(jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
0343
0344 class SourceTransitionParser(object):
0345     def __init__(self, payload):
0346         self.transition = int(payload[0])
0347         if self.transition == Phase.getNextTransition:
0348             self.time = int(payload[1])
0349             self.index = -1
0350             return
0351         self.index = int(payload[1])
0352         self.time = int(payload[2])
0353     def indentLevel(self):
0354         if self.transition == Phase.globalBeginRun:
0355             return 1
0356         if self.transition == Phase.globalBeginLumi:
0357             return 2
0358         if self.transition == Phase.Event:
0359             return 3
0360         if self.transition == Phase.construction:
0361             return 1
0362         if self.transition == Phase.getNextTransition:
0363             return 1
0364         return None
0365     def textPrefix(self):
0366         return textPrefix_(self.time, self.indentLevel())
0367     def textPostfix(self):
0368         return f'source during {transitionName(self.transition)} : id={self.index}'
0369     def text(self, context):
0370         return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'
0371
0372 class PreSourceTransitionParser(SourceTransitionParser):
0373     def __init__(self, payload, moduleCentric):
0374         self._moduleCentric = moduleCentric
0375         super().__init__(payload)
0376     def textSpecial(self):
0377         return "starting"
0378     def jsonInfo(self, counter, data):
0379         if self.transition == Phase.getNextTransition:
0380             data._nextTrans.append(jsonTransition(type=self.transition, id=self.index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0381             if self._moduleCentric:
0382                 #this all goes to a module ID sorted container so not knowing actual index is OK
0383                 data.findOpenSlotInModGlobals(0,0).append(data._nextTrans[-1])
0384             return
0385         elif self.transition == Phase.construction:
0386             index = counter.start()
0387             container = data.indexedGlobal(index)
0388         elif self.transition == Phase.Event:
0389             index = self.index
0390             container = data.indexedStream(index)
0391         else:
0392             index = self.index
0393             container = data.indexedGlobal(index)
0394         nextTrans = data._nextTrans
0395         if nextTrans:
0396             data._nextTrans = []
0397             for t in nextTrans:
0398                 t['id']=index
0399                 #find proper time order in the container
0400                 transStartTime = t['start']
0401                 inserted = False
0402                 for i in range(-1, -1*len(container), -1):
0403                     if transStartTime > container[i]['start']:
0404                         if i == -1:
0405                             container.append(t)
0406                             inserted = True
0407                             break
0408                         else:
0409                             container.insert(i+1,t)
0410                             inserted = True
0411                             break
0412                 if not inserted:
0413                     container.insert(0,t)
0414         container.append(jsonTransition(type=self.transition, id=index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
0415         if self._moduleCentric:
0416             if self.transition == Phase.Event:
0417                 data.findOpenSlotInModStreams(index,0).append(container[-1])
0418             else:
0419                 data.findOpenSlotInModGlobals(index,0).append(container[-1])
0420
0421 class PostSourceTransitionParser(SourceTransitionParser):
0422     def __init__(self, payload, moduleCentric):
0423         super().__init__(payload)
0424         self._moduleCentric = moduleCentric
0425     def textSpecial(self):
0426         return "finished"
0427     def jsonInfo(self, counter, data):
0428         index = self.index
0429         if self.transition == Phase.Event:
0430             container = data.indexedStream(index)
0431         elif self.transition == Phase.getNextTransition:
0432             data._nextTrans[-1]['finish'] = self.time*kMicroToSec
0433             return
0434         elif self.transition == Phase.construction:
0435             pre = None
0436             for i, g in enumerate(data.allGlobals()):
0437                 for t in reversed(g):
0438                     if t["type"] != Phase.construction:
0439                         break
0440                     if t["isSrc"]:
0441                         pre = t
0442                         break
0443                 if pre:
0444                     pre["finish"]=self.time*kMicroToSec
0445                     break
0446             counter.finish(i)
0447             return
0448         else:
0449             container = data.indexedGlobal(index)
0450
0451         container[-1]["finish"]=self.time*kMicroToSec
0452
0453 class EDModuleTransitionParser(object):
0454     def __init__(self, payload, moduleNames):
0455         self.transition = int(payload[0])
0456         self.index = int(payload[1])
0457         self.moduleID = int(payload[2])
0458         self.moduleName = moduleNames[self.moduleID]
0459         self.callID = int(payload[3])
0460         self.requestingModuleID = int(payload[4])
0461         self.requestingCallID = int(payload[5])
0462         self.requestingModuleName = None
0463         if self.requestingModuleID != 0:
0464             self.requestingModuleName = moduleNames[self.requestingModuleID]
0465         self.time = int(payload[6])
0466     def baseIndentLevel(self):
0467         return transitionIndentLevel(self.transition)
0468     def textPrefix(self, context):
0469         indent = 0
0470         if self.requestingModuleID != 0:
0471             indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
0472         context[(self.transition, self.index, self.moduleID, self.callID)] = indent+1
0473         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0474     def textPostfix(self):
0475         return f'{self.moduleName} during {transitionName(self.transition)} : id={self.index}'
0476     def textIfTransform(self):
0477         if self.callID:
0478             return f' transform {self.callID-1}'
0479         return ''
0480     def text(self, context):
0481         return f'{self.textPrefix(context)} {self.textSpecial()}{self.textIfTransform()}: {self.textPostfix()}'
0482     def _preJson(self, activity, counter, data, mayUseTemp = False):
0483         index = self.index
0484         found = False
0485         if mayUseTemp:
0486             compare = lambda x: x['type'] == self.transition and x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and (x['act'] == Activity.temporary or x['act'] == Activity.externalWork)
0487             if transitionIsGlobal(self.transition):
0488                 item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0489             else:
0490                 item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0491             if slot:
0492                 if item['act'] == Activity.temporary:
0493                     slot.pop()
0494                 else:
0495                     item['finish']=self.time*kMicroToSec
0496                 found = True
0497         if not found:
0498             if transitionIsGlobal(self.transition):
0499                 slot = data.findOpenSlotInModGlobals(index, self.moduleID)
0500             else:
0501                 slot = data.findOpenSlotInModStreams(index, self.moduleID)
0502         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=activity, start=self.time))
0503         return slot[-1]
0504     def _postJson(self, counter, data, injectAfter = None):
0505         compare = lambda x: x['id'] == self.index and x['mod'] == self.moduleID and x['call'] == self.callID and x['type'] == self.transition
0506         index = self.index
0507         if transitionIsGlobal(self.transition):
0508             item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
0509         else:
0510             item,slot = data.findLastInModStreams(index, self.moduleID, compare)
0511         if item is None:
0512             print(f"failed to find {self.moduleID} for {self.transition} in {self.index}")
0513         else:
0514             item["finish"]=self.time*kMicroToSec
0515             if injectAfter:
0516                 slot.append(injectAfter)
0517
0518 class PreEDModuleTransitionParser(EDModuleTransitionParser):
0519     def __init__(self, payload, names, moduleCentric):
0520         super().__init__(payload, names)
0521         self._moduleCentric = moduleCentric
0522     def textSpecial(self):
0523         return "starting action"
0524     def jsonInfo(self, counter, data):
0525         return self._preJson(Activity.process, counter,data, mayUseTemp=self._moduleCentric)
0526
0527 class PostEDModuleTransitionParser(EDModuleTransitionParser):
0528     def __init__(self, payload, names):
0529         super().__init__(payload, names)
0530     def textSpecial(self):
0531         return "finished action"
0532     def jsonInfo(self, counter, data):
0533         return self._postJson(counter,data)
0534
0535 class PreEDModulePrefetchingParser(EDModuleTransitionParser):
0536     def __init__(self, payload, names, moduleCentric):
0537         super().__init__(payload, names)
0538         self._moduleCentric = moduleCentric
0539     def textSpecial(self):
0540         return "starting prefetch"
0541     def jsonInfo(self, counter, data):
0542         #the total time in prefetching isn't useful, but seeing the start is
0543         entry = self._preJson(Activity.prefetch, counter,data)
0544         if self._moduleCentric:
0545             return entry
0546         kPrefetchLength = 2*kMicroToSec
0547         entry["finish"]=entry["start"]+kPrefetchLength
0548         return entry
0549
0550
0551 class PostEDModulePrefetchingParser(EDModuleTransitionParser):
0552     def __init__(self, payload, names, moduleCentric):
0553         super().__init__(payload, names)
0554         self._moduleCentric = moduleCentric
0555     def textSpecial(self):
0556         return "finished prefetch"
0557     def jsonInfo(self, counter, data):
0558         if self._moduleCentric:
0559             #inject a dummy at end of the same slot to guarantee module run is in that slot
0560             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.temporary, start=self.time))
0561         pass
0562
0563 class PreEDModuleAcquireParser(EDModuleTransitionParser):
0564     def __init__(self, payload, names, moduleCentric):
0565         super().__init__(payload, names)
0566         self._moduleCentric = moduleCentric
0567     def textSpecial(self):
0568         return "starting acquire"
0569     def jsonInfo(self, counter, data):
0570         return self._preJson(Activity.acquire, counter,data, mayUseTemp=self._moduleCentric)
0571
0572 class PostEDModuleAcquireParser(EDModuleTransitionParser):
0573     def __init__(self, payload, names, moduleCentric):
0574         super().__init__(payload, names)
0575         self._moduleCentric = moduleCentric
0576     def textSpecial(self):
0577         return "finished acquire"
0578     def jsonInfo(self, counter, data):
0579         if self._moduleCentric:
0580             #inject an external work at end of the same slot to guarantee module run is in that slot
0581             return self._postJson(counter, data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, activity=Activity.externalWork, start=self.time))
0582         return self._postJson(counter,data)
0583
0584 class PreEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0585     def __init__(self, payload, names):
0586         super().__init__(payload, names)
0587     def textSpecial(self):
0588         return "starting delayed get"
0589     def jsonInfo(self, counter, data):
0590         return self._preJson(Activity.delayedGet, counter,data)
0591
0592 class PostEDModuleEventDelayedGetParser(EDModuleTransitionParser):
0593     def __init__(self, payload, names):
0594         super().__init__(payload, names)
0595     def textSpecial(self):
0596         return "finished delayed get"
0597     def jsonInfo(self, counter, data):
0598         return self._postJson(counter,data)
0599
0600 class PreEventReadFromSourceParser(EDModuleTransitionParser):
0601     def __init__(self, payload, names):
0602         super().__init__(payload, names)
0603     def textSpecial(self):
0604         return "starting read from source"
0605     def jsonInfo(self, counter, data):
0606         slot = self._preJson(Activity.process, counter,data)
0607         slot['isSrc'] = True
0608         return slot
0609
0610 class PostEventReadFromSourceParser(EDModuleTransitionParser):
0611     def __init__(self, payload, names):
0612         super().__init__(payload, names)
0613     def textSpecial(self):
0614         return "finished read from source"
0615     def jsonInfo(self, counter, data):
0616         return self._postJson(counter,data)
0617
0618 class ESModuleTransitionParser(object):
0619     def __init__(self, payload, moduleNames, esModuleNames, recordNames):
0620         self.transition = int(payload[0])
0621         self.index = int(payload[1])
0622         self.moduleID = int(payload[2])
0623         self.moduleName = esModuleNames[self.moduleID]
0624         self.recordID = int(payload[3])
0625         self.recordName = recordNames[self.recordID]
0626         self.callID = int(payload[4])
0627         self.requestingModuleID = int(payload[5])
0628         self.requestingCallID = int(payload[6])
0629         self.requestingModuleName = None
0630         if self.requestingModuleID < 0 :
0631             self.requestingModuleName = esModuleNames[-1*self.requestingModuleID]
0632         else:
0633             self.requestingModuleName = moduleNames[self.requestingModuleID]
0634         self.time = int(payload[7])
0635     def baseIndentLevel(self):
0636         return transitionIndentLevel(self.transition)
0637     def textPrefix(self, context):
0638         indent = 0
0639         indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
0640         context[(self.transition, self.index, -1*self.moduleID, self.callID)] = indent+1
0641         return textPrefix_(self.time, indent+1+self.baseIndentLevel())
0642     def textPostfix(self):
0643         return f'esmodule {self.moduleName} in record {self.recordName} during {transitionName(self.transition)} : id={self.index}'
0644     def text(self, context):
0645         return f'{self.textPrefix(context)} {self.textSpecial()}: {self.textPostfix()}'
0646     def _preJson(self, activity, counter, data):
0647         index = self.index
0648         if transitionIsGlobal(self.transition):
0649             slot = data.findOpenSlotInModGlobals(index, -1*self.moduleID)
0650         else:
0651             slot = data.findOpenSlotInModStreams(index, -1*self.moduleID)
0652         slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=-1*self.moduleID, callID=self.callID, activity=activity, start=self.time))
0653         return slot[-1]
0654     def _postJson(self, counter, data):
0655         compare = lambda x: x['id'] == self.index and x['mod'] == -1*self.moduleID and x['call'] == self.callID
0656         index = self.index
0657         if transitionIsGlobal(self.transition):
0658             item,s = data.findLastInModGlobals(index, -1*self.moduleID, compare)
0659         else:
0660             item,s = data.findLastInModStreams(index, -1*self.moduleID, compare)
0661         if item is None:
0662             print(f"failed to find {-1*self.moduleID} for {self.transition} in {self.index}")
0663             return
0664         item["finish"]=self.time*kMicroToSec
0665
0666
0667 class PreESModuleTransitionParser(ESModuleTransitionParser):
0668     def __init__(self, payload, names, esNames, recordNames):
0669         super().__init__(payload, names, esNames, recordNames)
0670     def textSpecial(self):
0671         return "starting action"
0672     def jsonInfo(self, counter, data):
0673         return self._preJson(Activity.process, counter,data)
0674
0675 class PostESModuleTransitionParser(ESModuleTransitionParser):
0676     def __init__(self, payload, names, esNames, recordNames):
0677         super().__init__(payload, names, esNames, recordNames)
0678     def textSpecial(self):
0679         return "finished action"
0680     def jsonInfo(self, counter, data):
0681         return self._postJson(counter,data)
0682
0683 class PreESModulePrefetchingParser(ESModuleTransitionParser):
0684     def __init__(self, payload, names, esNames, recordNames, moduleCentric):
0685         super().__init__(payload, names, esNames, recordNames)
0686         self._moduleCentric = moduleCentric
0687     def textSpecial(self):
0688         return "starting prefetch"
0689     def jsonInfo(self, counter, data):
0690         entry = self._preJson(Activity.prefetch, counter,data)
0691         if not self._moduleCentric:
0692             entry["finish"] = entry["start"]+2*kMicroToSec;
0693         return entry
0694
0695 class PostESModulePrefetchingParser(ESModuleTransitionParser):
0696     def __init__(self, payload, names, esNames, recordNames, moduleCentric):
0697         super().__init__(payload, names, esNames, recordNames)
0698         self._moduleCentric = moduleCentric
0699     def textSpecial(self):
0700         return "finished prefetch"
0701     def jsonInfo(self, counter, data):
0702         if self._moduleCentric:
0703             return self._postJson(counter, data)
0704         pass
0705
0706 class PreESModuleAcquireParser(ESModuleTransitionParser):
0707     def __init__(self, payload, names, recordNames):
0708         super().__init__(payload, names, recordNames)
0709     def textSpecial(self):
0710         return "starting acquire"
0711     def jsonInfo(self, counter, data):
0712         return self._preJson(Activity.acquire, counter,data)
0713
0714 class PostESModuleAcquireParser(ESModuleTransitionParser):
0715     def __init__(self, payload, names, esNames, recordNames):
0716         super().__init__(payload, names, esNames, recordNames)
0717     def textSpecial(self):
0718         return "finished acquire"
0719     def jsonInfo(self, counter, data):
0720         return self._postJson(counter,data)
0721
0722
0723 def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
0724     if step == 'F':
0725         parser = PreFrameworkTransitionParser(payload)
0726         if parser.transition == Phase.esSyncEnqueue:
0727             return QueuingFrameworkTransitionParser(payload)
0728         return parser
0729     if step == 'f':
0730         return PostFrameworkTransitionParser(payload)
0731     if step == 'S':
0732         return PreSourceTransitionParser(payload, moduleCentric)
0733     if step == 's':
0734         return PostSourceTransitionParser(payload, moduleCentric)
0735     if frameworkOnly:
0736         return None
0737     if step == 'M':
0738         return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
0739     if step == 'm':
0740         return PostEDModuleTransitionParser(payload, moduleNames)
0741     if step == 'P':
0742         return PreEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
0743     if step == 'p':
0744         return PostEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
0745     if step == 'A':
0746         return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0747     if step == 'a':
0748         return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
0749     if step == 'D':
0750         return PreEDModuleEventDelayedGetParser(payload, moduleNames)
0751     if step == 'd':
0752         return PostEDModuleEventDelayedGetParser(payload, moduleNames)
0753     if step == 'R':
0754         return PreEventReadFromSourceParser(payload, moduleNames)
0755     if step == 'r':
0756         return PostEventReadFromSourceParser(payload, moduleNames)
0757     if step == 'N':
0758         return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0759     if step == 'n':
0760         return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
0761     if step == 'Q':
0762         return PreESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0763     if step == 'q':
0764         return PostESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
0765     if step == 'B':
0766         return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0767     if step == 'b':
0768         return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
0769
0770
0771 #----------------------------------------------
0772 def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
0773     for rawl in f:
0774         l = rawl.strip()
0775         if not l or l[0] == '#':
0776             continue
0777         (step,payload) = tuple(l.split(None,1))
0778         payload=payload.split()
0779
0780         parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
0781         if parser:
0782             yield parser
0783     return
0784
0785 class TracerCompactFileParser(object):
0786     def __init__(self,f, frameworkOnly, moduleCentric):
0787         streamBeginRun = str(Phase.streamBeginRun)
0788         numStreams = 0
0789         numStreamsFromSource = 0
0790         moduleNames = {}
0791         esModuleNames = {}
0792         recordNames = {}
0793         for rawl in f:
0794             l = rawl.strip()
0795             if l and l[0] == 'M':
0796                 i = l.split(' ')
0797                 if i[3] == streamBeginRun:
0798                     #found global begin run
0799                     numStreams = int(i[1])+1
0800                     break
0801             if numStreams == 0 and l and l[0] == 'S':
0802                 s = int(l.split(' ')[1])
0803                 if s > numStreamsFromSource:
0804                   numStreamsFromSource = s
0805             if len(l) > 5 and l[0:2] == "#M":
0806                 (id,name)=tuple(l[2:].split())
0807                 moduleNames[int(id)] = name
0808                 continue
0809             if len(l) > 5 and l[0:2] == "#N":
0810                 (id,name)=tuple(l[2:].split())
0811                 esModuleNames[int(id)] = name
0812                 continue
0813             if len(l) > 5 and l[0:2] == "#R":
0814                 (id,name)=tuple(l[2:].split())
0815                 recordNames[int(id)] = name
0816                 continue
0817
0818         self._f = f
0819         self._frameworkOnly = frameworkOnly
0820         self._moduleCentric = moduleCentric
0821         if numStreams == 0:
0822           numStreams = numStreamsFromSource +2
0823         self.numStreams =numStreams
0824         self._moduleNames = moduleNames
0825         self._esModuleNames = esModuleNames
0826         self._recordNames = recordNames
0827         self.maxNameSize =0
0828         for n in moduleNames.items():
0829             self.maxNameSize = max(self.maxNameSize,len(n))
0830         for n in esModuleNames.items():
0831             self.maxNameSize = max(self.maxNameSize,len(n))
0832         self.maxNameSize = max(self.maxNameSize,len(kSourceDelayedRead))
0833         self.maxNameSize = max(self.maxNameSize, len('streamBeginLumi'))
0834
0835     def processingSteps(self):
0836         """Create a generator which can step through the file and return each processing step.
0837         Using a generator reduces the memory overhead when parsing a large file.
0838             """
0839         self._f.seek(0)
0840         return processingStepsFromFile(self._f,self._moduleNames, self._esModuleNames, self._recordNames, self._frameworkOnly, self._moduleCentric)
0841
0842 def textOutput( parser ):
0843     context = {}
0844     for p in parser.processingSteps():
0845         print(p.text(context))
0846
0847 class Counter(object):
0848     def __init__(self):
0849         self.activeSlots = [False]
0850     def start(self):
0851         if 0 != self.activeSlots.count(False):
0852             index = self.activeSlots.index(False)
0853             self.activeSlots[index]=True
0854             return index
0855         index = len(self.activeSlots)
0856         self.activeSlots.append(True)
0857         return  index
0858     def finish(self, index):
0859         self.activeSlots[index] = False
0860
0861 class Containers(object):
0862     def __init__(self):
0863         self._modGlobals = [[]]
0864         self._modStreams = [[]]
0865         self._globals = [[]]
0866         self._streams = [[]]
0867         self._queued = []
0868         self._nextTrans = []
0869     def _extendIfNeeded(self, container, index):
0870         while len(container) < index+1:
0871             container.append([])
0872     def allGlobals(self):
0873         return self._globals
0874     def indexedGlobal(self, index):
0875         self._extendIfNeeded(self._globals, index)
0876         return self._globals[index]
0877     def allStreams(self):
0878         return self._streams
0879     def indexedStream(self, index):
0880         self._extendIfNeeded(self._streams, index)
0881         return self._streams[index]
0882     def _findOpenSlot(self, index, fullContainer):
0883         self._extendIfNeeded(fullContainer, index)
0884         container = fullContainer[index]
0885         #find open slot
0886         foundOpenSlot = False
0887         for slot in container:
0888             if len(slot) == 0:
0889                 foundOpenSlot = True
0890                 break
0891             if slot[-1]["finish"] != 0:
0892                 foundOpenSlot = True
0893                 break
0894         if not foundOpenSlot:
0895             container.append([])
0896             slot = container[-1]
0897         return slot
0898     def findOpenSlotInModGlobals(self, index, modID):
0899         return self._findOpenSlot(index, self._modGlobals)
0900     def findOpenSlotInModStreams(self, index, modID):
0901         return self._findOpenSlot(index, self._modStreams)
0902     def _findLastIn(self, index, fullContainer, comparer):
0903         container = fullContainer[index]
0904         #find slot containing the pre
0905         for slot in container:
0906             if comparer(slot[-1]):
0907                 return (slot[-1],slot)
0908         return (None,None)
0909     def findLastInModGlobals(self, index, modID, comparer):
0910         return self._findLastIn(index, self._modGlobals, comparer)
0911     def findLastInModStreams(self, index, modID, comparer):
0912         return self._findLastIn(index, self._modStreams, comparer)
0913
0914
0915 class ModuleCentricContainers(object):
0916     def __init__(self):
0917         self._modules= []
0918         self._globals = [[]]
0919         self._streams = [[]]
0920         self._queued = []
0921         self._nextTrans = []
0922         self._moduleIDOffset = 0
0923     def _moduleID2Index(self, modID):
0924         return modID + self._moduleIDOffset
0925     def _extendIfNeeded(self, container, index):
0926         while len(container) < index+1:
0927             container.append([])
0928     def _extendModulesIfNeeded(self, container, index):
0929         while index + self._moduleIDOffset < 0:
0930             container.insert(0,[])
0931             self._moduleIDOffset +=1
0932         self._extendIfNeeded(container, self._moduleID2Index(index))
0933     def allGlobals(self):
0934         return self._globals
0935     def indexedGlobal(self, index):
0936         self._extendIfNeeded(self._globals, index)
0937         return self._globals[index]
0938     def allStreams(self):
0939         return self._streams
0940     def indexedStream(self, index):
0941         self._extendIfNeeded(self._streams, index)
0942         return self._streams[index]
0943     def _findOpenSlot(self, index, fullContainer):
0944         self._extendModulesIfNeeded(fullContainer, index)
0945         container = fullContainer[self._moduleID2Index(index)]
0946         #find open slot
0947         foundOpenSlot = False
0948         for slot in container:
0949             if len(slot) == 0:
0950                 foundOpenSlot = True
0951                 break
0952             if slot[-1]["finish"] != 0:
0953                 foundOpenSlot = True
0954                 break
0955         if not foundOpenSlot:
0956             container.append([])
0957             slot = container[-1]
0958         return slot
0959     def findOpenSlotInModGlobals(self, index, modID):
0960         return self._findOpenSlot(modID, self._modules)
0961     def findOpenSlotInModStreams(self, index, modID):
0962         return self._findOpenSlot(modID, self._modules)
0963     def _findLastIn(self, index, fullContainer, comparer):
0964         if not fullContainer:
0965             return (None, None)
0966         if len(fullContainer) > self._moduleID2Index(index):
0967             container = fullContainer[self._moduleID2Index(index)]
0968         else:
0969             return (None, None)
0970         #find slot containing the pre
0971         for slot in container:
0972             if slot is not None and comparer(slot[-1]):
0973                 return (slot[-1],slot)
0974         return (None, None)
0975     def findLastInModGlobals(self, index, modID, comparer):
0976         return self._findLastIn(modID, self._modules, comparer)
0977     def findLastInModStreams(self, index, modID, comparer):
0978         return self._findLastIn(modID, self._modules, comparer)
0979
0980
0981
0982 def jsonTransition(type, id, sync, start, finish, isSrc=False):
0983     return {"type": type, "id": id, "sync": sync, "start": start*kMicroToSec, "finish": finish*kMicroToSec, "isSrc":isSrc}
0984
0985 def jsonModuleTransition(type, id, modID, callID, activity, start, finish=0):
0986     return {"type": type, "id": id, "mod": modID, "call": callID, "act": activity, "start": start*kMicroToSec, "finish": finish*kMicroToSec}
0987
0988 def startTime(x):
0989     return x["start"]
0990 def jsonInfo(parser):
0991     counter = Counter()
0992     if parser._moduleCentric:
0993         data = ModuleCentricContainers()
0994     else:
0995         data = Containers()
0996     for p in parser.processingSteps():
0997         p.jsonInfo(counter, data)
0998     #make sure everything is sorted
0999     for g in data.allGlobals():
1000         g.sort(key=startTime)
1001     final = {"transitions" : [] , "modules": [], "esModules": []}
1002     final["transitions"].append({ "name":"Global", "slots": []})
1003     globals = final["transitions"][-1]["slots"]
1004     for i, g in enumerate(data.allGlobals()):
1005         globals.append(g)
1006         if not parser._moduleCentric and not parser._frameworkOnly:
1007             if len(data._modGlobals) < i+1:
1008                 break
1009             for mod in data._modGlobals[i]:
1010                 globals.append(mod)
1011     for i,s in enumerate(data.allStreams()):
1012         final["transitions"].append({"name": f"Stream {i}", "slots":[]})
1013         stream = final["transitions"][-1]["slots"]
1014         stream.append(s)
1015         if not parser._moduleCentric and not parser._frameworkOnly:
1016             for mod in data._modStreams[i]:
1017                 stream.append(mod)
1018     if parser._moduleCentric:
1019         sourceSlot = data._modules[data._moduleID2Index(0)]
1020         modules = []
1021         for i,m in parser._moduleNames.items():
1022             modules.append({"name": f"{m}", "slots":[]})
1023             slots = modules[-1]["slots"]
1024             foundSlots = data._modules[data._moduleID2Index(i)]
1025             time = 0
1026             for s in foundSlots:
1027                 slots.append(s)
1028                 for t in s:
1029                     if t["act"] !=Activity.prefetch:
1030                         time += t["finish"]-t["start"]
1031             modules[-1]['time']=time
1032         for i,m in parser._esModuleNames.items():
1033             modules.append({"name": f"{m}", "slots":[]})
1034             slots = modules[-1]["slots"]
1035             foundSlots = data._modules[data._moduleID2Index(-1*i)]
1036             time = 0
1037             for s in foundSlots:
1038                 slots.append(s)
1039                 for t in s:
1040                     if t["act"] !=Activity.prefetch:
1041                         time += t["finish"]-t["start"]
1042             modules[-1]['time']=time
1043         modules.sort(key= lambda x : x['time'], reverse=True)
1044         final['transitions'].append({"name": "source", "slots":sourceSlot})
1045         for m in modules:
1046             final['transitions'].append(m)
1047
1048     if not parser._frameworkOnly:
1049         max = 0
1050         for k in parser._moduleNames.keys():
1051             if k > max:
1052                 max = k
1053
1054         final["modules"] =['']*(max+1)
1055         final["modules"][0] = 'source'
1056         for k,v in parser._moduleNames.items():
1057             final["modules"][k]=v
1058
1059         max = 0
1060         for k in parser._esModuleNames.keys():
1061             if k > max:
1062                 max = k
1063         final["esModules"] = ['']*(max+1)
1064         for k,v in parser._esModuleNames.items():
1065             final["esModules"][k] = v
1066     return final
1067
1068 #=======================================
1069 import unittest
1070
1071 class DummyFile(list):
1072     def __init__(self):
1073         super()
1074     def seek(self, i):
1075         pass
1076
1077 class TestModuleCommand(unittest.TestCase):
1078     def setUp(self):
1079         self.tracerFile = DummyFile()
1080         t = [0]
1081         def incr(t):
1082             t[0] += 1
1083             return t[0]
1084
1085         self.tracerFile.extend([
1086             '#R 1 Record',
1087             '#M 1 Module',
1088             '#N 1 ESModule',
1089              f'F {Phase.startTracing} 0 0 0 0 {incr(t)}',
1090              f'S {Phase.construction} 0 {incr(t)}',
1091              f's {Phase.construction} 0 {incr(t)}3',
1092              f'M {Phase.construction} 0 1 0 0 0 {incr(t)}',
1093              f'm {Phase.construction} 0 1 0 0 0 {incr(t)}',
1094              f'F {Phase.beginJob} 0 0 0 0 {incr(t)}',
1095              f'M {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
1096              f'm {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
1097              f'f {Phase.beginJob} 0 0 0 0 {incr(t)}',
1098              f'F {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1099              f'f {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
1100              f'S {Phase.getNextTransition} {incr(t)}',
1101              f's {Phase.getNextTransition} {incr(t)}',
1102              f'F {Phase.esSyncEnqueue} -1 1 0 0 {incr(t)}',
1103              f'F {Phase.esSync} -1 1 0 0 {incr(t)}',
1104              f'f {Phase.esSync} -1 1 0 0 {incr(t)}',
1105              f'S {Phase.globalBeginRun} 0 {incr(t)}',
1106              f's {Phase.globalBeginRun} 0 {incr(t)}',
1107              f'S {Phase.getNextTransition} {incr(t)}',
1108              f's {Phase.getNextTransition} {incr(t)}',
1109              f'F {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1110              f'P {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1111              f'p {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1112              f'M {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1113              f'm {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
1114              f'f {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
1115              f'F {Phase.esSyncEnqueue} -1 1 1 0 {incr(t)}',
1116              f'F {Phase.esSync} -1 1 1 0 {incr(t)}',
1117              f'f {Phase.esSync} -1 1 1 0 {incr(t)}',
1118              f'S {Phase.getNextTransition} {incr(t)}',
1119              f's {Phase.getNextTransition} {incr(t)}',
1120              f'F {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1121              f'M {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
1122              f'm {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
1123              f'f {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
1124              f'F {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1125              f'M {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
1126              f'm {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
1127              f'f {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
1128              f'S {Phase.globalBeginLumi} 0 {incr(t)}',
1129              f's {Phase.globalBeginLumi} 0 {incr(t)}',
1130              f'S {Phase.getNextTransition} {incr(t)}',
1131              f's {Phase.getNextTransition} {incr(t)}',
1132              f'F {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1133              f'P {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1134              f'p {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1135              f'M {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1136              f'm {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
1137              f'f {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
1138              f'F {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1139              f'f {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
1140              f'F {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1141              f'f {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
1142              f'S {Phase.Event} 0 {incr(t)}',
1143              f's {Phase.Event} 0 {incr(t)}',
1144              f'S {Phase.getNextTransition} {incr(t)}',
1145              f's {Phase.getNextTransition} {incr(t)}',
1146              f'F {Phase.Event} 0 1 1 1 {incr(t)}',
1147              f'S {Phase.Event} 1 {incr(t)}',
1148              f's {Phase.Event} 1 {incr(t)}',
1149              f'F {Phase.Event} 1 1 1 2 {incr(t)}',
1150              f'P {Phase.Event} 0 1 0 0 0 {incr(t)}',
1151              f'p {Phase.Event} 0 1 0 0 0 {incr(t)}',
1152              f'Q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1153              f'q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1154              f'N {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1155              f'n {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
1156              f'P {Phase.Event} 1 1 0 0 0 {incr(t)}',
1157              f'p {Phase.Event} 1 1 0 0 0 {incr(t)}',
1158              f'M {Phase.Event} 0 1 0 0 0 {incr(t)}',
1159              f'M {Phase.Event} 1 1 0 0 0 {incr(t)}',
1160              f'm {Phase.Event} 1 1 0 0 0 {incr(t)}',
1161              f'm {Phase.Event} 0 1 0 0 0 {incr(t)}',
1162              f'f {Phase.Event} 0 1 1 1 {incr(t)}',
1163              f'f {Phase.Event} 1 1 1 2 {incr(t)}'])
1164
1165         None
1166     def testContainers(self):
1167         c = Containers()
1168         c.indexedGlobal(2)
1169         self.assertEqual(len(c.allGlobals()), 3)
1170         c.indexedStream(2)
1171         self.assertEqual(len(c.allStreams()), 3)
1172         slot = c.findOpenSlotInModGlobals(2, 1)
1173         self.assertEqual(len(c._modGlobals),3)
1174         self.assertEqual(len(slot),0)
1175         slot.append({"start":1, "finish":0, "id":1})
1176         def testFind(item):
1177             return item["id"]==1
1178         item,s = c.findLastInModGlobals(2, 1, testFind)
1179         self.assertEqual(item["id"],1)
1180         self.assertEqual(slot,s)
1181         slot = c.findOpenSlotInModStreams(2, 1)
1182         self.assertEqual(len(c._modStreams),3)
1183         self.assertEqual(len(slot),0)
1184         slot.append({"start":1, "finish":0, "id":1})
1185         item,s = c.findLastInModStreams(2, 1, testFind)
1186         self.assertEqual(item["id"],1)
1187         self.assertEqual(slot,s)
1188     def testFrameworkOnly(self):
1189         parser = TracerCompactFileParser(self.tracerFile, True, False)
1190         j = jsonInfo(parser)
1191         #print(j)
1192         self.assertEqual(len(j["modules"]), 0)
1193         self.assertEqual(len(j["esModules"]), 0)
1194         self.assertEqual(len(j['transitions']), 3)
1195         self.assertEqual(j['transitions'][0]['name'], "Global")
1196         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1197         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1198         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1199         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1200         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1201         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1202         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1203         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1204     def testFull(self):
1205         parser = TracerCompactFileParser(self.tracerFile, False, False)
1206         j = jsonInfo(parser)
1207         #print(j)
1208         self.assertEqual(len(j["modules"]), 2)
1209         self.assertEqual(len(j["esModules"]), 2)
1210         self.assertEqual(len(j['transitions']), 3)
1211         self.assertEqual(j['transitions'][0]['name'], "Global")
1212         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1213         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1214         self.assertEqual(len(j["transitions"][0]["slots"]), 2)
1215         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1216         self.assertEqual(len(j["transitions"][0]["slots"][1]), 6)
1217         self.assertEqual(len(j["transitions"][1]["slots"]), 2)
1218         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1219         self.assertEqual(len(j["transitions"][1]["slots"][1]), 5)
1220         self.assertEqual(len(j["transitions"][2]["slots"]), 2)
1221         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1222         self.assertEqual(len(j["transitions"][2]["slots"][1]), 3)
1223     def testModuleCentric(self):
1224         parser = TracerCompactFileParser(self.tracerFile, False, True)
1225         j = jsonInfo(parser)
1226         #print(j)
1227         self.assertEqual(len(j["modules"]), 2)
1228         self.assertEqual(len(j["esModules"]), 2)
1229         self.assertEqual(len(j['transitions']), 6)
1230         self.assertEqual(j['transitions'][0]['name'], "Global")
1231         self.assertEqual(j['transitions'][1]['name'], "Stream 0")
1232         self.assertEqual(j['transitions'][2]['name'], "Stream 1")
1233         self.assertEqual(j['transitions'][3]['name'], "source")
1234         self.assertEqual(j['transitions'][4]['name'], "Module")
1235         self.assertEqual(j['transitions'][5]['name'], "ESModule")
1236         self.assertEqual(len(j["transitions"][0]["slots"]), 1)
1237         self.assertEqual(len(j["transitions"][0]["slots"][0]), 15)
1238         self.assertEqual(len(j["transitions"][1]["slots"]), 1)
1239         self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
1240         self.assertEqual(len(j["transitions"][2]["slots"]), 1)
1241         self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
1242         self.assertEqual(len(j["transitions"][4]["slots"]), 2)
1243         self.assertEqual(len(j["transitions"][4]["slots"][0]), 10)
1244         self.assertEqual(len(j["transitions"][4]["slots"][1]), 2)
1245         self.assertTrue(j["transitions"][4]["slots"][1][-1]['finish'] != 0.0)
1246         self.assertEqual(len(j["transitions"][5]["slots"]), 1)
1247         self.assertEqual(len(j["transitions"][5]["slots"][0]), 2)
1248
1249 def runTests():
1250     return unittest.main(argv=sys.argv[:1])
1251
1252 #=======================================
1253 if __name__=="__main__":
1254     import argparse
1255     import re
1256     import sys
1257
1258     # Program options
1259     parser = argparse.ArgumentParser(description='Convert a compact tracer file into human readable output.',
1260                                      formatter_class=argparse.RawDescriptionHelpFormatter,
1261                                      epilog=printHelp())
1262     parser.add_argument('filename',
1263                         type=argparse.FileType('r'), # open file
1264                         help='file to process')
1265     parser.add_argument('-f', '--frameworkOnly',
1266                         action='store_true',
1267                         help='''Output only the framework transitions, excluding the individual module transitions.''')
1268     parser.add_argument('-j', '--json',
1269                         action='store_true',
1270                         help='''Write output in json format.''' )
1271     parser.add_argument('-w', '--web',
1272                         action='store_true',
1273                         help='''Writes data.js file that can be used with the web based inspector. To use, copy directory ${CMSSW_RELEASE_BASE}/src/FWCore/Services/template/web to a web accessible area and move data.js into that directory.''')
1274     parser.add_argument('-m', '--module_centric',
1275                         action = 'store_true',
1276                         help='''For --json or --web, organize data by module instead of by global/stream.''' )
1277     parser.add_argument('-t', '--test',
1278                         action='store_true',
1279                         help='''Run internal tests.''')
1280
1281     args = parser.parse_args()
1282     if args.test:
1283         runTests()
1284     else :
1285         parser = TracerCompactFileParser(args.filename, args.frameworkOnly, args.module_centric)
1286         if args.json or args.web:
1287             j = json.dumps(jsonInfo(parser))
1288             if args.json:
1289                 print(j)
1290             if args.web:
1291                 f=open('data.json', 'w')
1292                 f.write(j)
1293                 f.close()
1294         else:
1295             textOutput(parser)