Performance/scripts/cmsSimplememchecker_parser.py

0001 #! /usr/bin/env python3
0002
0003
0004 def manipulate_log(outdir,logfile_name,startevt):
0005
0006     import time
0007     import sys
0008     import ROOT
0009
0010     os.system('pwd')
0011
0012     # the fundamental structure: the key is the evt number the value is a list containing
0013     # VSIZE deltaVSIZE RSS deltaRSS
0014     data=[]
0015     values_set=('vsize','delta_vsize','rss','delta_rss')
0016     report=''
0017
0018     # open file and read it and fill the structure!
0019     logfile=open(logfile_name,'r')
0020     logfile_lines=logfile.readlines()
0021     logfile.close()
0022
0023     # we get the info we need!
0024     i=0
0025     max_rss=(0,0)
0026     parse_report=True
0027     while i < len(logfile_lines):
0028         line=logfile_lines[i]
0029         if '%MSG-w MemoryCheck:' in line:
0030             line=line[:-1] #no \n!
0031             line_content_list=line.split(' ')
0032             event_number=int(line_content_list[-1])
0033             if event_number<startevt:
0034                 i+=1
0035                 continue
0036             i+=1 # we inspect the following line
0037             line=logfile_lines[i]
0038             line=line[:-1] #no \n!
0039             line_content_list=line.split(' ')
0040             vsize=float(line_content_list[4])
0041             delta_vsize=float(line_content_list[5])
0042             rss=float(line_content_list[7])
0043             delta_rss=float(line_content_list[8])
0044
0045             data.append((event_number,{'vsize':vsize,
0046                                        'delta_vsize':delta_vsize,
0047                                        'rss':rss,
0048                                        'delta_rss':delta_rss}))
0049             # find maximum rss of the job
0050             if rss > max_rss[1]:
0051                 max_rss=(event_number, rss)
0052
0053         # include memory report
0054         elif parse_report and 'MemoryReport' in line:
0055             while 'TimeReport' not in line:
0056                 report += line.replace('MemoryReport', '')
0057                 i+=1
0058                 line = logfile_lines[i]
0059             parse_report=False
0060         i+=1
0061
0062     # print maximum rss for this job
0063     print('Maximum rss =', max_rss[1])
0064
0065     # skim the second entry when the event number is the same BUG!!!!!!!
0066     # i take elements in couples!
0067     new_data=[]
0068     if len(data)>2:
0069         if data[0][0]==data[1][0]:
0070             print('Two modules seem to have some output.\nCollapsing ...')
0071             i=0
0072             while True:
0073                 dataline1=data[i]
0074                 i+=1
0075                 dataline2=data[i]
0076                 new_eventnumber=dataline1[0]
0077                 new_vsize=dataline2[1]['vsize']
0078                 new_delta_vsize=dataline1[1]['delta_vsize']+dataline2[1]['delta_vsize']
0079                 new_rss=dataline2[1]['rss']
0080                 new_delta_rss=dataline1[1]['delta_rss']+dataline2[1]['delta_rss']
0081
0082                 new_data.append((new_eventnumber,{'vsize':new_vsize,
0083                                                   'delta_vsize':new_delta_vsize,
0084                                                   'rss':new_rss,
0085                                                   'delta_rss':new_delta_rss}))
0086                 i+=1
0087                 if i==len(data): break
0088
0089             data=new_data
0090             print('Collapsing: Done!')
0091
0092     npoints=len(data)
0093
0094     print('%s values read and stored ...' %npoints)
0095
0096
0097     # The Graphs
0098     __argv=sys.argv # trick for a strange behaviour of the TApp..
0099     sys.argv=sys.argv[:1]
0100     ROOT.gROOT.SetStyle("Plain") # style paranoia
0101     sys.argv=__argv
0102
0103     #Cannot use this option when the logfile includes
0104     #a large number of events... PyRoot seg-faults.
0105     #Set ROOT in batch mode to avoid canvases popping up!
0106     ROOT.gROOT.SetBatch(1)
0107
0108     # Save in file
0109     rootfilename='%s/graphs.root' %outdir
0110     myfile=ROOT.TFile(rootfilename,'RECREATE')
0111
0112     # dictionary of graphs!
0113     graph_dict={}
0114     for value in values_set:
0115         #graph_dict[value]
0116         graph=ROOT.TGraph(npoints)
0117         graph.SetMarkerStyle(8)
0118         graph.SetMarkerSize(.7)
0119         graph.SetMarkerColor(1)
0120         graph.SetLineWidth(3)
0121         graph.SetLineColor(2)
0122         graph.SetTitle(value)
0123         graph.SetName('%s_graph' %value)
0124
0125
0126         #fill the graphs
0127         point_counter=0
0128         for event_number,vals_dict in data:
0129             graph.SetPoint(point_counter,
0130                                        event_number,
0131                                        vals_dict[value])
0132             point_counter+=1
0133
0134         graph.GetXaxis().SetTitle("Event")
0135         last_event=data[-1][0]
0136         graph.GetXaxis().SetRangeUser(0,last_event+1)
0137         graph.GetYaxis().SetTitleOffset(1.3)
0138         graph.GetYaxis().SetTitle("MB")
0139
0140
0141
0142         #print the graphs as files :)
0143         mycanvas=ROOT.TCanvas('%s_canvas' %value)
0144         mycanvas.cd()
0145         graph.Draw("ALP")
0146
0147         mycanvas.Print("%s/%s_graph.png"%(outdir,value),"png")
0148
0149         # write it on file
0150         graph.Write()
0151         mycanvas.Write()
0152
0153     myfile.Close()
0154
0155     os.system('pwd')
0156
0157     # The html page!------------------------------------------------------------------------------
0158
0159     titlestring='<b>Report executed with release %s on %s.</b>\n<br>\n<hr>\n'\
0160                                    %(os.environ['CMSSW_VERSION'],time.asctime())
0161     #Introducing this if to catch the cmsRelvalreport.py use case of "reuse" of TimingReport
0162     #profile when doing the SimpleMemReport... otherwise the filename for the html
0163     #would be misleadingly TimingReport...
0164     if len(logfile_name)>16 and 'TimingReport.log' in logfile_name[-16:]:
0165         file_name=logfile_name[:-16]+"_SimpleMemReport"
0166     else:
0167         file_name=logfile_name[:-4]+"_SimpleMemReport"
0168     html_file_name='%s/%s.html' %(outdir,file_name)
0169     html_file=open(html_file_name,'w')
0170     html_file.write('<html>\n<body>\n'+\
0171                     titlestring)
0172     html_file.write('<table>\n'+\
0173                     '<tr>\n<td><img  src=vsize_graph.png></img></td>\n'+\
0174                     '<td><img src=rss_graph.png></img></td>\n</tr>\n'+\
0175                     '<tr>\n<td><img  src=delta_vsize_graph.png></img></td>\n'+\
0176                     '<td><img  src=delta_rss_graph.png></img></td>\n</tr>\n' +\
0177                     '</table>\n')
0178     html_file.write('<hr>\n<h1>Memory Checker Report</h1>\n<pre>\n' + report + '</pre>')
0179     html_file.write('\n</body>\n</html>')
0180     html_file.close()
0181
0182
0183 #################################################################################################
0184
0185 if __name__ == '__main__':
0186
0187     import optparse
0188     import os
0189
0190     # Here we define an option parser to handle commandline options..
0191     usage='simplememchecker_parser.py <options>'
0192     parser = optparse.OptionParser(usage)
0193     parser.add_option('-i', '--in_  profile',
0194                       help='The profile to manipulate' ,
0195                       default='',
0196                       dest='profile')
0197
0198     parser.add_option('-o', '--outdir',
0199                       help='The directory of the output' ,
0200                       default='',
0201                       dest='outdir')
0202
0203     parser.add_option('-n',
0204                       help='The event number from which we start. Default is 1.' ,
0205                       default='1',
0206                       dest='startevt')
0207
0208     (options,args) = parser.parse_args()
0209
0210     # Now some fault control..If an error is found we raise an exception
0211     if options.profile=='' or\
0212        options.outdir=='':
0213         raise('Please select a profile and an output dir!')
0214
0215     if not os.path.exists(options.profile) or\
0216        not os.path.exists(options.outdir):
0217         raise ('Outdir or input profile not present!')
0218
0219     try:
0220         startevt=int(options.startevt)
0221     except ValueError:
0222          print('Problems in convertng starting event value!')
0223
0224
0225     #launch the function!
0226     manipulate_log(options.outdir,options.profile,startevt)
0227
0228