Performance/scripts/cmsSimplememchecker_parser.py

0001 #! /usr/bin/env python3
0002
0003
0004 from __future__ import print_function
0005 def manipulate_log(outdir,logfile_name,startevt):
0006
0007     import time
0008     import sys
0009     import ROOT
0010
0011     os.system('pwd')
0012
0013     # the fundamental structure: the key is the evt number the value is a list containing
0014     # VSIZE deltaVSIZE RSS deltaRSS
0015     data=[]
0016     values_set=('vsize','delta_vsize','rss','delta_rss')
0017     report=''
0018
0019     # open file and read it and fill the structure!
0020     logfile=open(logfile_name,'r')
0021     logfile_lines=logfile.readlines()
0022     logfile.close()
0023
0024     # we get the info we need!
0025     i=0
0026     max_rss=(0,0)
0027     parse_report=True
0028     while i < len(logfile_lines):
0029         line=logfile_lines[i]
0030         if '%MSG-w MemoryCheck:' in line:
0031             line=line[:-1] #no \n!
0032             line_content_list=line.split(' ')
0033             event_number=int(line_content_list[-1])
0034             if event_number<startevt:
0035                 i+=1
0036                 continue
0037             i+=1 # we inspect the following line
0038             line=logfile_lines[i]
0039             line=line[:-1] #no \n!
0040             line_content_list=line.split(' ')
0041             vsize=float(line_content_list[4])
0042             delta_vsize=float(line_content_list[5])
0043             rss=float(line_content_list[7])
0044             delta_rss=float(line_content_list[8])
0045
0046             data.append((event_number,{'vsize':vsize,
0047                                        'delta_vsize':delta_vsize,
0048                                        'rss':rss,
0049                                        'delta_rss':delta_rss}))
0050             # find maximum rss of the job
0051             if rss > max_rss[1]:
0052                 max_rss=(event_number, rss)
0053
0054         # include memory report
0055         elif parse_report and 'MemoryReport' in line:
0056             while 'TimeReport' not in line:
0057                 report += line.replace('MemoryReport', '')
0058                 i+=1
0059                 line = logfile_lines[i]
0060             parse_report=False
0061         i+=1
0062
0063     # print maximum rss for this job
0064     print('Maximum rss =', max_rss[1])
0065
0066     # skim the second entry when the event number is the same BUG!!!!!!!
0067     # i take elements in couples!
0068     new_data=[]
0069     if len(data)>2:
0070         if data[0][0]==data[1][0]:
0071             print('Two modules seem to have some output.\nCollapsing ...')
0072             i=0
0073             while True:
0074                 dataline1=data[i]
0075                 i+=1
0076                 dataline2=data[i]
0077                 new_eventnumber=dataline1[0]
0078                 new_vsize=dataline2[1]['vsize']
0079                 new_delta_vsize=dataline1[1]['delta_vsize']+dataline2[1]['delta_vsize']
0080                 new_rss=dataline2[1]['rss']
0081                 new_delta_rss=dataline1[1]['delta_rss']+dataline2[1]['delta_rss']
0082
0083                 new_data.append((new_eventnumber,{'vsize':new_vsize,
0084                                                   'delta_vsize':new_delta_vsize,
0085                                                   'rss':new_rss,
0086                                                   'delta_rss':new_delta_rss}))
0087                 i+=1
0088                 if i==len(data): break
0089
0090             data=new_data
0091             print('Collapsing: Done!')
0092
0093     npoints=len(data)
0094
0095     print('%s values read and stored ...' %npoints)
0096
0097
0098     # The Graphs
0099     __argv=sys.argv # trick for a strange behaviour of the TApp..
0100     sys.argv=sys.argv[:1]
0101     ROOT.gROOT.SetStyle("Plain") # style paranoia
0102     sys.argv=__argv
0103
0104     #Cannot use this option when the logfile includes
0105     #a large number of events... PyRoot seg-faults.
0106     #Set ROOT in batch mode to avoid canvases popping up!
0107     ROOT.gROOT.SetBatch(1)
0108
0109     # Save in file
0110     rootfilename='%s/graphs.root' %outdir
0111     myfile=ROOT.TFile(rootfilename,'RECREATE')
0112
0113     # dictionary of graphs!
0114     graph_dict={}
0115     for value in values_set:
0116         #graph_dict[value]
0117         graph=ROOT.TGraph(npoints)
0118         graph.SetMarkerStyle(8)
0119         graph.SetMarkerSize(.7)
0120         graph.SetMarkerColor(1)
0121         graph.SetLineWidth(3)
0122         graph.SetLineColor(2)
0123         graph.SetTitle(value)
0124         graph.SetName('%s_graph' %value)
0125
0126
0127         #fill the graphs
0128         point_counter=0
0129         for event_number,vals_dict in data:
0130             graph.SetPoint(point_counter,
0131                                        event_number,
0132                                        vals_dict[value])
0133             point_counter+=1
0134
0135         graph.GetXaxis().SetTitle("Event")
0136         last_event=data[-1][0]
0137         graph.GetXaxis().SetRangeUser(0,last_event+1)
0138         graph.GetYaxis().SetTitleOffset(1.3)
0139         graph.GetYaxis().SetTitle("MB")
0140
0141
0142
0143         #print the graphs as files :)
0144         mycanvas=ROOT.TCanvas('%s_canvas' %value)
0145         mycanvas.cd()
0146         graph.Draw("ALP")
0147
0148         mycanvas.Print("%s/%s_graph.png"%(outdir,value),"png")
0149
0150         # write it on file
0151         graph.Write()
0152         mycanvas.Write()
0153
0154     myfile.Close()
0155
0156     os.system('pwd')
0157
0158     # The html page!------------------------------------------------------------------------------
0159
0160     titlestring='<b>Report executed with release %s on %s.</b>\n<br>\n<hr>\n'\
0161                                    %(os.environ['CMSSW_VERSION'],time.asctime())
0162     #Introducing this if to catch the cmsRelvalreport.py use case of "reuse" of TimingReport
0163     #profile when doing the SimpleMemReport... otherwise the filename for the html
0164     #would be misleadingly TimingReport...
0165     if len(logfile_name)>16 and 'TimingReport.log' in logfile_name[-16:]:
0166         file_name=logfile_name[:-16]+"_SimpleMemReport"
0167     else:
0168         file_name=logfile_name[:-4]+"_SimpleMemReport"
0169     html_file_name='%s/%s.html' %(outdir,file_name)
0170     html_file=open(html_file_name,'w')
0171     html_file.write('<html>\n<body>\n'+\
0172                     titlestring)
0173     html_file.write('<table>\n'+\
0174                     '<tr>\n<td><img  src=vsize_graph.png></img></td>\n'+\
0175                     '<td><img src=rss_graph.png></img></td>\n</tr>\n'+\
0176                     '<tr>\n<td><img  src=delta_vsize_graph.png></img></td>\n'+\
0177                     '<td><img  src=delta_rss_graph.png></img></td>\n</tr>\n' +\
0178                     '</table>\n')
0179     html_file.write('<hr>\n<h1>Memory Checker Report</h1>\n<pre>\n' + report + '</pre>')
0180     html_file.write('\n</body>\n</html>')
0181     html_file.close()
0182
0183
0184 #################################################################################################
0185
0186 if __name__ == '__main__':
0187
0188     import optparse
0189     import os
0190
0191     # Here we define an option parser to handle commandline options..
0192     usage='simplememchecker_parser.py <options>'
0193     parser = optparse.OptionParser(usage)
0194     parser.add_option('-i', '--in_  profile',
0195                       help='The profile to manipulate' ,
0196                       default='',
0197                       dest='profile')
0198
0199     parser.add_option('-o', '--outdir',
0200                       help='The directory of the output' ,
0201                       default='',
0202                       dest='outdir')
0203
0204     parser.add_option('-n',
0205                       help='The event number from which we start. Default is 1.' ,
0206                       default='1',
0207                       dest='startevt')
0208
0209     (options,args) = parser.parse_args()
0210
0211     # Now some fault control..If an error is found we raise an exception
0212     if options.profile=='' or\
0213        options.outdir=='':
0214         raise('Please select a profile and an output dir!')
0215
0216     if not os.path.exists(options.profile) or\
0217        not os.path.exists(options.outdir):
0218         raise ('Outdir or input profile not present!')
0219
0220     try:
0221         startevt=int(options.startevt)
0222     except ValueError:
0223          print('Problems in convertng starting event value!')
0224
0225
0226     #launch the function!
0227     manipulate_log(options.outdir,options.profile,startevt)
0228
0229