Back to home page

Project CMSSW displayed by LXR

 
 

    


File indexing completed on 2021-08-10 22:50:02

0001 #! /usr/bin/env python3
0002 
0003     
0004 from __future__ import print_function
0005 def manipulate_log(outdir,logfile_name,startevt):
0006 
0007     import time
0008     import sys
0009     import ROOT       
0010     
0011     os.system('pwd')
0012     
0013     # the fundamental structure: the key is the evt number the value is a list containing
0014     # VSIZE deltaVSIZE RSS deltaRSS
0015     data=[]
0016     values_set=('vsize','delta_vsize','rss','delta_rss')
0017     report=''
0018 
0019     # open file and read it and fill the structure!
0020     logfile=open(logfile_name,'r')
0021     logfile_lines=logfile.readlines()
0022     logfile.close()
0023 
0024     # we get the info we need!
0025     i=0
0026     max_rss=(0,0)
0027     parse_report=True
0028     while i < len(logfile_lines):
0029         line=logfile_lines[i]
0030         if '%MSG-w MemoryCheck:' in line:
0031             line=line[:-1] #no \n!
0032             line_content_list=line.split(' ')
0033             event_number=int(line_content_list[-1])
0034             if event_number<startevt:
0035                 i+=1
0036                 continue
0037             i+=1 # we inspect the following line
0038             line=logfile_lines[i]
0039             line=line[:-1] #no \n!
0040             line_content_list=line.split(' ')
0041             vsize=float(line_content_list[4])
0042             delta_vsize=float(line_content_list[5])
0043             rss=float(line_content_list[7])
0044             delta_rss=float(line_content_list[8])
0045             
0046             data.append((event_number,{'vsize':vsize,
0047                                        'delta_vsize':delta_vsize,
0048                                        'rss':rss,
0049                                        'delta_rss':delta_rss}))
0050             # find maximum rss of the job
0051             if rss > max_rss[1]:
0052                 max_rss=(event_number, rss)
0053 
0054         # include memory report
0055         elif parse_report and 'MemoryReport' in line:
0056             while 'TimeReport' not in line:
0057                 report += line.replace('MemoryReport', '')
0058                 i+=1 
0059                 line = logfile_lines[i]
0060             parse_report=False
0061         i+=1
0062 
0063     # print maximum rss for this job
0064     print('Maximum rss =', max_rss[1])
0065                                     
0066     # skim the second entry when the event number is the same BUG!!!!!!!
0067     # i take elements in couples!
0068     new_data=[]
0069     if len(data)>2:
0070         if data[0][0]==data[1][0]:
0071             print('Two modules seem to have some output.\nCollapsing ...')
0072             i=0
0073             while True:
0074                 dataline1=data[i]
0075                 i+=1
0076                 dataline2=data[i]
0077                 new_eventnumber=dataline1[0]
0078                 new_vsize=dataline2[1]['vsize']
0079                 new_delta_vsize=dataline1[1]['delta_vsize']+dataline2[1]['delta_vsize']
0080                 new_rss=dataline2[1]['rss']
0081                 new_delta_rss=dataline1[1]['delta_rss']+dataline2[1]['delta_rss']
0082                 
0083                 new_data.append((new_eventnumber,{'vsize':new_vsize,
0084                                                   'delta_vsize':new_delta_vsize,
0085                                                   'rss':new_rss,
0086                                                   'delta_rss':new_delta_rss}))
0087                 i+=1
0088                 if i==len(data): break
0089                      
0090             data=new_data
0091             print('Collapsing: Done!')        
0092         
0093     npoints=len(data)
0094     
0095     print('%s values read and stored ...' %npoints)
0096 
0097             
0098     # The Graphs 
0099     __argv=sys.argv # trick for a strange behaviour of the TApp..
0100     sys.argv=sys.argv[:1]
0101     ROOT.gROOT.SetStyle("Plain") # style paranoia
0102     sys.argv=__argv
0103 
0104     #Cannot use this option when the logfile includes
0105     #a large number of events... PyRoot seg-faults.
0106     #Set ROOT in batch mode to avoid canvases popping up!
0107     ROOT.gROOT.SetBatch(1)
0108 
0109     # Save in file
0110     rootfilename='%s/graphs.root' %outdir
0111     myfile=ROOT.TFile(rootfilename,'RECREATE')    
0112        
0113     # dictionary of graphs!
0114     graph_dict={}
0115     for value in values_set:
0116         #graph_dict[value]
0117         graph=ROOT.TGraph(npoints)
0118         graph.SetMarkerStyle(8)
0119         graph.SetMarkerSize(.7)
0120         graph.SetMarkerColor(1)
0121         graph.SetLineWidth(3)
0122         graph.SetLineColor(2)        
0123         graph.SetTitle(value)
0124         graph.SetName('%s_graph' %value)
0125         
0126     
0127         #fill the graphs
0128         point_counter=0
0129         for event_number,vals_dict in data:
0130             graph.SetPoint(point_counter,
0131                                        event_number,
0132                                        vals_dict[value])
0133             point_counter+=1
0134         
0135         graph.GetXaxis().SetTitle("Event")
0136         last_event=data[-1][0]
0137         graph.GetXaxis().SetRangeUser(0,last_event+1)
0138         graph.GetYaxis().SetTitleOffset(1.3)
0139         graph.GetYaxis().SetTitle("MB")
0140                           
0141         
0142             
0143         #print the graphs as files :)
0144         mycanvas=ROOT.TCanvas('%s_canvas' %value)
0145         mycanvas.cd()
0146         graph.Draw("ALP")
0147     
0148         mycanvas.Print("%s/%s_graph.png"%(outdir,value),"png")
0149         
0150         # write it on file
0151         graph.Write()
0152         mycanvas.Write()
0153         
0154     myfile.Close() 
0155         
0156     os.system('pwd') 
0157                 
0158     # The html page!------------------------------------------------------------------------------
0159     
0160     titlestring='<b>Report executed with release %s on %s.</b>\n<br>\n<hr>\n'\
0161                                    %(os.environ['CMSSW_VERSION'],time.asctime())
0162     #Introducing this if to catch the cmsRelvalreport.py use case of "reuse" of TimingReport
0163     #profile when doing the SimpleMemReport... otherwise the filename for the html
0164     #would be misleadingly TimingReport...
0165     if len(logfile_name)>16 and 'TimingReport.log' in logfile_name[-16:]:
0166         file_name=logfile_name[:-16]+"_SimpleMemReport"
0167     else:
0168         file_name=logfile_name[:-4]+"_SimpleMemReport"
0169     html_file_name='%s/%s.html' %(outdir,file_name)
0170     html_file=open(html_file_name,'w')
0171     html_file.write('<html>\n<body>\n'+\
0172                     titlestring)
0173     html_file.write('<table>\n'+\
0174                     '<tr>\n<td><img  src=vsize_graph.png></img></td>\n'+\
0175                     '<td><img src=rss_graph.png></img></td>\n</tr>\n'+\
0176                     '<tr>\n<td><img  src=delta_vsize_graph.png></img></td>\n'+\
0177                     '<td><img  src=delta_rss_graph.png></img></td>\n</tr>\n' +\
0178                     '</table>\n')
0179     html_file.write('<hr>\n<h1>Memory Checker Report</h1>\n<pre>\n' + report + '</pre>')
0180     html_file.write('\n</body>\n</html>')
0181     html_file.close()    
0182     
0183     
0184 #################################################################################################    
0185         
0186 if __name__ == '__main__':
0187     
0188     import optparse
0189     import os
0190     
0191     # Here we define an option parser to handle commandline options..
0192     usage='simplememchecker_parser.py <options>'
0193     parser = optparse.OptionParser(usage)
0194     parser.add_option('-i', '--in_  profile',
0195                       help='The profile to manipulate' ,
0196                       default='',
0197                       dest='profile')
0198                       
0199     parser.add_option('-o', '--outdir',
0200                       help='The directory of the output' ,
0201                       default='',
0202                       dest='outdir')
0203 
0204     parser.add_option('-n', 
0205                       help='The event number from which we start. Default is 1.' ,
0206                       default='1',
0207                       dest='startevt')                      
0208                                             
0209     (options,args) = parser.parse_args()
0210     
0211     # Now some fault control..If an error is found we raise an exception
0212     if options.profile=='' or\
0213        options.outdir=='':
0214         raise('Please select a profile and an output dir!')
0215     
0216     if not os.path.exists(options.profile) or\
0217        not os.path.exists(options.outdir):
0218         raise ('Outdir or input profile not present!')
0219     
0220     try:
0221         startevt=int(options.startevt)        
0222     except ValueError:
0223          print('Problems in convertng starting event value!')
0224          
0225             
0226     #launch the function!
0227     manipulate_log(options.outdir,options.profile,startevt)
0228     
0229