Tools/scripts/timingPdfMaker.py

0001 #! /usr/bin/env python
0002
0003 #################################################################
0004 # timingPdfMaker.py - python script
0005 # Created by Alejandro Gomez
0006 # For more info, please check:
0007 # https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary
0008 # or by email: alejandro.gomez@cern.ch
0009
0010 # This python script creates managebles pdf for HLT Timing Studies.
0011 # It saves some.png files from a former root file, creates a tex file,
0012 # compiles it twice for better results and remove all the temp files.
0013 # Please check the usage function to see how it works.
0014 ###############################################################
0015
0016 #############################################
0017 ## Imports
0018 ############################################
0019
0020 import os, glob
0021 import os.path
0022 import operator
0023 import subprocess
0024 import sys, getopt
0025 sys.argv.append('-b')
0026 from ROOT import gROOT, gStyle, TFile, TCanvas
0027 import cStringIO
0028
0029
0030 gROOT.SetStyle("Plain")
0031 gStyle.SetOptStat(111111)
0032 gStyle.SetHistFillColor(kBlue)
0033
0034 ###############################################################
0035 def usage():
0036 ###############################################################
0037     print("\nThis is the usage function\n")
0038     print('Usage: '+sys.argv[0]+' -i <file1> [option]')
0039     print('e.g.:  '+sys.argv[0]+' -i outputTiming.root -t')
0040     print('e.g.:  '+sys.argv[0]+' -i outputTiming.root -s HLT_Jet300_v5\n')
0041
0042     print('\n-----Options-----')
0043     print(' -i          Input File')
0044     print(' -o          Output File (optional)')
0045     print(' -t          For only main time info per event. It will take less than 1 min.')
0046     print(' -p          For path time info. It will take approx 25 min.')
0047     print(' -m          For module time info. It will take approx 25 min.')
0048     print(' -s Path_Name            (For an specific path). Ti will take less than 1 min.')
0049     print('\n For -p or -m option, the process needs like 200 Mb in disk space,')
0050     print(' but please dont be afraid. Before the process ends, all the temporal files')
0051     print(' will be erased.')
0052
0053
0054 ###############################################################
0055 def maininfo(infile, outfile):
0056     ''' Creates main info tex file'''
0057 ##############################################################
0058     texpreamble = ['\documentclass[10pt,a5paper,landscape]{report}\n',
0059                    '\\usepackage{graphicx}\n',
0060                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0061                     '\\usepackage[linktocpage]{hyperref}\n',
0062                     '\hypersetup{backref, colorlinks=true}\n',
0063                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Main Info }\n',
0064                     '\\author{\Large{CMS Experiment}}\n',
0065                     '\date{\\today}\n',
0066                     '\\begin{document}\n',
0067                     '\maketitle\n',
0068                     '\\newpage\n',
0069                     '\clearpage\n'
0070                     '\\tableofcontents\n',
0071                     '\\newpage\n',
0072                     '\\newpage \chapter{Total time for all modules per event} \\newpage \centering \includegraphics[scale=0.6]{totalTimetemp.png}\n']
0073
0074     names1 = {}
0075     names4 = {}
0076     specific = {}
0077     file1 = TFile(infile,'read')
0078     for k in file1.GetListOfKeys():
0079         allnames = k.ReadObj().GetName()
0080         if 'pathTime_' in allnames:
0081             pathname = '_'.join(allnames.split('_')[1:])
0082             if not pathname in names1:
0083                 names1[pathname] = k.ReadObj().GetMean()
0084         elif 'incPathTime_' in allnames:
0085             pathname = '_'.join(allnames.split('_')[1:])
0086             if not pathname in names4:
0087                 names4[pathname] = k.ReadObj().GetMean()
0088
0089     names2 = dict(sorted(names1.items(), key=operator.itemgetter(1),reverse=True)[:10])
0090     names3 = sorted(names2, key=names2.get, reverse=True)
0091     names5 = dict(sorted(names4.items(), key=operator.itemgetter(1),reverse=True)[:10])
0092     names6 = sorted(names5, key=names5.get, reverse=True)
0093
0094     texfile = open(outfile+'-main.tex', 'w')
0095     texfile.writelines(texpreamble)
0096     if os.path.exists(infile.replace('.root','')+'-summary.txt'):
0097         excludefile = open(infile.replace('.root','')+'-summary.txt', 'r')
0098         texfile.write('\\newpage \section{Summary} \n \\begin{verbatim} \n')
0099         for line in excludefile.readlines():
0100             texfile.write(line+'\n')
0101         excludefile.close()
0102         texfile.write('\end{verbatim}')
0103     texfile.write('\\newpage \\chapter{10 Slowest Paths}\n')
0104     texfile.write('\section{Average module (in path) time}\n')
0105     for path in names3:
0106         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathTimeSummary'+ path.replace('_','') +'temp.png}\n')
0107         get_plot2(infile,'moduleInPathTimeSummary_'+path)
0108     texfile.write('\section{Average module (in path) running time}\n')
0109     for path in names3:
0110         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathScaledTimeSummary'+ path.replace('_','') +'temp.png}\n')
0111         get_plot2(infile,'moduleInPathScaledTimeSummary_'+path)
0112     texfile.write('\section{Per event time for path}\n')
0113     for path in names3:
0114         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{pathTime'+ path.replace('_','') +'temp.png}\n')
0115         get_plot1(infile,'pathTime_'+path)
0116     texfile.write('\section{Per event incremental time for path}\n')
0117     for path in names6:
0118         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{incPathTime'+ path.replace('_','') +'temp.png}\n')
0119         get_plot1(infile,'incPathTime_'+path)
0120     texfile.write('\end{document}')
0121     texfile.close()
0122
0123     texfile.close()
0124
0125     get_plot1(infile,'totalTime')
0126
0127
0128
0129 #################################################
0130 def pathsinfo(infile,outfile):
0131     '''Create the paths info tex file'''
0132 ################################################
0133
0134     texpreamble = ['\documentclass[10pt,a5paper,landscape]{book}\n',
0135                     '\\usepackage{graphicx}\n',
0136                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0137                     '\\usepackage[linktocpage]{hyperref}\n',
0138                     '\\usepackage[titles]{tocloft}\n'
0139                     '\hypersetup{backref, colorlinks=true}\n',
0140                     '\setlength{\cftsecnumwidth}{4em}\n'
0141                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Paths Info }\n',
0142                     '\\author{\Large{CMS Experiment}}\n',
0143                     '\date{\\today}\n',
0144                     '\\begin{document}\n',
0145                     '\maketitle\n',
0146                     '\\newpage\n',
0147                     '\\tableofcontents\n',
0148                     '\\newpage\n']
0149
0150     names1 = {}
0151     file = TFile(infile,'read')
0152     for k in file.GetListOfKeys():
0153         allnames= k.ReadObj().GetName()
0154         mean = 1
0155         if 'moduleInPathScaledTime_' in allnames:
0156             pathname = '_'.join(allnames.split('_')[1:-1])
0157             if not pathname in names1:
0158                 names1[pathname] = mean
0159     names = sorted(names1.keys())
0160
0161     texfile = open(outfile+'-paths.tex', 'w')
0162     texfile.writelines(texpreamble)
0163
0164     texfile.write('\\chapter{Average module (in path) time}\n')
0165     for path in names:
0166         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathTimeSummary'+ path.replace('_','') +'temp.png}\n')
0167         get_plot2(infile,'moduleInPathTimeSummary_'+path)
0168     texfile.write('\\chapter{Average module (in path) running time}\n')
0169     for path in names:
0170         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathScaledTimeSummary'+ path.replace('_','') +'temp.png}\n')
0171         get_plot2(infile,'moduleInPathScaledTimeSummary_'+path)
0172     texfile.write('\\chapter{Failing module (by path)}')
0173     for path in names:
0174         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{failedModule'+ path.replace('_','') +'temp.png}\n')
0175         get_plot2(infile,'failedModule_'+path)
0176     texfile.write('\\chapter{Per event time for path}\n')
0177     for path in names:
0178         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{pathTime'+ path.replace('_','') +'temp.png}\n')
0179         get_plot1(infile,'pathTime_'+path)
0180     texfile.write('\\chapter{Per event incremental time for path}\n')
0181     for path in names:
0182         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{incPathTime'+ path.replace('_','') +'temp.png}\n')
0183         get_plot1(infile,'incPathTime_'+path)
0184
0185     texfile.write('\end{document}')
0186     texfile.close()
0187
0188
0189 #################################################
0190 def moduleinfo(infile,outfile):
0191     '''Create the paths info tex file'''
0192 ################################################
0193
0194     texpreamble = ['\documentclass[10pt,a5paper,landscape]{report}\n',
0195                     '\\usepackage{graphicx}\n',
0196                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0197                     '\\usepackage[linktocpage]{hyperref}\n',
0198                     '\hypersetup{backref, colorlinks=true}\n',
0199                     '\\usepackage[titles]{tocloft}\n'
0200                     '\setlength{\cftsecnumwidth}{4em}\n'
0201                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Modules Info }\n',
0202                     '\\author{\Large{CMS Experiment}}\n',
0203                     '\date{\\today}\n',
0204                     '\\begin{document}\n',
0205                     '\maketitle\n',
0206                     '\\newpage\n',
0207                     '\\tableofcontents\n',
0208                     '\\newpage\n']
0209
0210     names1 = {}
0211     file = TFile(infile,'read')
0212     for k in file.GetListOfKeys():
0213         allnames = k.ReadObj().GetName()
0214         mean = 1
0215         if 'moduleTime_' in allnames:
0216             modname = ''.join(allnames.split('_')[1:])
0217             if not (('!' in modname) or ('-' in modname)):
0218                 if not modname in names1:
0219                     names1[modname] = mean
0220     names = sorted(names1.keys())
0221
0222     texfile1 = open(outfile+'-modules.tex', 'w')
0223     texfile1.writelines(texpreamble)
0224
0225     texfile1.write('\\chapter{Time per event for module} \n \\newpage')
0226     for modules in names:
0227         texfile1.write('\section{'+modules+'}')
0228         texfile1.write('\centering \includegraphics[scale=0.4]{moduleTime'+ modules +'temp.png}\n')
0229         get_plot1(infile,'moduleTime_'+modules)
0230     texfile1.write('\end{document}')
0231     texfile1.close()
0232
0233     texfile2 = open(outfile+'-runningModules.tex', 'w')
0234     texfile2.writelines(texpreamble)
0235     texfile2.write('\\chapter{Running Time per event for module} \n \\newpage')
0236     for modules in names:
0237         texfile2.write('\section{'+modules+'}')
0238         texfile2.write('\centering \includegraphics[scale=0.45]{moduleScaledTime'+modules+'temp.png}\n')
0239         get_plot1(infile,'moduleScaledTime_'+ modules)
0240
0241     texfile2.write('\end{document}')
0242     texfile2.close()
0243
0244 ###############################################################
0245 def specificpathinfo(infile, outfile, path):
0246     ''' Creates an specific path info tex file'''
0247 ##############################################################
0248     texpreamble = ['\documentclass[10pt,a5paper,landscape]{report}\n',
0249                     '\\usepackage{graphicx}\n',
0250                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0251                     '\\usepackage[linktocpage]{hyperref}\n',
0252                     '\\usepackage[titles]{tocloft}\n'
0253                     '\hypersetup{backref, colorlinks=true}\n',
0254                     '\setlength{\cftsubsecnumwidth}{4em}\n'
0255                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Main Info + ' + path.replace('_','\_') +' info }\n',
0256                     '\\author{\Large{CMS Experiment}}\n',
0257                     '\date{\\today}\n',
0258                     '\\begin{document}\n',
0259                     '\maketitle\n',
0260                     '\\tableofcontents \n'
0261                     '\\newpage\n \\chapter{Main Info} \n',
0262                     '\\newpage \centering \section{Total time for all modules per event} \includegraphics[scale=0.6]{totalTimetemp.png}\n']
0263
0264     texfile = open(outfile+'-'+path+'.tex', 'w')
0265     texfile.writelines(texpreamble)
0266     get_plot1(infile,'totalTime')
0267
0268     names = {}
0269     file1 = TFile(infile,'read')
0270     for k in file1.GetListOfKeys():
0271         allnames = k.ReadObj().GetName()
0272         if 'moduleInPathScaledTime_' in allnames:
0273             pathname = '_'.join(allnames.split('_')[1:-1])
0274             if not pathname in names:
0275                 names[pathname] = {}
0276     for pathnames in names.keys():
0277         histo = file1.Get('moduleInPathTimeSummary_'+pathnames)
0278         for bin in range(histo.GetNbinsX()):
0279             label = histo.GetXaxis().GetBinLabel(bin+1)
0280             names[pathnames][bin+1] = label
0281
0282     for pathname in names:
0283         if path in pathname:
0284             texfile.write('\chapter{' + path.replace('_','\_')+ ' Info} \n')
0285             texfile.write('\\newpage \section{Average module in '+ path.replace('_','\_') +' time} \centering \includegraphics[scale=0.35]{moduleInPathTimeSummary'+ path.replace('_','') +'temp.png}\n')
0286             get_plot2(infile,'moduleInPathTimeSummary_'+path)
0287             texfile.write('\\newpage \section{Average module in '+ path.replace('_','\_') +' running time} \centering \includegraphics[scale=0.35]{moduleInPathScaledTimeSummary'+ path.replace('_','') +'temp.png}\n')
0288             get_plot2(infile,'moduleInPathScaledTimeSummary_'+path)
0289             texfile.write('\\newpage \section{Per event time for '+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{pathTime'+ path.replace('_','') +'temp.png}\n')
0290             get_plot1(infile,'pathTime_'+path)
0291             texfile.write('\\newpage \section{Per event incremental time for '+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{incPathTime'+ path.replace('_','') +'temp.png}\n')
0292             get_plot1(infile,'incPathTime_'+path)
0293             texfile.write('\section{Running time per event for '+path.replace('_','\_')+'}')
0294             for modules in names[path].values():
0295                 texfile.write('\subsection{'+ modules +'} \centering \includegraphics[scale=0.6]{moduleInPathScaledTime'+ path.replace('_','') + modules +'temp.png}\n')
0296                 get_plot1(infile,'moduleInPathScaledTime_'+ path +'_'+ modules)
0297     texfile.write('\\end{document}')
0298     texfile.close()
0299
0300
0301
0302
0303 ###################################################################
0304 def get_plot1(file,allnames):
0305     ''' Function to create the plot and save it as.png file '''
0306 ###################################################################
0307     file = TFile(file,'read')
0308     histo = file.Get(allnames)
0309     can = TCanvas('can', '', 800,600)
0310     can.cd()
0311     histo.UseCurrentStyle()
0312     histo.Draw()
0313     can.SetBorderMode(0)
0314     can.SetFillColor(kWhite)
0315     can.SaveAs(allnames.replace('_','')+'temp.png')
0316     del can
0317
0318
0319
0320 ###################################################################
0321 def get_plot2(infile,allnames):
0322     ''' Function to create the plot and save it as.png file '''
0323 ###################################################################
0324     file1 = TFile(infile,'read')
0325     histo = file1.Get(allnames)
0326     can = TCanvas('can', '', 1600,1000)
0327     can.cd()
0328     histo.UseCurrentStyle()
0329     histo.Draw()
0330     if histo.GetNbinsX() > 50:
0331         histo.GetXaxis().SetLabelSize(0.02)
0332     else:
0333         histo.GetXaxis().SetLabelSize(0.03)
0334     can.SetBorderMode(0)
0335     can.SetBorderSize(0)
0336     can.SetFillColor(kWhite)
0337     can.SetBottomMargin(0.4)
0338     can.SaveAs(allnames.replace('_','')+'temp.png')
0339     del can
0340
0341 ###############################################################
0342 def main(argv):
0343 ###############################################################
0344     print("\nPython script that creates Timing Summary pdf files")
0345     print("For more info, please contact Alejandro Gomez")
0346     print("email: alejandro.gomez@cern.ch\n")
0347
0348     infile = None
0349     outfile = None
0350     path = None
0351     call_maininfo = False
0352     call_pathsinfo = False
0353     call_modulesinfo = False
0354     call_specificinfo = False
0355     try:
0356         opts, args = getopt.getopt(argv, 'hi:o:tbpms:', ['help', 'input=', 'output='])
0357         if not opts:
0358             print('No options supplied')
0359             usage()
0360     except getopt.GetoptError as e:
0361         print(e)
0362         usage()
0363         sys.exit(2)
0364     for opt, arg in opts:
0365         if opt in ('h', '--help'):
0366             usage()
0367             sys.exit(2)
0368         elif opt == '-b':
0369             print('Running in batch mode')
0370         elif opt in ('-i', '--input'):
0371             infile = arg
0372             outfile = infile.replace('.root','')
0373         elif opt in ('-o', '--output'):
0374             outfile = arg
0375         elif opt == '-t':
0376             call_maininfo = True
0377         elif opt == '-p':
0378             call_pathsinfo = True
0379         elif opt == '-m':
0380             call_modulesinfo = True
0381         elif opt == '-s':
0382             path = arg
0383             call_specificinfo = True
0384         else:
0385             usage()
0386             sys.exit(2)
0387
0388
0389     if call_maininfo:
0390         print('Creating the Main Info Timing Summary pdf')
0391         print('Creating plots...')
0392         maininfo(infile,outfile)
0393         print('Compiling tex file......')
0394         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-main.tex'])
0395         print('Verifing......')
0396         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-main.tex'])    #twice for better compilation
0397         print('Removing temp files.........')
0398         os.remove(outfile+'-main.aux')
0399         os.remove(outfile+'-main.log')
0400         os.remove(outfile+'-main.out')
0401         os.remove(outfile+'-main.tex')
0402         os.remove(outfile+'-main.toc')
0403         for filename in glob.glob('*temp.png'):
0404             os.remove(filename)
0405         print('{0}-main.pdf is done'.format(outfile))
0406
0407     if call_pathsinfo:
0408         print('Creating the Paths Info Timing Summary pdf')
0409         print('This process takes awhile... please be patient')
0410         print('Creating plots...')
0411         pathsinfo(infile,outfile)
0412         print('Compiling tex file......')
0413         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-paths.tex'])
0414         print('Verifing......')
0415         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-paths.tex'])    #twice for better compilation
0416         print('Removing temp files.........')
0417         os.remove(outfile+'-paths.aux')
0418         os.remove(outfile+'-paths.log')
0419         os.remove(outfile+'-paths.out')
0420         os.remove(outfile+'-paths.tex')
0421         os.remove(outfile+'-paths.toc')
0422         for filename in glob.glob('*temp.png'):
0423             os.remove(filename)
0424         print('{0}-paths.pdf is done'.format(outfile))
0425
0426     if call_modulesinfo:
0427         print('Creating the Modules Info Timing Summary pdf')
0428         print('This process takes awhile... please be patient')
0429         print('Creating plots...')
0430         moduleinfo(infile,outfile)
0431         print('Compiling tex file......')
0432         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-modules.tex'])
0433         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-runningModules.tex'])
0434         print('Verifing......')
0435         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-modules.tex'])    #twice for better compilation
0436         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-runningModules.tex'])    #twice for better compilation
0437         print('Removing temp files.........')
0438         os.remove(outfile+'-modules.aux')
0439         os.remove(outfile+'-runningModules.aux')
0440         os.remove(outfile+'-modules.log')
0441         os.remove(outfile+'-runningModules.log')
0442         os.remove(outfile+'-modules.out')
0443         os.remove(outfile+'-runningModules.out')
0444         os.remove(outfile+'-modules.tex')
0445         os.remove(outfile+'-runningModules.tex')
0446         os.remove(outfile+'-modules.toc')
0447         os.remove(outfile+'-runningModules.toc')
0448         for filename in glob.glob('*temp.png'):
0449             os.remove(filename)
0450         print('{0}-modules.pdf is done'.format(outfile))
0451         print('{0}-runningModules.pdf is done'.format(outfile))
0452
0453     if call_specificinfo:
0454         print('Creating the Main Info + '+ path +' Timing Summary pdf')
0455         print('This process takes awhile... please be patient')
0456         print('Creating plots...')
0457         specificpathinfo(infile,outfile,path)
0458         print('Compiling tex file......')
0459         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-'+path+'.tex'])
0460         print('Verifing......')
0461         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-'+path+'.tex'])    #twice for better compilation
0462         print('Removing temp files.........')
0463         os.remove(outfile+'-'+path+'.aux')
0464         os.remove(outfile+'-'+path+'.log')
0465         os.remove(outfile+'-'+path+'.out')
0466         os.remove(outfile+'-'+path+'.tex')
0467         os.remove(outfile+'-'+path+'.toc')
0468         for filename in glob.glob('*temp.png'):
0469             os.remove(filename)
0470         print('{0}-'.format(outfile)+path+'.pdf is done')
0471
0472
0473 #######################################################
0474 if __name__ =='__main__':
0475 #######################################################
0476     main(sys.argv[1:])