Tools/scripts/timingPdfMaker.py

0001 #! /usr/bin/env python
0002
0003 #################################################################
0004 # timingPdfMaker.py - python script
0005 # Created by Alejandro Gomez
0006 # For more info, please check:
0007 # https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary
0008 # or by email: alejandro.gomez@cern.ch
0009
0010 # This python script creates managebles pdf for HLT Timing Studies.
0011 # It saves some.png files from a former root file, creates a tex file,
0012 # compiles it twice for better results and remove all the temp files.
0013 # Please check the usage function to see how it works.
0014 ###############################################################
0015
0016 #############################################
0017 ## Imports
0018 ############################################
0019
0020 from __future__ import print_function
0021 import os, glob
0022 import os.path
0023 import operator
0024 import subprocess
0025 import sys, getopt
0026 sys.argv.append('-b')
0027 from ROOT import *
0028 import cStringIO
0029
0030
0031 gROOT.SetStyle("Plain")
0032 gStyle.SetOptStat(111111)
0033 gStyle.SetHistFillColor(kBlue)
0034
0035 ###############################################################
0036 def usage():
0037 ###############################################################
0038     print("\nThis is the usage function\n")
0039     print('Usage: '+sys.argv[0]+' -i <file1> [option]')
0040     print('e.g.:  '+sys.argv[0]+' -i outputTiming.root -t')
0041     print('e.g.:  '+sys.argv[0]+' -i outputTiming.root -s HLT_Jet300_v5\n')
0042
0043     print('\n-----Options-----')
0044     print(' -i          Input File')
0045     print(' -o          Output File (optional)')
0046     print(' -t          For only main time info per event. It will take less than 1 min.')
0047     print(' -p          For path time info. It will take approx 25 min.')
0048     print(' -m          For module time info. It will take approx 25 min.')
0049     print(' -s Path_Name            (For an specific path). Ti will take less than 1 min.')
0050     print('\n For -p or -m option, the process needs like 200 Mb in disk space,')
0051     print(' but please dont be afraid. Before the process ends, all the temporal files')
0052     print(' will be erased.')
0053
0054
0055 ###############################################################
0056 def maininfo(infile, outfile):
0057     ''' Creates main info tex file'''
0058 ##############################################################
0059     texpreamble = ['\documentclass[10pt,a5paper,landscape]{report}\n',
0060                    '\\usepackage{graphicx}\n',
0061                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0062                     '\\usepackage[linktocpage]{hyperref}\n',
0063                     '\hypersetup{backref, colorlinks=true}\n',
0064                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Main Info }\n',
0065                     '\\author{\Large{CMS Experiment}}\n',
0066                     '\date{\\today}\n',
0067                     '\\begin{document}\n',
0068                     '\maketitle\n',
0069                     '\\newpage\n',
0070                     '\clearpage\n'
0071                     '\\tableofcontents\n',
0072                     '\\newpage\n',
0073                     '\\newpage \chapter{Total time for all modules per event} \\newpage \centering \includegraphics[scale=0.6]{totalTimetemp.png}\n']
0074
0075     names1 = {}
0076     names4 = {}
0077     specific = {}
0078     file1 = TFile(infile,'read')
0079     for k in file1.GetListOfKeys():
0080         allnames = k.ReadObj().GetName()
0081         if 'pathTime_' in allnames:
0082             pathname = '_'.join(allnames.split('_')[1:])
0083             if not pathname in names1:
0084                 names1[pathname] = k.ReadObj().GetMean()
0085         elif 'incPathTime_' in allnames:
0086             pathname = '_'.join(allnames.split('_')[1:])
0087             if not pathname in names4:
0088                 names4[pathname] = k.ReadObj().GetMean()
0089
0090     names2 = dict(sorted(names1.items(), key=operator.itemgetter(1),reverse=True)[:10])
0091     names3 = sorted(names2, key=names2.get, reverse=True)
0092     names5 = dict(sorted(names4.items(), key=operator.itemgetter(1),reverse=True)[:10])
0093     names6 = sorted(names5, key=names5.get, reverse=True)
0094
0095     texfile = open(outfile+'-main.tex', 'w')
0096     texfile.writelines(texpreamble)
0097     if os.path.exists(infile.replace('.root','')+'-summary.txt'):
0098         excludefile = open(infile.replace('.root','')+'-summary.txt', 'r')
0099         texfile.write('\\newpage \section{Summary} \n \\begin{verbatim} \n')
0100         for line in excludefile.readlines():
0101             texfile.write(line+'\n')
0102         excludefile.close()
0103         texfile.write('\end{verbatim}')
0104     texfile.write('\\newpage \\chapter{10 Slowest Paths}\n')
0105     texfile.write('\section{Average module (in path) time}\n')
0106     for path in names3:
0107         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathTimeSummary'+ path.replace('_','') +'temp.png}\n')
0108         get_plot2(infile,'moduleInPathTimeSummary_'+path)
0109     texfile.write('\section{Average module (in path) running time}\n')
0110     for path in names3:
0111         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathScaledTimeSummary'+ path.replace('_','') +'temp.png}\n')
0112         get_plot2(infile,'moduleInPathScaledTimeSummary_'+path)
0113     texfile.write('\section{Per event time for path}\n')
0114     for path in names3:
0115         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{pathTime'+ path.replace('_','') +'temp.png}\n')
0116         get_plot1(infile,'pathTime_'+path)
0117     texfile.write('\section{Per event incremental time for path}\n')
0118     for path in names6:
0119         texfile.write('\\newpage \subsection{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{incPathTime'+ path.replace('_','') +'temp.png}\n')
0120         get_plot1(infile,'incPathTime_'+path)
0121     texfile.write('\end{document}')
0122     texfile.close()
0123
0124     texfile.close()
0125
0126     get_plot1(infile,'totalTime')
0127
0128
0129
0130 #################################################
0131 def pathsinfo(infile,outfile):
0132     '''Create the paths info tex file'''
0133 ################################################
0134
0135     texpreamble = ['\documentclass[10pt,a5paper,landscape]{book}\n',
0136                     '\\usepackage{graphicx}\n',
0137                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0138                     '\\usepackage[linktocpage]{hyperref}\n',
0139                     '\\usepackage[titles]{tocloft}\n'
0140                     '\hypersetup{backref, colorlinks=true}\n',
0141                     '\setlength{\cftsecnumwidth}{4em}\n'
0142                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Paths Info }\n',
0143                     '\\author{\Large{CMS Experiment}}\n',
0144                     '\date{\\today}\n',
0145                     '\\begin{document}\n',
0146                     '\maketitle\n',
0147                     '\\newpage\n',
0148                     '\\tableofcontents\n',
0149                     '\\newpage\n']
0150
0151     names1 = {}
0152     file = TFile(infile,'read')
0153     for k in file.GetListOfKeys():
0154         allnames= k.ReadObj().GetName()
0155         mean = 1
0156         if 'moduleInPathScaledTime_' in allnames:
0157             pathname = '_'.join(allnames.split('_')[1:-1])
0158             if not pathname in names1:
0159                 names1[pathname] = mean
0160     names = sorted(names1.keys())
0161
0162     texfile = open(outfile+'-paths.tex', 'w')
0163     texfile.writelines(texpreamble)
0164
0165     texfile.write('\\chapter{Average module (in path) time}\n')
0166     for path in names:
0167         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathTimeSummary'+ path.replace('_','') +'temp.png}\n')
0168         get_plot2(infile,'moduleInPathTimeSummary_'+path)
0169     texfile.write('\\chapter{Average module (in path) running time}\n')
0170     for path in names:
0171         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{moduleInPathScaledTimeSummary'+ path.replace('_','') +'temp.png}\n')
0172         get_plot2(infile,'moduleInPathScaledTimeSummary_'+path)
0173     texfile.write('\\chapter{Failing module (by path)}')
0174     for path in names:
0175         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.35]{failedModule'+ path.replace('_','') +'temp.png}\n')
0176         get_plot2(infile,'failedModule_'+path)
0177     texfile.write('\\chapter{Per event time for path}\n')
0178     for path in names:
0179         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{pathTime'+ path.replace('_','') +'temp.png}\n')
0180         get_plot1(infile,'pathTime_'+path)
0181     texfile.write('\\chapter{Per event incremental time for path}\n')
0182     for path in names:
0183         texfile.write('\\newpage \section{'+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{incPathTime'+ path.replace('_','') +'temp.png}\n')
0184         get_plot1(infile,'incPathTime_'+path)
0185
0186     texfile.write('\end{document}')
0187     texfile.close()
0188
0189
0190 #################################################
0191 def moduleinfo(infile,outfile):
0192     '''Create the paths info tex file'''
0193 ################################################
0194
0195     texpreamble = ['\documentclass[10pt,a5paper,landscape]{report}\n',
0196                     '\\usepackage{graphicx}\n',
0197                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0198                     '\\usepackage[linktocpage]{hyperref}\n',
0199                     '\hypersetup{backref, colorlinks=true}\n',
0200                     '\\usepackage[titles]{tocloft}\n'
0201                     '\setlength{\cftsecnumwidth}{4em}\n'
0202                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Modules Info }\n',
0203                     '\\author{\Large{CMS Experiment}}\n',
0204                     '\date{\\today}\n',
0205                     '\\begin{document}\n',
0206                     '\maketitle\n',
0207                     '\\newpage\n',
0208                     '\\tableofcontents\n',
0209                     '\\newpage\n']
0210
0211     names1 = {}
0212     file = TFile(infile,'read')
0213     for k in file.GetListOfKeys():
0214         allnames = k.ReadObj().GetName()
0215         mean = 1
0216         if 'moduleTime_' in allnames:
0217             modname = ''.join(allnames.split('_')[1:])
0218             if not (('!' in modname) or ('-' in modname)):
0219                 if not modname in names1:
0220                     names1[modname] = mean
0221     names = sorted(names1.keys())
0222
0223     texfile1 = open(outfile+'-modules.tex', 'w')
0224     texfile1.writelines(texpreamble)
0225
0226     texfile1.write('\\chapter{Time per event for module} \n \\newpage')
0227     for modules in names:
0228         texfile1.write('\section{'+modules+'}')
0229         texfile1.write('\centering \includegraphics[scale=0.4]{moduleTime'+ modules +'temp.png}\n')
0230         get_plot1(infile,'moduleTime_'+modules)
0231     texfile1.write('\end{document}')
0232     texfile1.close()
0233
0234     texfile2 = open(outfile+'-runningModules.tex', 'w')
0235     texfile2.writelines(texpreamble)
0236     texfile2.write('\\chapter{Running Time per event for module} \n \\newpage')
0237     for modules in names:
0238         texfile2.write('\section{'+modules+'}')
0239         texfile2.write('\centering \includegraphics[scale=0.45]{moduleScaledTime'+modules+'temp.png}\n')
0240         get_plot1(infile,'moduleScaledTime_'+ modules)
0241
0242     texfile2.write('\end{document}')
0243     texfile2.close()
0244
0245 ###############################################################
0246 def specificpathinfo(infile, outfile, path):
0247     ''' Creates an specific path info tex file'''
0248 ##############################################################
0249     texpreamble = ['\documentclass[10pt,a5paper,landscape]{report}\n',
0250                     '\\usepackage{graphicx}\n',
0251                     '\\usepackage[a5paper,vmargin={5mm,2mm},hmargin={5mm,5mm}]{geometry}\n',
0252                     '\\usepackage[linktocpage]{hyperref}\n',
0253                     '\\usepackage[titles]{tocloft}\n'
0254                     '\hypersetup{backref, colorlinks=true}\n',
0255                     '\setlength{\cftsubsecnumwidth}{4em}\n'
0256                     '\\title{ \\textbf{\Huge{HLT Timing Summary}} \\footnote{\large{Documentation at \\url{https://twiki.cern.ch/twiki/bin/view/CMS/SWGuideHLTTimingSummary}}} \\\\ Main Info + ' + path.replace('_','\_') +' info }\n',
0257                     '\\author{\Large{CMS Experiment}}\n',
0258                     '\date{\\today}\n',
0259                     '\\begin{document}\n',
0260                     '\maketitle\n',
0261                     '\\tableofcontents \n'
0262                     '\\newpage\n \\chapter{Main Info} \n',
0263                     '\\newpage \centering \section{Total time for all modules per event} \includegraphics[scale=0.6]{totalTimetemp.png}\n']
0264
0265     texfile = open(outfile+'-'+path+'.tex', 'w')
0266     texfile.writelines(texpreamble)
0267     get_plot1(infile,'totalTime')
0268
0269     names = {}
0270     file1 = TFile(infile,'read')
0271     for k in file1.GetListOfKeys():
0272         allnames = k.ReadObj().GetName()
0273         if 'moduleInPathScaledTime_' in allnames:
0274             pathname = '_'.join(allnames.split('_')[1:-1])
0275             if not pathname in names:
0276                 names[pathname] = {}
0277     for pathnames in names.keys():
0278         histo = file1.Get('moduleInPathTimeSummary_'+pathnames)
0279         for bin in range(histo.GetNbinsX()):
0280             label = histo.GetXaxis().GetBinLabel(bin+1)
0281             names[pathnames][bin+1] = label
0282
0283     for pathname in names:
0284         if path in pathname:
0285             texfile.write('\chapter{' + path.replace('_','\_')+ ' Info} \n')
0286             texfile.write('\\newpage \section{Average module in '+ path.replace('_','\_') +' time} \centering \includegraphics[scale=0.35]{moduleInPathTimeSummary'+ path.replace('_','') +'temp.png}\n')
0287             get_plot2(infile,'moduleInPathTimeSummary_'+path)
0288             texfile.write('\\newpage \section{Average module in '+ path.replace('_','\_') +' running time} \centering \includegraphics[scale=0.35]{moduleInPathScaledTimeSummary'+ path.replace('_','') +'temp.png}\n')
0289             get_plot2(infile,'moduleInPathScaledTimeSummary_'+path)
0290             texfile.write('\\newpage \section{Per event time for '+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{pathTime'+ path.replace('_','') +'temp.png}\n')
0291             get_plot1(infile,'pathTime_'+path)
0292             texfile.write('\\newpage \section{Per event incremental time for '+ path.replace('_','\_') +'} \centering \includegraphics[scale=0.6]{incPathTime'+ path.replace('_','') +'temp.png}\n')
0293             get_plot1(infile,'incPathTime_'+path)
0294             texfile.write('\section{Running time per event for '+path.replace('_','\_')+'}')
0295             for modules in names[path].values():
0296                 texfile.write('\subsection{'+ modules +'} \centering \includegraphics[scale=0.6]{moduleInPathScaledTime'+ path.replace('_','') + modules +'temp.png}\n')
0297                 get_plot1(infile,'moduleInPathScaledTime_'+ path +'_'+ modules)
0298     texfile.write('\\end{document}')
0299     texfile.close()
0300
0301
0302
0303
0304 ###################################################################
0305 def get_plot1(file,allnames):
0306     ''' Function to create the plot and save it as.png file '''
0307 ###################################################################
0308     file = TFile(file,'read')
0309     histo = file.Get(allnames)
0310     can = TCanvas('can', '', 800,600)
0311     can.cd()
0312     histo.UseCurrentStyle()
0313     histo.Draw()
0314     can.SetBorderMode(0)
0315     can.SetFillColor(kWhite)
0316     can.SaveAs(allnames.replace('_','')+'temp.png')
0317     del can
0318
0319
0320
0321 ###################################################################
0322 def get_plot2(infile,allnames):
0323     ''' Function to create the plot and save it as.png file '''
0324 ###################################################################
0325     file1 = TFile(infile,'read')
0326     histo = file1.Get(allnames)
0327     can = TCanvas('can', '', 1600,1000)
0328     can.cd()
0329     histo.UseCurrentStyle()
0330     histo.Draw()
0331     if histo.GetNbinsX() > 50:
0332         histo.GetXaxis().SetLabelSize(0.02)
0333     else:
0334         histo.GetXaxis().SetLabelSize(0.03)
0335     can.SetBorderMode(0)
0336     can.SetBorderSize(0)
0337     can.SetFillColor(kWhite)
0338     can.SetBottomMargin(0.4)
0339     can.SaveAs(allnames.replace('_','')+'temp.png')
0340     del can
0341
0342 ###############################################################
0343 def main(argv):
0344 ###############################################################
0345     print("\nPython script that creates Timing Summary pdf files")
0346     print("For more info, please contact Alejandro Gomez")
0347     print("email: alejandro.gomez@cern.ch\n")
0348
0349     infile = None
0350     outfile = None
0351     path = None
0352     call_maininfo = False
0353     call_pathsinfo = False
0354     call_modulesinfo = False
0355     call_specificinfo = False
0356     try:
0357         opts, args = getopt.getopt(argv, 'hi:o:tbpms:', ['help', 'input=', 'output='])
0358         if not opts:
0359             print('No options supplied')
0360             usage()
0361     except getopt.GetoptError as e:
0362         print(e)
0363         usage()
0364         sys.exit(2)
0365     for opt, arg in opts:
0366         if opt in ('h', '--help'):
0367             usage()
0368             sys.exit(2)
0369         elif opt == '-b':
0370             print('Running in batch mode')
0371         elif opt in ('-i', '--input'):
0372             infile = arg
0373             outfile = infile.replace('.root','')
0374         elif opt in ('-o', '--output'):
0375             outfile = arg
0376         elif opt == '-t':
0377             call_maininfo = True
0378         elif opt == '-p':
0379             call_pathsinfo = True
0380         elif opt == '-m':
0381             call_modulesinfo = True
0382         elif opt == '-s':
0383             path = arg
0384             call_specificinfo = True
0385         else:
0386             usage()
0387             sys.exit(2)
0388
0389
0390     if call_maininfo:
0391         print('Creating the Main Info Timing Summary pdf')
0392         print('Creating plots...')
0393         maininfo(infile,outfile)
0394         print('Compiling tex file......')
0395         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-main.tex'])
0396         print('Verifing......')
0397         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-main.tex'])    #twice for better compilation
0398         print('Removing temp files.........')
0399         os.remove(outfile+'-main.aux')
0400         os.remove(outfile+'-main.log')
0401         os.remove(outfile+'-main.out')
0402         os.remove(outfile+'-main.tex')
0403         os.remove(outfile+'-main.toc')
0404         for filename in glob.glob('*temp.png'):
0405             os.remove(filename)
0406         print('{0}-main.pdf is done'.format(outfile))
0407
0408     if call_pathsinfo:
0409         print('Creating the Paths Info Timing Summary pdf')
0410         print('This process takes awhile... please be patient')
0411         print('Creating plots...')
0412         pathsinfo(infile,outfile)
0413         print('Compiling tex file......')
0414         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-paths.tex'])
0415         print('Verifing......')
0416         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-paths.tex'])    #twice for better compilation
0417         print('Removing temp files.........')
0418         os.remove(outfile+'-paths.aux')
0419         os.remove(outfile+'-paths.log')
0420         os.remove(outfile+'-paths.out')
0421         os.remove(outfile+'-paths.tex')
0422         os.remove(outfile+'-paths.toc')
0423         for filename in glob.glob('*temp.png'):
0424             os.remove(filename)
0425         print('{0}-paths.pdf is done'.format(outfile))
0426
0427     if call_modulesinfo:
0428         print('Creating the Modules Info Timing Summary pdf')
0429         print('This process takes awhile... please be patient')
0430         print('Creating plots...')
0431         moduleinfo(infile,outfile)
0432         print('Compiling tex file......')
0433         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-modules.tex'])
0434         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-runningModules.tex'])
0435         print('Verifing......')
0436         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-modules.tex'])    #twice for better compilation
0437         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-runningModules.tex'])    #twice for better compilation
0438         print('Removing temp files.........')
0439         os.remove(outfile+'-modules.aux')
0440         os.remove(outfile+'-runningModules.aux')
0441         os.remove(outfile+'-modules.log')
0442         os.remove(outfile+'-runningModules.log')
0443         os.remove(outfile+'-modules.out')
0444         os.remove(outfile+'-runningModules.out')
0445         os.remove(outfile+'-modules.tex')
0446         os.remove(outfile+'-runningModules.tex')
0447         os.remove(outfile+'-modules.toc')
0448         os.remove(outfile+'-runningModules.toc')
0449         for filename in glob.glob('*temp.png'):
0450             os.remove(filename)
0451         print('{0}-modules.pdf is done'.format(outfile))
0452         print('{0}-runningModules.pdf is done'.format(outfile))
0453
0454     if call_specificinfo:
0455         print('Creating the Main Info + '+ path +' Timing Summary pdf')
0456         print('This process takes awhile... please be patient')
0457         print('Creating plots...')
0458         specificpathinfo(infile,outfile,path)
0459         print('Compiling tex file......')
0460         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-'+path+'.tex'])
0461         print('Verifing......')
0462         subprocess.call(['pdflatex', '-interaction=batchmode', outfile+'-'+path+'.tex'])    #twice for better compilation
0463         print('Removing temp files.........')
0464         os.remove(outfile+'-'+path+'.aux')
0465         os.remove(outfile+'-'+path+'.log')
0466         os.remove(outfile+'-'+path+'.out')
0467         os.remove(outfile+'-'+path+'.tex')
0468         os.remove(outfile+'-'+path+'.toc')
0469         for filename in glob.glob('*temp.png'):
0470             os.remove(filename)
0471         print('{0}-'.format(outfile)+path+'.pdf is done')
0472
0473
0474 #######################################################
0475 if __name__ =='__main__':
0476 #######################################################
0477     main(sys.argv[1:])