FileIO/scripts/dqmMemoryStats.py

0001 #!/bin/env python3
0002
0003 from __future__ import print_function
0004 import sys
0005
0006 from DQMServices.FileIO.DQM import DQMReader
0007 from collections import namedtuple
0008
0009 HistogramEntry = namedtuple('HistogramEntry', ['type', 'bin_size', 'bin_count', 'extra', 'total_bytes'])
0010
0011 class HistogramAnalyzer(object):
0012     BIN_SIZE = {
0013         'TH1S': 2, 'TH2S': 2,
0014         'TH1F': 4, 'TH2F': 4,
0015         'TH1D': 8, 'TH2D': 8,
0016         'TH3F': 4,
0017         'TProfile': 8, 'TProfile2D': 8,
0018     }
0019
0020     def __init__(self):
0021         self._all = {}
0022
0023     def analyze(self, fn, obj):
0024         name = fn.split("/")[-1]
0025
0026         if hasattr(obj, 'ClassName'):
0027             # this is a root type
0028             t = str(obj.ClassName())
0029             bin_size = self.BIN_SIZE.get(t, None)
0030             if bin_size is None:
0031                 sys.stderr.write("warning: unknown root type: %s\n" % t)
0032                 sys.stderr.flush()
0033                 bin_size = 8
0034
0035             bin_count = obj.GetNcells()
0036             extra = len(fn)
0037             total_bytes = bin_count * bin_size + extra
0038
0039             self._all[fn] = HistogramEntry(t, bin_size, bin_count, extra, total_bytes)
0040         else:
0041             t = str(type(obj))
0042             #bin_count, bin_size, extra = 0, 0, len(str(obj)) + len(fn)
0043             # assume constant size for strings
0044             bin_count, bin_size, extra = 0, 0, 10 + len(fn)
0045             total_bytes = bin_count * bin_size + extra
0046
0047             self._all[fn] = HistogramEntry(t, bin_size, bin_count, extra, total_bytes)
0048
0049     def group(self, level, countObjects):
0050         group_stats = {}
0051
0052         for k, v in self._all.items():
0053             group_key = "/".join(k.split("/")[:level])
0054
0055             current = group_stats.get(group_key, 0)
0056             group_stats[group_key] = current + (1 if countObjects else v.total_bytes)
0057
0058         return group_stats
0059
0060     def difference(self, ref):
0061         results = HistogramAnalyzer()
0062         results._all = dict(self._all)
0063
0064         zero = HistogramEntry("null", 0, 0, 0, 0)
0065         def cmp(a, b):
0066             return HistogramEntry(b.type, b.bin_size,
0067                 a.bin_count - b.bin_count,
0068                 a.extra - b.extra,
0069                 a.total_bytes - b.total_bytes )
0070
0071         for k, refv in ref._all.items():
0072             results._all[k] = cmp(self._all.get(k, zero), refv)
0073
0074         return results
0075
0076
0077 def kibisize(num,args):
0078     if args.count:
0079       return str(num)
0080     pStr="%."+str(args.precision)+"f %s"
0081     for prefix in ['KiB','MiB','GiB']:
0082         num /= 1024.0
0083
0084         if num < 1024.0 or args.units == prefix:
0085             return pStr % (num, prefix)
0086     return pStr % (num, prefix)
0087
0088 def displayDirectoryStatistics(stats, args):
0089     group_stats = stats.group(args.depth, args.count)
0090
0091     cutoff, display = args.cutoff * 1024, args.display
0092
0093     as_list = [(v, k, ) for (k, v) in group_stats.items()]
0094     as_list.sort(reverse=True, key=lambda v_k1: abs(v_k1[0]))
0095
0096     if cutoff is not None:
0097         as_list = [v_k for v_k in as_list if abs(v_k[0]) > cutoff]
0098
0099     if display is not None:
0100         as_list = as_list[:display]
0101
0102     if args.human:
0103         print("*" * 80)
0104         print((" DQM level %d folder breakdown " % args.depth).center(80, "*"))
0105         if cutoff:
0106             print(("* Size cutoff: %s" % kibisize(cutoff,args)).ljust(79) + "*")
0107         if display:
0108             print(("* Showing top %d entries." % display).ljust(79) + "*")
0109         print("*" * 80)
0110
0111     for v, k in as_list:
0112         if args.human:
0113             print(kibisize(v,args).ljust(16, " "), k)
0114         else:
0115             print(v, k)
0116
0117 if __name__ == '__main__':
0118     import argparse
0119
0120     parser = argparse.ArgumentParser()
0121     parser.add_argument("-i", "--input", help = "Input DQM ROOT file")
0122     parser.add_argument("-r", "--ref", help = "Reference DQM ROOT file (to diff)")
0123     parser.add_argument("--summary", help = "Dump summary", action = "store_true")
0124     parser.add_argument("--count", help = "Count Histograms", action = "store_true")
0125     parser.add_argument("-x", "--human", help = "Human readable output.", action = "store_true")
0126     parser.add_argument("-n", "--display", help = "Max entries to display in --summary.", type = int, default = None)
0127     parser.add_argument("-c", "--cutoff", help = "Max cutoff to display in --summary.", type = float, default = 512, metavar="KiB")
0128     parser.add_argument("-d", "--depth", help = "Folder depth in --summary.", type = int, default = 2)
0129     parser.add_argument("-u", "--units", help = "Memory units to use (KiB,MiB,GiB) if fixed output desired", type = str, default = "None")
0130     parser.add_argument("-p", "--precision", help = "Places after decimal to display.", type = int, default = 2)
0131
0132     args = parser.parse_args()
0133
0134     stats = HistogramAnalyzer()
0135     reader = DQMReader(args.input)
0136     for (fn, v) in reader.read_objects():
0137         stats.analyze(fn, v)
0138     reader.close()
0139
0140     if args.ref:
0141         reader = DQMReader(args.ref)
0142         ref_stats = HistogramAnalyzer()
0143         for (fn, v) in reader.read_objects():
0144             ref_stats.analyze(fn, v)
0145         reader.close()
0146
0147         stats = stats.difference(ref_stats)
0148
0149     if args.summary:
0150         displayDirectoryStatistics(stats, args)
0151
0152     total = stats.group(0, args.count)
0153     if args.human:
0154         print("Total bytes: %s" % kibisize(total[""],args))
0155     else:
0156         print(total[""])