FileIO/scripts/dqmMemoryStats.py

0001 #!/bin/env python3
0002
0003 import sys
0004
0005 from DQMServices.FileIO.DQM import DQMReader
0006 from collections import namedtuple
0007
0008 HistogramEntry = namedtuple('HistogramEntry', ['type', 'bin_size', 'bin_count', 'extra', 'total_bytes'])
0009
0010 class HistogramAnalyzer(object):
0011     BIN_SIZE = {
0012         'TH1S': 2, 'TH2S': 2,
0013         'TH1F': 4, 'TH2F': 4,
0014         'TH1D': 8, 'TH2D': 8,
0015         'TH3F': 4,
0016         'TProfile': 8, 'TProfile2D': 8,
0017     }
0018
0019     def __init__(self):
0020         self._all = {}
0021
0022     def analyze(self, fn, obj):
0023         name = fn.split("/")[-1]
0024
0025         if hasattr(obj, 'ClassName'):
0026             # this is a root type
0027             t = str(obj.ClassName())
0028             bin_size = self.BIN_SIZE.get(t, None)
0029             if bin_size is None:
0030                 sys.stderr.write("warning: unknown root type: %s\n" % t)
0031                 sys.stderr.flush()
0032                 bin_size = 8
0033
0034             bin_count = obj.GetNcells()
0035             extra = len(fn)
0036             total_bytes = bin_count * bin_size + extra
0037
0038             self._all[fn] = HistogramEntry(t, bin_size, bin_count, extra, total_bytes)
0039         else:
0040             t = str(type(obj))
0041             #bin_count, bin_size, extra = 0, 0, len(str(obj)) + len(fn)
0042             # assume constant size for strings
0043             bin_count, bin_size, extra = 0, 0, 10 + len(fn)
0044             total_bytes = bin_count * bin_size + extra
0045
0046             self._all[fn] = HistogramEntry(t, bin_size, bin_count, extra, total_bytes)
0047
0048     def group(self, level, countObjects):
0049         group_stats = {}
0050
0051         for k, v in self._all.items():
0052             group_key = "/".join(k.split("/")[:level])
0053
0054             current = group_stats.get(group_key, 0)
0055             group_stats[group_key] = current + (1 if countObjects else v.total_bytes)
0056
0057         return group_stats
0058
0059     def difference(self, ref):
0060         results = HistogramAnalyzer()
0061         results._all = dict(self._all)
0062
0063         zero = HistogramEntry("null", 0, 0, 0, 0)
0064         def cmp(a, b):
0065             return HistogramEntry(b.type, b.bin_size,
0066                 a.bin_count - b.bin_count,
0067                 a.extra - b.extra,
0068                 a.total_bytes - b.total_bytes )
0069
0070         for k, refv in ref._all.items():
0071             results._all[k] = cmp(self._all.get(k, zero), refv)
0072
0073         return results
0074
0075
0076 def kibisize(num,args):
0077     if args.count:
0078       return str(num)
0079     pStr="%."+str(args.precision)+"f %s"
0080     for prefix in ['KiB','MiB','GiB']:
0081         num /= 1024.0
0082
0083         if num < 1024.0 or args.units == prefix:
0084             return pStr % (num, prefix)
0085     return pStr % (num, prefix)
0086
0087 def displayDirectoryStatistics(stats, args):
0088     group_stats = stats.group(args.depth, args.count)
0089
0090     cutoff, display = args.cutoff * 1024, args.display
0091
0092     as_list = [(v, k, ) for (k, v) in group_stats.items()]
0093     as_list.sort(reverse=True, key=lambda v_k1: abs(v_k1[0]))
0094
0095     if cutoff is not None:
0096         as_list = [v_k for v_k in as_list if abs(v_k[0]) > cutoff]
0097
0098     if display is not None:
0099         as_list = as_list[:display]
0100
0101     if args.human:
0102         print("*" * 80)
0103         print((" DQM level %d folder breakdown " % args.depth).center(80, "*"))
0104         if cutoff:
0105             print(("* Size cutoff: %s" % kibisize(cutoff,args)).ljust(79) + "*")
0106         if display:
0107             print(("* Showing top %d entries." % display).ljust(79) + "*")
0108         print("*" * 80)
0109
0110     for v, k in as_list:
0111         if args.human:
0112             print(kibisize(v,args).ljust(16, " "), k)
0113         else:
0114             print(v, k)
0115
0116 if __name__ == '__main__':
0117     import argparse
0118
0119     parser = argparse.ArgumentParser()
0120     parser.add_argument("-i", "--input", help = "Input DQM ROOT file")
0121     parser.add_argument("-r", "--ref", help = "Reference DQM ROOT file (to diff)")
0122     parser.add_argument("--summary", help = "Dump summary", action = "store_true")
0123     parser.add_argument("--count", help = "Count Histograms", action = "store_true")
0124     parser.add_argument("-x", "--human", help = "Human readable output.", action = "store_true")
0125     parser.add_argument("-n", "--display", help = "Max entries to display in --summary.", type = int, default = None)
0126     parser.add_argument("-c", "--cutoff", help = "Max cutoff to display in --summary.", type = float, default = 512, metavar="KiB")
0127     parser.add_argument("-d", "--depth", help = "Folder depth in --summary.", type = int, default = 2)
0128     parser.add_argument("-u", "--units", help = "Memory units to use (KiB,MiB,GiB) if fixed output desired", type = str, default = "None")
0129     parser.add_argument("-p", "--precision", help = "Places after decimal to display.", type = int, default = 2)
0130
0131     args = parser.parse_args()
0132
0133     stats = HistogramAnalyzer()
0134     reader = DQMReader(args.input)
0135     for (fn, v) in reader.read_objects():
0136         stats.analyze(fn, v)
0137     reader.close()
0138
0139     if args.ref:
0140         reader = DQMReader(args.ref)
0141         ref_stats = HistogramAnalyzer()
0142         for (fn, v) in reader.read_objects():
0143             ref_stats.analyze(fn, v)
0144         reader.close()
0145
0146         stats = stats.difference(ref_stats)
0147
0148     if args.summary:
0149         displayDirectoryStatistics(stats, args)
0150
0151     total = stats.group(0, args.count)
0152     if args.human:
0153         print("Total bytes: %s" % kibisize(total[""],args))
0154     else:
0155         print(total[""])