Components/scripts/dqm-mbProfile.py

0001 #!/usr/bin/env python3
0002
0003 import os
0004 import collections
0005 import logging
0006 import resource
0007 import time
0008 import argparse
0009 import subprocess
0010 import signal
0011 import json
0012 import inspect
0013 import shutil
0014
0015 LOG_FORMAT='%(asctime)s: %(name)-20s - %(levelname)-8s - %(message)s'
0016 logging.basicConfig(format=LOG_FORMAT)
0017 log = logging.getLogger("mbProfile")
0018 log.setLevel(logging.INFO)
0019
0020 def read_procfs(ppath, only_ppid=True):
0021     def read(f):
0022         fp = os.path.join(ppath, f)
0023         with open(fp) as fd:
0024             return fd.read()
0025
0026     def read_status():
0027         st = {}
0028
0029         fp = os.path.join(ppath, "status")
0030         with open(fp) as fd:
0031             for line in fd.readlines():
0032                 if not line: continue
0033
0034                 key, value = line.split(":", 1)
0035                 st[key] = value.strip()
0036
0037         return st
0038
0039     try:
0040         dct = {}
0041
0042         dct["statm"] = read("statm").strip()
0043         dct["stat"] = read("stat").strip()
0044         dct["cmdline"] = read("cmdline").strip().replace("\0", " ")
0045
0046         status = read_status()
0047         dct["status"] = status
0048         dct["pid"] = int(status["Pid"])
0049         dct["parent_pid"] = int(status["PPid"])
0050
0051         return dct
0052     except:
0053         log.warning("Exception in read_procfs.", exc_info=True)
0054         pass
0055
0056 def build_process_list():
0057     lst = os.listdir("/proc/")
0058     for f in lst:
0059         if not f.isdigit(): continue
0060
0061         proc = read_procfs(os.path.join("/proc", f))
0062         if proc:
0063             yield proc
0064
0065 def get_children(ppid):
0066     """ Select all processes which are descendant from ppid (exclusive). """
0067
0068     pid_dct = {}
0069     for proc in build_process_list():
0070         proc["_children"] = []
0071         pid_dct[proc["pid"]] = proc
0072
0073     # fill in children array
0074     for pid in list(pid_dct.keys()):
0075         parent_pid = pid_dct[pid]["parent_pid"]
0076
0077         if parent_pid in pid_dct:
0078             pid_dct[parent_pid]["_children"].append(pid)
0079
0080     # now just walk down the tree
0081     if ppid is None or ppid not in pid_dct:
0082         # process has quit, we exit
0083         return []
0084
0085     accepted = []
0086     to_accept = collections.deque([ppid, ])
0087
0088     while to_accept:
0089         head = pid_dct[to_accept.popleft()]
0090
0091         # do not include the monitoring pid
0092         if head["pid"] != ppid:
0093             accepted.append(head)
0094
0095         to_accept.extend(head.get("_children", []))
0096         head["children"] = head["_children"]
0097         del head["_children"]
0098
0099         # deleting children breaks infinite loops
0100         # but Dima, can a process tree contain a loop? yes - via race-condition in reading procfs
0101
0102     return accepted
0103
0104 class Profile(object):
0105     def __init__(self, args):
0106         self.time = time.time()
0107         self.final = False
0108         self.pid = None
0109         self.known_pids = {}
0110
0111         self.ru = {}
0112         self.ru_diff = {}
0113
0114         self._offset_ru = None
0115         self._args = args
0116
0117         if self._args.file:
0118             self._file = open(self._args.file, "w")
0119         else:
0120             self._file = None
0121
0122         self.update()
0123
0124     def update_ru(self):
0125         fields_to_subtract = (
0126             "ru_utime", "ru_stime", "ru_maxrss", "ru_minflt", "ru_majflt", "ru_nswap",
0127             "ru_inblock", "ru_oublock", "ru_msgsnd", "ru_msgrcv", "ru_nsignals", "ru_nvcsw", "ru_nivcsw",
0128         )
0129
0130         rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
0131         self.ru = rusage
0132
0133         if self._offset_ru is None:
0134             self._offset_ru = rusage
0135
0136         for field in fields_to_subtract:
0137             current = getattr(self.ru, field)
0138             base = getattr(self._offset_ru, field)
0139
0140             self.ru_diff[field] = current - base
0141
0142     # this is taken from: http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py
0143     def read_smaps(self, proc_dict):
0144         Private, Shared, Pss = 0, 0, 0
0145
0146         fp = os.path.join("/proc/%d" % proc_dict["pid"], "smaps")
0147         with open(fp) as fd:
0148             for line in fd.readlines():
0149                 if line.startswith("Shared"):
0150                     Shared += int(line.split()[1])
0151                 elif line.startswith("Private"):
0152                     Private += int(line.split()[1])
0153                 elif line.startswith("Pss"):
0154                     Pss += int(line.split()[1])
0155
0156         proc_dict["smaps_shared"] = Shared * 1024
0157         proc_dict["smaps_private"] = Private * 1024
0158         proc_dict["smaps_pss"] = Pss * 1024
0159
0160     def update_proc(self):
0161         procs = get_children(os.getpid())
0162
0163         # we can only do it here, permision-wise
0164         # ie only for owned processes
0165         for proc in procs:
0166             try:
0167                 self.read_smaps(proc)
0168             except:
0169                 log.warning("Exception in read_smaps.", exc_info=True)
0170
0171         # we need to mark not-running ones as such
0172         stopped = set(self.known_pids.keys())
0173         for proc in procs:
0174             proc["running"] = True
0175
0176             pid = proc["pid"]
0177             self.known_pids[pid] = proc
0178
0179             if pid in stopped:
0180                 stopped.remove(pid)
0181
0182         for pid in stopped:
0183             self.known_pids[pid]["running"] = False
0184
0185     def update(self):
0186         self.time = time.time()
0187
0188         self.update_ru()
0189         self.update_proc()
0190
0191         if self._file:
0192             json.dump(self.to_dict(), self._file)
0193             self._file.write("\n")
0194             self._file.flush()
0195
0196         log.info("Written profile to: %s, took=%.03f", self._args.file, time.time() - self.time)
0197
0198     def to_dict(self):
0199         dct = collections.OrderedDict()
0200         dct['time']         = self.time
0201         dct['pid']          = self.pid
0202         dct['final']        = self.final
0203
0204         dct['ru_diff']      = dict(self.ru_diff)
0205         dct['ru']           = dict((k, v) for k, v in inspect.getmembers(self.ru) if k.startswith('ru_'))
0206         dct['known_pids']   = dict(self.known_pids)
0207         return dct
0208
0209     def finish(self):
0210         self.final = True
0211         self.update()
0212
0213         if self._file:
0214             self._file.close()
0215             self._file = None
0216         else:
0217             log.info("ru_diff: %s", self.ru_diff)
0218
0219
0220 ALARM_TIMER = 1
0221 ALARM_P_OBJECT = None
0222
0223 def handle_alarm(num, frame):
0224     if ALARM_P_OBJECT:
0225         ALARM_P_OBJECT.update()
0226
0227     signal.alarm(ALARM_TIMER)
0228
0229 def run_and_monitor(args):
0230     profile = Profile(args)
0231
0232     proc = subprocess.Popen(args.pargs)
0233     profile.pid = proc.pid
0234
0235     global ALARM_P_OBJECT
0236     ALARM_P_OBJECT = profile
0237
0238     signal.signal(signal.SIGALRM, handle_alarm)
0239     signal.alarm(ALARM_TIMER)
0240
0241     proc.wait()
0242     profile.finish()
0243
0244 def find_and_write_html(p, args):
0245     # create the dir if necessary
0246     if p and not os.path.exists(p):
0247         os.makedirs(p)
0248
0249     html_paths = [
0250         os.path.join(os.getenv("CMSSW_BASE"),"src/DQMServices/Components/data/html"),
0251         os.path.join(os.getenv("CMSSW_RELEASE_BASE"), "src/DQMServices/Components/data/html"),
0252     ]
0253
0254     def find_file(f):
0255         fails = []
0256         for p in html_paths:
0257             x = os.path.join(p, f)
0258             if os.path.exists(x):
0259                 return x
0260             else:
0261                 fails.append(x)
0262
0263         log.warning("Could not find html file: %s (%s)", f, fails)
0264
0265     for f in ['mbGraph.js', 'mbGraph.html']:
0266         target_fn = os.path.join(p, f)
0267         source_fn = find_file(f)
0268         if source_fn:
0269             log.info("Copying %s to %s", source_fn, target_fn)
0270             shutil.copyfile(source_fn, target_fn)
0271
0272     # create json file
0273     target_fn = os.path.join(p, "mbGraph.json")
0274     log.info("Creating %s", target_fn)
0275     with open(target_fn, "w") as fp:
0276         dct = {
0277             "file": os.path.basename(args.file),
0278             "interval": args.i,
0279             "env": {
0280                 "CMSSW_GIT_HASH": os.getenv("CMSSW_GIT_HASH"),
0281                 "CMSSW_RELEASE_BASE": os.getenv("CMSSW_RELEASE_BASE"),
0282                 "SCRAM_ARCH": os.getenv("SCRAM_ARCH"),
0283             },
0284         }
0285
0286         json.dump(dct, fp, indent=2)
0287
0288
0289 if __name__ == "__main__":
0290     parser = argparse.ArgumentParser(description="Profile child processes and produce data for rss and such graphs.")
0291     parser.add_argument("-f", "--file", type=str, default="performance.json", help="Filename to write.", metavar="performance.json")
0292     parser.add_argument("-i", type=int, help="Time interval between profiles.", default=15)
0293     parser.add_argument('-q', action='store_true', help="Reduce logging.")
0294     parser.add_argument('-w', action='store_true', help="Write html helper files for rendering the performance file.")
0295     parser.add_argument('pargs', nargs=argparse.REMAINDER)
0296
0297     args = parser.parse_args()
0298
0299     if not args.pargs:
0300         parser.print_help()
0301         sys.exit(-1)
0302     elif args.pargs[0] == "--":
0303         # compat with 2.6
0304         args.pargs = args.pargs[1:]
0305
0306     ALARM_TIMER = args.i
0307
0308     if args.q:
0309         log.setLevel(logging.WARNING)
0310
0311     if args.w:
0312         p = os.path.dirname(args.file)
0313         find_and_write_html(p, args)
0314
0315     ## do some signal magic
0316     #signal.signal(signal.SIGINT, handle_signal)
0317     #signal.signal(signal.SIGTERM, handle_signal)
0318
0319     run_and_monitor(args)
0320