openpilot v0.5.12 release

selfdrive/debug/cpu_usage_stat.py

@@ -5,6 +5,7 @@ import sys
 import numpy as np
 import argparse
 import re
+from collections import defaultdict
 
 '''
 System tools like top/htop can only show current cpu usage values, so I write this script to do statistics jobs.
@@ -27,9 +28,11 @@ PRINT_INTERVAL = 5
 SLEEP_INTERVAL = 0.2
 
 monitored_proc_names = [
-  'ubloxd', 'thermald', 'uploader', 'controlsd', 'plannerd', 'radard', 'mapd', 'loggerd' , 'logmessaged', 'tombstoned',
+  'ubloxd', 'thermald', 'uploader', 'deleter', 'controlsd', 'plannerd', 'radard', 'mapd', 'loggerd' , 'logmessaged', 'tombstoned',
   'logcatd', 'proclogd', 'boardd', 'pandad', './ui', 'calibrationd', 'locationd', 'visiond', 'sensord', 'updated', 'gpsd', 'athena']
+cpu_time_names = ['user', 'system', 'children_user', 'children_system']
 
+timer = getattr(time, 'monotonic', time.time)
 
 def get_arg_parser():
   parser = argparse.ArgumentParser(
@@ -38,8 +41,10 @@ def get_arg_parser():
 
   parser.add_argument("proc_names", nargs="?", default='',
                       help="Process names to be monitored, comma seperated")
-  parser.add_argument("--list_all", nargs="?", type=bool, default=False,
+  parser.add_argument("--list_all", action='store_true',
                       help="Show all running processes' cmdline")
+  parser.add_argument("--detailed_times", action='store_true',
+                      help="show cpu time details (split by user, system, child user, child system)")
   return parser
 
 
@@ -61,39 +66,54 @@ if __name__ == "__main__":
     if matched:
       k = ' '.join(p.cmdline())
       print('Add monitored proc:', k)
-      stats[k] = {'cpu_samples': [], 'avg_cpu': None, 'min': None, 'max': None}
+      stats[k] = {'cpu_samples': defaultdict(list), 'min': defaultdict(lambda: None), 'max': defaultdict(lambda: None),
+                  'avg': defaultdict(lambda: 0.0), 'last_cpu_times': None, 'last_sys_time':None}
+      stats[k]['last_sys_time'] = timer()
+      stats[k]['last_cpu_times'] = p.cpu_times()
       monitored_procs.append(p)
   i = 0
   interval_int = int(PRINT_INTERVAL / SLEEP_INTERVAL)
   while True:
     for p in monitored_procs:
       k = ' '.join(p.cmdline())
-      stats[k]['cpu_samples'].append(p.cpu_percent())
+      cur_sys_time = timer()
+      cur_cpu_times = p.cpu_times()
+      cpu_times = np.subtract(cur_cpu_times, stats[k]['last_cpu_times']) / (cur_sys_time - stats[k]['last_sys_time'])
+      stats[k]['last_sys_time'] = cur_sys_time
+      stats[k]['last_cpu_times'] = cur_cpu_times
+      cpu_percent = 0
+      for num, name in enumerate(cpu_time_names):
+        stats[k]['cpu_samples'][name].append(cpu_times[num])
+        cpu_percent += cpu_times[num]
+      stats[k]['cpu_samples']['total'].append(cpu_percent)
     time.sleep(SLEEP_INTERVAL)
     i += 1
     if i % interval_int == 0:
       l = []
-      avg_cpus = []
       for k, stat in stats.items():
         if len(stat['cpu_samples']) <= 0:
           continue
-        avg_cpu = np.array(stat['cpu_samples']).mean()
-        c = len(stat['cpu_samples'])
-        stat['cpu_samples'] = []
-        if not stat['avg_cpu']:
-          stat['avg_cpu'] = avg_cpu
-        else:
-          stat['avg_cpu'] = (stat['avg_cpu'] * (c + i) + avg_cpu * c) / (c + i + c)
-        if not stat['min'] or avg_cpu < stat['min']:
-          stat['min'] = avg_cpu
-        if not stat['max'] or avg_cpu > stat['max']:
-          stat['max'] = avg_cpu
-        msg = 'avg: {1:.2f}%, min: {2:.2f}%, max: {3:.2f}% {0}'.format(os.path.basename(k), stat['avg_cpu'], stat['min'], stat['max'])
-        l.append((os.path.basename(k), avg_cpu, msg))
-        avg_cpus.append(avg_cpu)
+        for name, samples in stat['cpu_samples'].iteritems():
+          samples = np.array(samples)
+          avg = samples.mean()
+          c = samples.size
+          min_cpu = np.amin(samples)
+          max_cpu = np.amax(samples)
+          if stat['min'][name] is None or min_cpu < stat['min'][name]:
+            stat['min'][name] = min_cpu
+          if stat['max'][name] is None or max_cpu > stat['max'][name]:
+            stat['max'][name] = max_cpu
+          stat['avg'][name] = (stat['avg'][name] * (i - c) + avg * c) / (i)
+          stat['cpu_samples'][name] = []
+
+        msg = 'avg: {1:.2%}, min: {2:.2%}, max: {3:.2%} {0}'.format(os.path.basename(k), stat['avg']['total'], stat['min']['total'], stat['max']['total'])
+        if args.detailed_times:
+          for stat_type in ['avg', 'min', 'max']:
+            msg += '\n {}: {}'.format(stat_type, [name + ':' + str(round(stat[stat_type][name]*100, 2)) for name in cpu_time_names])
+        l.append((os.path.basename(k), stat['avg']['total'], msg))
       l.sort(key= lambda x: -x[1])
       for x in l:
         print(x[2])
-      print('avg sum: {0:.2f}%\n'.format(
-        sum([stat['avg_cpu'] for k, stat in stats.items()])
+      print('avg sum: {0:.2%} over {1} samples {2} seconds\n'.format(
+        sum([stat['avg']['total'] for k, stat in stats.items()]), i, i * SLEEP_INTERVAL
       ))

selfdrive/debug/tuner.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""
+This tool can be used to quickly changes the values in a JSON file used for tuning
+Keys like in vim:
+ - h: decrease by 0.05
+ - l: increase by 0.05
+ - k: move pointer up
+ - j: move pointer down
+"""
+
+import tty
+import sys
+import json
+import termios
+from collections import OrderedDict
+
+FILENAME = '/data/tuning.json'
+
+def read_tuning():
+  while True:
+    try:
+      return json.loads(open(FILENAME).read())
+    except:
+      pass
+
+def main():
+  dat = json.loads(open(FILENAME, 'r').read())
+  dat = OrderedDict(sorted(dat.items(), key=lambda i: i[0]))
+
+  cur = 0
+  while True:
+    sys.stdout.write("\x1Bc")
+
+    for i, (k, v) in enumerate(dat.items()):
+      prefix = "> " if cur == i else "  "
+      print((prefix + k).ljust(20) + "%.2f" % v)
+
+    key = sys.stdin.read(1)[0]
+
+    write = False
+    if key == "k":
+      cur = max(0, cur - 1)
+    elif key == "j":
+      cur = min(len(dat.keys()) - 1, cur + 1)
+    elif key == "l":
+      dat[dat.keys()[cur]] += 0.05
+      write = True
+    elif key == "h":
+      dat[dat.keys()[cur]] -= 0.05
+      write = True
+    elif key == "q":
+      break
+
+    if write:
+      open(FILENAME, 'w').write(json.dumps(dat))
+
+
+if __name__ == "__main__":
+  orig_settings = termios.tcgetattr(sys.stdin)
+  tty.setcbreak(sys.stdin)
+
+  try:
+    main()
+    termios.tcsetattr(sys.stdin, termios.TCSADRAIN, orig_settings)
+  except:
+    termios.tcsetattr(sys.stdin, termios.TCSADRAIN, orig_settings)
+    raise