replace dictdiffer with native capnp differ (#37279)

* replace dictdiffer with native capnp differ

* capnp diff

pyproject.toml

@@ -96,7 +96,6 @@ testing = [
 
 dev = [
   "av",
-  "dictdiffer",
   "matplotlib",
   "opencv-python-headless",
   "parameterized >=0.8, <0.9",

selfdrive/test/process_replay/compare_logs.py

@@ -3,13 +3,16 @@ import sys
 import math
 import capnp
 import numbers
-import dictdiffer
 from collections import Counter
 
 from openpilot.tools.lib.logreader import LogReader
 
 EPSILON = sys.float_info.epsilon
 
+_DynamicStructReader = capnp.lib.capnp._DynamicStructReader
+_DynamicListReader = capnp.lib.capnp._DynamicListReader
+_DynamicEnum = capnp.lib.capnp._DynamicEnum
+
 
 def remove_ignored_fields(msg, ignore):
   msg = msg.as_builder()
@@ -39,6 +42,61 @@ def remove_ignored_fields(msg, ignore):
   return msg
 
 
+def _diff_capnp(r1, r2, path, tolerance):
+  """Walk two capnp struct readers and yield (action, dotted_path, value) diffs.
+
+  Floats are compared with the given tolerance (combined absolute+relative).
+  """
+  schema = r1.schema
+
+  for fname in schema.non_union_fields:
+    child_path = path + (fname,)
+    v1 = getattr(r1, fname)
+    v2 = getattr(r2, fname)
+    yield from _diff_capnp_values(v1, v2, child_path, tolerance)
+
+  if schema.union_fields:
+    w1, w2 = r1.which(), r2.which()
+    if w1 != w2:
+      yield 'change', '.'.join(path), (w1, w2)
+    else:
+      child_path = path + (w1,)
+      v1, v2 = getattr(r1, w1), getattr(r2, w2)
+      yield from _diff_capnp_values(v1, v2, child_path, tolerance)
+
+
+def _diff_capnp_values(v1, v2, path, tolerance):
+  if isinstance(v1, _DynamicStructReader):
+    yield from _diff_capnp(v1, v2, path, tolerance)
+
+  elif isinstance(v1, _DynamicListReader):
+    dot = '.'.join(path)
+    n1, n2 = len(v1), len(v2)
+    n = min(n1, n2)
+    for i in range(n):
+      yield from _diff_capnp_values(v1[i], v2[i], path + (str(i),), tolerance)
+    if n2 > n:
+      yield 'add', dot, list(enumerate(v2[n:], n))
+    if n1 > n:
+      yield 'remove', dot, list(reversed([(i, v1[i]) for i in range(n, n1)]))
+
+  elif isinstance(v1, _DynamicEnum):
+    s1, s2 = str(v1), str(v2)
+    if s1 != s2:
+      yield 'change', '.'.join(path), (s1, s2)
+
+  elif isinstance(v1, float):
+    if not (v1 == v2 or (
+      math.isfinite(v1) and math.isfinite(v2) and
+      abs(v1 - v2) <= max(tolerance, tolerance * max(abs(v1), abs(v2)))
+    )):
+      yield 'change', '.'.join(path), (v1, v2)
+
+  else:
+    if v1 != v2:
+      yield 'change', '.'.join(path), (v1, v2)
+
+
 def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None,):
   if ignore_fields is None:
     ignore_fields = []
@@ -65,26 +123,7 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=Non
     msg2 = remove_ignored_fields(msg2, ignore_fields)
 
     if msg1.to_bytes() != msg2.to_bytes():
-      msg1_dict = msg1.as_reader().to_dict(verbose=True)
-      msg2_dict = msg2.as_reader().to_dict(verbose=True)
-
-      dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore_fields)
-
-      # Dictdiffer only supports relative tolerance, we also want to check for absolute
-      # TODO: add this to dictdiffer
-      def outside_tolerance(diff):
-        try:
-          if diff[0] == "change":
-            a, b = diff[2]
-            finite = math.isfinite(a) and math.isfinite(b)
-            if finite and isinstance(a, numbers.Number) and isinstance(b, numbers.Number):
-              return abs(a - b) > max(tolerance, tolerance * max(abs(a), abs(b)))
-        except TypeError:
-          pass
-        return True
-
-      dd = list(filter(outside_tolerance, dd))
-
+      dd = list(_diff_capnp(msg1.as_reader(), msg2.as_reader(), (), tolerance))
       diff.extend(dd)
   return diff
 

uv.lock

@@ -356,15 +356,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5e/58/d01538556103d544a5a5b4cbcb00646ff92d8a97f0a6283a56bede4307c8/dearpygui-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:9f2291313d2035f8a4108e13f60d8c1a0e7c19af7554a7739a3fd15b3d5af8f7", size = 1808971, upload-time = "2025-11-14T14:47:28.15Z" },
 ]
 
-[[package]]
-name = "dictdiffer"
-version = "0.9.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/61/7b/35cbccb7effc5d7e40f4c55e2b79399e1853041997fcda15c9ff160abba0/dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578", size = 31513, upload-time = "2021-07-22T13:24:29.276Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" },
-]
-
 [[package]]
 name = "dnspython"
 version = "2.8.0"
@@ -846,7 +837,6 @@ dependencies = [
 [package.optional-dependencies]
 dev = [
     { name = "av" },
-    { name = "dictdiffer" },
     { name = "matplotlib" },
     { name = "opencv-python-headless" },
     { name = "parameterized" },
@@ -887,7 +877,6 @@ requires-dist = [
     { name = "crcmod-plus" },
     { name = "cython" },
     { name = "dearpygui", marker = "(platform_machine != 'aarch64' and extra == 'tools') or (sys_platform != 'linux' and extra == 'tools')", specifier = ">=2.1.0" },
-    { name = "dictdiffer", marker = "extra == 'dev'" },
     { name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" },
     { name = "inputs" },
     { name = "jeepney" },