pandad: pure Python capnp helpers (#37025)

* pandad: pure Python capnp helpers

* cleanup

selfdrive/controls/lib/longitudinal_mpc_lib/SConscript

@@ -1,4 +1,4 @@
-Import('env', 'envCython', 'arch', 'msgq_python', 'common_python', 'pandad_python', 'np_version')
+Import('env', 'envCython', 'arch', 'msgq_python', 'common_python', 'np_version')
 
 gen = "c_generated_code"
 
@@ -67,7 +67,7 @@ lenv.Clean(generated_files, Dir(gen))
 generated_long = lenv.Command(generated_files,
                               source_list,
                               f"cd {Dir('.').abspath} && python3 long_mpc.py")
-lenv.Depends(generated_long, [msgq_python, common_python, pandad_python])
+lenv.Depends(generated_long, [msgq_python, common_python])
 
 lenv["CFLAGS"].append("-DACADOS_WITH_QPOASES")
 lenv["CXXFLAGS"].append("-DACADOS_WITH_QPOASES")

selfdrive/pandad/SConscript

@@ -1,13 +1,9 @@
-Import('env', 'envCython', 'common', 'messaging')
+Import('env', 'common', 'messaging')
 
 libs = ['usb-1.0', common, messaging, 'pthread']
 panda = env.Library('panda', ['panda.cc', 'panda_comms.cc', 'spi.cc'])
 
 env.Program('pandad', ['main.cc', 'pandad.cc', 'panda_safety.cc'], LIBS=[panda] + libs)
-env.Library('libcan_list_to_can_capnp', ['can_list_to_can_capnp.cc'])
-
-pandad_python = envCython.Program('pandad_api_impl.so', 'pandad_api_impl.pyx', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"])
-Export('pandad_python')
 
 if GetOption('extras'):
   env.Program('tests/test_pandad_usbprotocol', ['tests/test_pandad_usbprotocol.cc'], LIBS=[panda] + libs)

selfdrive/pandad/__init__.py

@@ -1,4 +1,3 @@
-# Cython, now uses scons to build
 from openpilot.selfdrive.pandad.pandad_api_impl import can_list_to_can_capnp, can_capnp_to_list
 assert can_list_to_can_capnp
 assert can_capnp_to_list

selfdrive/pandad/can_list_to_can_capnp.cc

@@ -1,50 +0,0 @@
-#include "cereal/messaging/messaging.h"
-#include "selfdrive/pandad/can_types.h"
-
-void can_list_to_can_capnp_cpp(const std::vector<CanFrame> &can_list, std::string &out, bool sendcan, bool valid) {
-  MessageBuilder msg;
-  auto event = msg.initEvent(valid);
-
-  auto canData = sendcan ? event.initSendcan(can_list.size()) : event.initCan(can_list.size());
-  int j = 0;
-  for (auto it = can_list.begin(); it != can_list.end(); it++, j++) {
-    auto c = canData[j];
-    c.setAddress(it->address);
-    c.setDat(kj::arrayPtr((uint8_t*)it->dat.data(), it->dat.size()));
-    c.setSrc(it->src);
-  }
-  const uint64_t msg_size = capnp::computeSerializedSizeInWords(msg) * sizeof(capnp::word);
-  out.resize(msg_size);
-  kj::ArrayOutputStream output_stream(kj::ArrayPtr<capnp::byte>((unsigned char *)out.data(), msg_size));
-  capnp::writeMessage(output_stream, msg);
-}
-
-// Converts a vector of Cap'n Proto serialized can strings into a vector of CanData structures.
-void can_capnp_to_can_list_cpp(const std::vector<std::string> &strings, std::vector<CanData> &can_list, bool sendcan) {
-  AlignedBuffer aligned_buf;
-  can_list.reserve(strings.size());
-
-  for (const auto &str : strings) {
-    // extract the messages
-    capnp::FlatArrayMessageReader reader(aligned_buf.align(str.data(), str.size()));
-    cereal::Event::Reader event = reader.getRoot<cereal::Event>();
-
-    auto frames = sendcan ? event.getSendcan() : event.getCan();
-
-    // Add new CanData entry
-    CanData &can_data = can_list.emplace_back();
-    can_data.nanos = event.getLogMonoTime();
-    can_data.frames.reserve(frames.size());
-
-    // Populate CAN frames
-    for (const auto &frame : frames) {
-      CanFrame &can_frame = can_data.frames.emplace_back();
-      can_frame.src = frame.getSrc();
-      can_frame.address = frame.getAddress();
-
-      // Copy CAN data
-      auto dat = frame.getDat();
-      can_frame.dat.assign(dat.begin(), dat.end());
-    }
-  }
-}

selfdrive/pandad/can_types.h

@@ -1,15 +0,0 @@
-#pragma once
-
-#include <vector>
-#include <cstdint>
-
-struct CanFrame {
-  long src;
-  uint32_t address;
-  std::vector<uint8_t> dat;
-};
-
-struct CanData {
-  uint64_t nanos;
-  std::vector<CanFrame> frames;
-};

selfdrive/pandad/pandad_api_impl.py

@@ -0,0 +1,88 @@
+import time
+from cereal import log
+
+NO_TRAVERSAL_LIMIT = 2**64 - 1
+
+# Cache schema fields for faster access (avoids string lookup on each field access)
+_cached_reader_fields = None  # (address_field, dat_field, src_field) for reading
+_cached_writer_fields = None  # (address_field, dat_field, src_field) for writing
+
+
+def _get_reader_fields(schema):
+  """Get cached schema field objects for reading."""
+  global _cached_reader_fields
+  if _cached_reader_fields is None:
+    fields = schema.fields
+    _cached_reader_fields = (fields['address'], fields['dat'], fields['src'])
+  return _cached_reader_fields
+
+
+def _get_writer_fields(schema):
+  """Get cached schema field objects for writing."""
+  global _cached_writer_fields
+  if _cached_writer_fields is None:
+    fields = schema.fields
+    _cached_writer_fields = (fields['address'], fields['dat'], fields['src'])
+  return _cached_writer_fields
+
+
+def can_list_to_can_capnp(can_msgs, msgtype='can', valid=True):
+  """Convert list of CAN messages to Cap'n Proto serialized bytes.
+
+  Args:
+    can_msgs: List of tuples [(address, data_bytes, src), ...]
+    msgtype: 'can' or 'sendcan'
+    valid: Whether the event is valid
+
+  Returns:
+    Cap'n Proto serialized bytes
+  """
+  global _cached_writer_fields
+
+  dat = log.Event.new_message(valid=valid, logMonoTime=int(time.monotonic() * 1e9))
+  can_data = dat.init(msgtype, len(can_msgs))
+
+  # Cache schema fields on first call
+  if _cached_writer_fields is None and len(can_msgs) > 0:
+    _cached_writer_fields = _get_writer_fields(can_data[0].schema)
+
+  if _cached_writer_fields is not None:
+    addr_f, dat_f, src_f = _cached_writer_fields
+    for i, msg in enumerate(can_msgs):
+      f = can_data[i]
+      f._set_by_field(addr_f, msg[0])
+      f._set_by_field(dat_f, msg[1])
+      f._set_by_field(src_f, msg[2])
+
+  return dat.to_bytes()
+
+
+def can_capnp_to_list(strings, msgtype='can'):
+  """Convert Cap'n Proto serialized bytes to list of CAN messages.
+
+  Args:
+    strings: Tuple/list of serialized Cap'n Proto bytes
+    msgtype: 'can' or 'sendcan'
+
+  Returns:
+    List of tuples [(nanos, [(address, data, src), ...]), ...]
+  """
+  global _cached_reader_fields
+  result = []
+
+  for s in strings:
+    with log.Event.from_bytes(s, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as event:
+      frames = getattr(event, msgtype)
+
+      # Cache schema fields on first frame for faster access
+      if _cached_reader_fields is None and len(frames) > 0:
+        _cached_reader_fields = _get_reader_fields(frames[0].schema)
+
+      if _cached_reader_fields is not None:
+        addr_f, dat_f, src_f = _cached_reader_fields
+        frame_list = [(f._get_by_field(addr_f), f._get_by_field(dat_f), f._get_by_field(src_f)) for f in frames]
+      else:
+        frame_list = []
+
+      result.append((event.logMonoTime, frame_list))
+  return result

selfdrive/pandad/pandad_api_impl.pyx

@@ -1,56 +0,0 @@
-# distutils: language = c++
-# cython: language_level=3
-from cython.operator cimport dereference as deref, preincrement as preinc
-from libcpp.vector cimport vector
-from libcpp.string cimport string
-from libcpp cimport bool
-from libc.stdint cimport uint8_t, uint32_t, uint64_t
-
-cdef extern from "selfdrive/pandad/can_types.h":
-  cdef struct CanFrame:
-    long src
-    uint32_t address
-    vector[uint8_t] dat
-
-  cdef struct CanData:
-    uint64_t nanos
-    vector[CanFrame] frames
-
-cdef extern from "can_list_to_can_capnp.cc":
-  void can_list_to_can_capnp_cpp(const vector[CanFrame] &can_list, string &out, bool sendcan, bool valid) nogil
-  void can_capnp_to_can_list_cpp(const vector[string] &strings, vector[CanData] &can_data, bool sendcan)
-
-def can_list_to_can_capnp(can_msgs, msgtype='can', valid=True):
-  cdef CanFrame *f
-  cdef vector[CanFrame] can_list
-  cdef uint32_t cpp_can_msgs_len = len(can_msgs)
-
-  with nogil:
-    can_list.reserve(cpp_can_msgs_len)
-
-  for can_msg in can_msgs:
-    f = &(can_list.emplace_back())
-    f.address = can_msg[0]
-    f.dat = can_msg[1]
-    f.src = can_msg[2]
-
-  cdef string out
-  cdef bool is_sendcan = (msgtype == 'sendcan')
-  cdef bool is_valid = valid
-  with nogil:
-    can_list_to_can_capnp_cpp(can_list, out, is_sendcan, is_valid)
-  return out
-
-def can_capnp_to_list(strings, msgtype='can'):
-  cdef vector[CanData] data
-  can_capnp_to_can_list_cpp(strings, data, msgtype == 'sendcan')
-
-  result = []
-  cdef CanData *d
-  cdef vector[CanData].iterator it = data.begin()
-  while it != data.end():
-    d = &deref(it)
-    frames = [(f.address, (<char *>&f.dat[0])[:f.dat.size()], f.src) for f in d.frames]
-    result.append((d.nanos, frames))
-    preinc(it)
-  return result