Update pycapnp and Cython (#29021)

* cython + pycapnp

* from_bytes with context

* poetry lock

* limit numpy

* force build

* Revert "force build"

This reverts commit 8c7cb00421da9d0f1b7da2c94f75e9c43733f840.

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

SConstruct

@@ -263,6 +263,7 @@ py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
 envCython["CPPPATH"] += [py_include, np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":

pyproject.toml

@@ -16,7 +16,7 @@ casadi = "==3.6.3"
 cffi = "^1.15.1"
 crcmod = "^1.7"
 cryptography = "^37.0.4"
-Cython = "^0.29.30"
+Cython = "^3.0.0"
 flake8 = "^4.0.1"
 Flask = "^2.1.2"
 future-fstrings = "^1.2.0"  # for acados
@@ -27,14 +27,14 @@ Jinja2 = "^3.1.2"
 json-rpc = "^1.13.0"
 libusb1 = "^3.0.0"
 nose = "^1.3.7"
-numpy = "^1.23.0"
+numpy = "==1.24.2"
 onnx = "^1.14.0"
 onnxruntime-gpu = { version = "^1.15.1", platform = "linux", markers = "platform_machine == 'x86_64'" }
 pillow = "^9.2.0"
 poetry = "==1.2.2"
 protobuf = "==3.20.3"
 psutil = "^5.9.1"
-pycapnp = "==1.1.0"
+pycapnp = "^1.3.0"
 pycryptodome = "^3.15.0"
 PyJWT = "^2.5.0"
 pyopencl = "^2022.2.4"

selfdrive/car/car_helpers.py

@@ -89,9 +89,9 @@ def fingerprint(logcan, sendcan, num_pandas):
 
     cached_params = params.get("CarParamsCache")
     if cached_params is not None:
-      cached_params = car.CarParams.from_bytes(cached_params)
-      if cached_params.carName == "mock":
-        cached_params = None
+      with car.CarParams.from_bytes(cached_params) as cached_params:
+        if cached_params.carName == "mock":
+          cached_params = None
 
     if cached_params is not None and len(cached_params.carFw) > 0 and \
        cached_params.carVin is not VIN_UNKNOWN and not disable_fw_cache:

selfdrive/controls/controlsd.py

@@ -206,8 +206,8 @@ class Controls:
     if REPLAY:
       controls_state = Params().get("ReplayControlsState")
       if controls_state is not None:
-        controls_state = log.ControlsState.from_bytes(controls_state)
-        self.v_cruise_helper.v_cruise_kph = controls_state.vCruise
+        with log.ControlsState.from_bytes(controls_state) as controls_state:
+          self.v_cruise_helper.v_cruise_kph = controls_state.vCruise
 
       if any(ps.controlsAllowed for ps in self.sm['pandaStates']):
         self.state = State.enabled

selfdrive/controls/plannerd.py

@@ -32,7 +32,8 @@ def plannerd_thread(sm=None, pm=None):
 
   cloudlog.info("plannerd is waiting for CarParams")
   params = Params()
-  CP = car.CarParams.from_bytes(params.get("CarParams", block=True))
+  with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg:
+    CP = msg
   cloudlog.info("plannerd got CarParams: %s", CP.carName)
 
   debug_mode = bool(int(os.getenv("DEBUG", "0")))

selfdrive/controls/radard.py

@@ -185,7 +185,8 @@ def radard_thread(sm=None, pm=None, can_sock=None):
 
   # wait for stats about the car to come in from controls
   cloudlog.info("radard is waiting for CarParams")
-  CP = car.CarParams.from_bytes(Params().get("CarParams", block=True))
+  with car.CarParams.from_bytes(Params().get("CarParams", block=True)) as msg:
+    CP = msg
   cloudlog.info("radard got CarParams")
 
   # import the radar from the fingerprint

selfdrive/debug/dump.py

@@ -41,7 +41,8 @@ if __name__ == "__main__":
     polld = poller.poll(100)
     for sock in polld:
       msg = sock.receive()
-      evt = log.Event.from_bytes(msg)
+      with log.Event.from_bytes(msg) as log_evt:
+        evt = log_evt
 
       if not args.no_print:
         if args.pipe:

selfdrive/debug/internal/measure_torque_time_to_max.py

@@ -34,7 +34,8 @@ if __name__ == "__main__":
     polld = poller.poll(1000)
     for sock in polld:
       msg = sock.receive()
-      evt = log.Event.from_bytes(msg)
+      with log.Event.from_bytes(msg) as log_evt:
+        evt = log_evt
 
       for item in evt.can:
         if item.address == 0xe4 and item.src == 128:

selfdrive/locationd/calibrationd.py

@@ -73,11 +73,11 @@ class Calibrator:
 
     if param_put and calibration_params:
       try:
-        msg = log.Event.from_bytes(calibration_params)
-        rpy_init = np.array(msg.liveCalibration.rpyCalib)
-        valid_blocks = msg.liveCalibration.validBlocks
-        wide_from_device_euler = np.array(msg.liveCalibration.wideFromDeviceEuler)
-        height = np.array(msg.liveCalibration.height)
+        with log.Event.from_bytes(calibration_params) as msg:
+          rpy_init = np.array(msg.liveCalibration.rpyCalib)
+          valid_blocks = msg.liveCalibration.validBlocks
+          wide_from_device_euler = np.array(msg.liveCalibration.wideFromDeviceEuler)
+          height = np.array(msg.liveCalibration.height)
       except Exception:
         cloudlog.exception("Error reading cached CalibrationParams")
 

selfdrive/locationd/laikad.py

@@ -116,9 +116,9 @@ class Laikad:
 
     nav_dict = {}
     try:
-      ephem_cache = ephemeris_structs.EphemerisCache.from_bytes(cache_bytes)
-      glonass_navs = [GLONASSEphemeris(data_struct, file_name=EPHEMERIS_CACHE) for data_struct in ephem_cache.glonassEphemerides]
-      gps_navs = [GPSEphemeris(data_struct, file_name=EPHEMERIS_CACHE) for data_struct in ephem_cache.gpsEphemerides]
+      with ephemeris_structs.EphemerisCache.from_bytes(cache_bytes) as ephem_cache:
+        glonass_navs = [GLONASSEphemeris(data_struct, file_name=EPHEMERIS_CACHE) for data_struct in ephem_cache.glonassEphemerides]
+        gps_navs = [GPSEphemeris(data_struct, file_name=EPHEMERIS_CACHE) for data_struct in ephem_cache.gpsEphemerides]
       for e in sum([glonass_navs, gps_navs], []):
         if e.prn not in nav_dict:
           nav_dict[e.prn] = []

selfdrive/locationd/paramsd.py

@@ -129,7 +129,8 @@ def main(sm=None, pm=None):
   params_reader = Params()
   # wait for stats about the car to come in from controls
   cloudlog.info("paramsd is waiting for CarParams")
-  CP = car.CarParams.from_bytes(params_reader.get("CarParams", block=True))
+  with car.CarParams.from_bytes(params_reader.get("CarParams", block=True)) as msg:
+    CP = msg
   cloudlog.info("paramsd got CarParams")
 
   min_sr, max_sr = 0.5 * CP.steerRatio, 2.0 * CP.steerRatio

selfdrive/locationd/test/_test_locationd_lib.py

@@ -40,7 +40,8 @@ void localizer_handle_msg_bytes(Localizer_t localizer, const char *data, size_t
 
   def localizer_get_msg(self, t=0, inputsOK=True, sensorsOK=True, gpsOK=True, msgValid=True):
     self.lib.localizer_get_message_bytes(self.localizer, inputsOK, sensorsOK, gpsOK, msgValid, self.ffi.addressof(self.msg_buff, 0), self.buff_size)
-    return log.Event.from_bytes(self.ffi.buffer(self.msg_buff), nesting_limit=self.buff_size // 8)
+    with log.Event.from_bytes(self.ffi.buffer(self.msg_buff), nesting_limit=self.buff_size // 8) as log_evt:
+      return log_evt
 
   def test_liblocalizer(self):
     msg = messaging.new_message('liveCalibration')

selfdrive/locationd/torqued.py

@@ -139,8 +139,10 @@ class TorqueEstimator:
     torque_cache = params.get("LiveTorqueParameters")
     if params_cache is not None and torque_cache is not None:
       try:
-        cache_ltp = log.Event.from_bytes(torque_cache).liveTorqueParameters
-        cache_CP = car.CarParams.from_bytes(params_cache)
+        with log.Event.from_bytes(torque_cache).liveTorqueParameters as log_evt:
+          cache_ltp = log_evt
+        with car.CarParams.from_bytes(params_cache) as msg:
+          cache_CP = msg
         if self.get_restore_key(cache_CP, cache_ltp.version) == self.get_restore_key(CP, VERSION):
           if cache_ltp.liveValid:
             initial_params = {
@@ -262,8 +264,8 @@ def main(sm=None, pm=None):
     pm = messaging.PubMaster(['liveTorqueParameters'])
 
   params = Params()
-  CP = car.CarParams.from_bytes(params.get("CarParams", block=True))
-  estimator = TorqueEstimator(CP)
+  with car.CarParams.from_bytes(params.get("CarParams", block=True)) as CP:
+    estimator = TorqueEstimator(CP)
 
   def cache_params(sig, frame):
     signal.signal(sig, signal.SIG_DFL)