DM: Large Donut model 🍩 (#36198)

* 59cfd731-6f80-4857-9271-10d952165079/225

* deprecate at the end

cereal/log.capnp

@@ -2146,13 +2146,10 @@ struct Joystick {
 struct DriverStateV2 {
   frameId @0 :UInt32;
   modelExecutionTime @1 :Float32;
-  dspExecutionTimeDEPRECATED @2 :Float32;
   gpuExecutionTime @8 :Float32;
   rawPredictions @3 :Data;
 
-  poorVisionProb @4 :Float32;
   wheelOnRightProb @5 :Float32;
-
   leftDriverData @6 :DriverData;
   rightDriverData @7 :DriverData;
 
@@ -2167,10 +2164,13 @@ struct DriverStateV2 {
     leftBlinkProb @7 :Float32;
     rightBlinkProb @8 :Float32;
     sunglassesProb @9 :Float32;
-    occludedProb @10 :Float32;
-    readyProb @11 :List(Float32);
     notReadyProb @12 :List(Float32);
+    occludedProbDEPRECATED @10 :Float32;
+    readyProbDEPRECATED @11 :List(Float32);
   }
+
+  dspExecutionTimeDEPRECATED @2 :Float32;
+  poorVisionProbDEPRECATED @4 :Float32;
 }
 
 struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {

selfdrive/modeld/dmonitoringmodeld.py

@@ -25,13 +25,13 @@ from openpilot.selfdrive.modeld.runners.tinygrad_helpers import qcom_tensor_from
 MODEL_WIDTH, MODEL_HEIGHT = DM_INPUT_SIZE
 CALIB_LEN = 3
 FEATURE_LEN = 512
-OUTPUT_SIZE = 84 + FEATURE_LEN
+OUTPUT_SIZE = 83 + FEATURE_LEN
 
 PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld"
 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 MODEL_PKL_PATH = Path(__file__).parent / 'models/dmonitoring_model_tinygrad.pkl'
 
-
+# TODO: slice from meta
 class DriverStateResult(ctypes.Structure):
   _fields_ = [
     ("face_orientation", ctypes.c_float*3),
@@ -46,8 +46,8 @@ class DriverStateResult(ctypes.Structure):
     ("left_blink_prob", ctypes.c_float),
     ("right_blink_prob", ctypes.c_float),
     ("sunglasses_prob", ctypes.c_float),
-    ("occluded_prob", ctypes.c_float),
-    ("ready_prob", ctypes.c_float*4),
+    ("_unused_c", ctypes.c_float),
+    ("_unused_d", ctypes.c_float*4),
     ("not_ready_prob", ctypes.c_float*2)]
 
 
@@ -55,7 +55,6 @@ class DMonitoringModelResult(ctypes.Structure):
   _fields_ = [
     ("driver_state_lhd", DriverStateResult),
     ("driver_state_rhd", DriverStateResult),
-    ("poor_vision_prob", ctypes.c_float),
     ("wheel_on_right_prob", ctypes.c_float),
     ("features", ctypes.c_float*FEATURE_LEN)]
 
@@ -107,8 +106,6 @@ def fill_driver_state(msg, ds_result: DriverStateResult):
   msg.leftBlinkProb = float(sigmoid(ds_result.left_blink_prob))
   msg.rightBlinkProb = float(sigmoid(ds_result.right_blink_prob))
   msg.sunglassesProb = float(sigmoid(ds_result.sunglasses_prob))
-  msg.occludedProb = float(sigmoid(ds_result.occluded_prob))
-  msg.readyProb = [float(sigmoid(x)) for x in ds_result.ready_prob]
   msg.notReadyProb = [float(sigmoid(x)) for x in ds_result.not_ready_prob]
 
 
@@ -119,7 +116,6 @@ def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts:
   ds.frameId = frame_id
   ds.modelExecutionTime = execution_time
   ds.gpuExecutionTime = gpu_execution_time
-  ds.poorVisionProb = float(sigmoid(model_result.poor_vision_prob))
   ds.wheelOnRightProb = float(sigmoid(model_result.wheel_on_right_prob))
   ds.rawPredictions = model_output.tobytes() if SEND_RAW_PRED else b''
   fill_driver_state(ds.leftDriverData, model_result.driver_state_lhd)

selfdrive/modeld/models/README.md

@@ -62,6 +62,5 @@ Refer to **slice_outputs** and **parse_vision_outputs/parse_policy_outputs** in
     * (deprecated) distracted probabilities: 2
     * using phone probability: 1
     * distracted probability: 1
-  * common outputs 2
-    * poor camera vision probability: 1
+  * common outputs 1
     * left hand drive probability: 1

selfdrive/modeld/models/dmonitoring_model.current

@@ -1,2 +0,0 @@
-fa69be01-b430-4504-9d72-7dcb058eb6dd
-d9fb22d1c4fa3ca3d201dbc8edf1d0f0918e53e6

selfdrive/modeld/models/dmonitoring_model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50efe6451a3fb3fa04b6bb0e846544533329bd46ecefe9e657e91214dee2aaeb
-size 7196502
+oid sha256:9b2117ee4907add59e3fbe6829cda74e0ad71c0835b0ebb9373ba9425de0d336
+size 7191776

selfdrive/monitoring/helpers.py

@@ -38,8 +38,6 @@ class DRIVER_MONITOR_SETTINGS:
     self._EE_THRESH12 = 15.0
     self._EE_MAX_OFFSET1 = 0.06
     self._EE_MIN_OFFSET1 = 0.025
-    self._EE_THRESH21 = 0.01
-    self._EE_THRESH22 = 0.35
 
     self._POSE_PITCH_THRESHOLD = 0.3133
     self._POSE_PITCH_THRESHOLD_SLACK = 0.3237
@@ -137,11 +135,8 @@ class DriverMonitoring:
     self.pose = DriverPose(self.settings._POSE_OFFSET_MAX_COUNT)
     self.blink = DriverBlink()
     self.eev1 = 0.
-    self.eev2 = 1.
     self.ee1_offseter = RunningStatFilter(max_trackable=self.settings._POSE_OFFSET_MAX_COUNT)
-    self.ee2_offseter = RunningStatFilter(max_trackable=self.settings._POSE_OFFSET_MAX_COUNT)
     self.ee1_calibrated = False
-    self.ee2_calibrated = False
 
     self.always_on = always_on
     self.distracted_types = []
@@ -262,7 +257,7 @@ class DriverMonitoring:
     driver_data = driver_state.rightDriverData if self.wheel_on_right else driver_state.leftDriverData
     if not all(len(x) > 0 for x in (driver_data.faceOrientation, driver_data.facePosition,
                                     driver_data.faceOrientationStd, driver_data.facePositionStd,
-                                    driver_data.readyProb, driver_data.notReadyProb)):
+                                    driver_data.notReadyProb)):
       return
 
     self.face_detected = driver_data.faceProb > self.settings._FACE_THRESHOLD
@@ -279,7 +274,6 @@ class DriverMonitoring:
     self.blink.right = driver_data.rightBlinkProb * (driver_data.rightEyeProb > self.settings._EYE_THRESHOLD) \
                                                                   * (driver_data.sunglassesProb < self.settings._SG_THRESHOLD)
     self.eev1 = driver_data.notReadyProb[0]
-    self.eev2 = driver_data.readyProb[0]
 
     self.distracted_types = self._get_distracted_types()
     self.driver_distracted = (DistractedType.DISTRACTED_E2E in self.distracted_types or DistractedType.DISTRACTED_POSE in self.distracted_types
@@ -293,12 +287,10 @@ class DriverMonitoring:
       self.pose.pitch_offseter.push_and_update(self.pose.pitch)
       self.pose.yaw_offseter.push_and_update(self.pose.yaw)
       self.ee1_offseter.push_and_update(self.eev1)
-      self.ee2_offseter.push_and_update(self.eev2)
 
     self.pose.calibrated = self.pose.pitch_offseter.filtered_stat.n > self.settings._POSE_OFFSET_MIN_COUNT and \
                                        self.pose.yaw_offseter.filtered_stat.n > self.settings._POSE_OFFSET_MIN_COUNT
     self.ee1_calibrated = self.ee1_offseter.filtered_stat.n > self.settings._POSE_OFFSET_MIN_COUNT
-    self.ee2_calibrated = self.ee2_offseter.filtered_stat.n > self.settings._POSE_OFFSET_MIN_COUNT
 
     self.is_model_uncertain = self.hi_stds > self.settings._HI_STD_FALLBACK_TIME
     self._set_timers(self.face_detected and not self.is_model_uncertain)

selfdrive/monitoring/test_monitoring.py

@@ -25,7 +25,6 @@ def make_msg(face_detected, distracted=False, model_uncertain=False):
   ds.leftDriverData.faceOrientationStd = [1.*model_uncertain, 1.*model_uncertain, 1.*model_uncertain]
   ds.leftDriverData.facePositionStd = [1.*model_uncertain, 1.*model_uncertain]
   # TODO: test both separately when e2e is used
-  ds.leftDriverData.readyProb = [0., 0., 0., 0.]
   ds.leftDriverData.notReadyProb = [0., 0.]
   return ds