EfficientNet driver monitoring (#1907)

* e96f9be6

* bump cereal

* filter sunglasses

* fix unittest

* update refs

* udpate refs

* update refs

* add tolerance for dmonitoringd

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 2cfa0edb4c

models/dmonitoring_model.current

@@ -1 +1 @@
-43221d85-46fd-40b9-bff0-2b1b18a86b07
+e96f9be6-5741-42ea-bdcd-0be6515b4230

models/dmonitoring_model.keras

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c39a2096f7058541b5339ec36bc4c468955e67285078080ed6d8802fed06c1d
-size 814176
+oid sha256:09aa11a17a5a8173e231071898c499f9ea632e6e64285586122828b1bbc70d41
+size 4165968

models/dmonitoring_model_q.dlc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29504dfd101ba2a0b48550fac2f86f9d0b8d1245af3d2d8d658247b4a73077a2
-size 230121
+oid sha256:beecf140ddc5da96cbdae3b869ebb3f5453dcd8e61e09d7d079c91e006b6df98
+size 1134208

selfdrive/modeld/models/dmonitoring.cc

@@ -5,9 +5,9 @@
 
 #include <libyuv.h>
 
-#define MODEL_WIDTH 160
-#define MODEL_HEIGHT 320
-#define FULL_W 426
+#define MODEL_WIDTH 320
+#define MODEL_HEIGHT 640
+#define FULL_W 852
 
 #if defined(QCOM) || defined(QCOM2)
 #define input_lambda(x) (x - 128.f) * 0.0078125f
@@ -136,6 +136,7 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
   memcpy(&ret.right_eye_prob, &s->output[30], sizeof ret.right_eye_prob);
   memcpy(&ret.left_blink_prob, &s->output[31], sizeof ret.right_eye_prob);
   memcpy(&ret.right_blink_prob, &s->output[32], sizeof ret.right_eye_prob);
+  memcpy(&ret.sg_prob, &s->output[33], sizeof ret.sg_prob);
   ret.face_orientation_meta[0] = softplus(ret.face_orientation_meta[0]);
   ret.face_orientation_meta[1] = softplus(ret.face_orientation_meta[1]);
   ret.face_orientation_meta[2] = softplus(ret.face_orientation_meta[2]);
@@ -166,6 +167,7 @@ void dmonitoring_publish(PubMaster &pm, uint32_t frame_id, const DMonitoringResu
   framed.setRightEyeProb(res.right_eye_prob);
   framed.setLeftBlinkProb(res.left_blink_prob);
   framed.setRightBlinkProb(res.right_blink_prob);
+  framed.setSgProb(res.sg_prob);
 
   pm.send("driverState", msg);
 }

selfdrive/modeld/models/dmonitoring.h

@@ -9,7 +9,7 @@
 extern "C" {
 #endif
 
-#define OUTPUT_SIZE 33
+#define OUTPUT_SIZE 34
 #define RHD_CHECK_INTERVAL 10
 
 typedef struct DMonitoringResult {
@@ -22,6 +22,7 @@ typedef struct DMonitoringResult {
   float right_eye_prob;
   float left_blink_prob;
   float right_blink_prob;
+  float sg_prob;
 } DMonitoringResult;
 
 typedef struct DMonitoringModelState {

selfdrive/monitoring/driver_monitor.py

@@ -21,8 +21,9 @@ _DISTRACTED_TIME = 11.
 _DISTRACTED_PRE_TIME_TILL_TERMINAL = 8.
 _DISTRACTED_PROMPT_TIME_TILL_TERMINAL = 6.
 
-_FACE_THRESHOLD = 0.4
+_FACE_THRESHOLD = 0.6
 _EYE_THRESHOLD = 0.6
+_SG_THRESHOLD = 0.5
 _BLINK_THRESHOLD = 0.5  # 0.225
 _BLINK_THRESHOLD_SLACK = 0.65
 _BLINK_THRESHOLD_STRICT = 0.5
@@ -189,8 +190,8 @@ class DriverStatus():
     # self.pose.roll_std = driver_state.faceOrientationStd[2]
     model_std_max = max(self.pose.pitch_std, self.pose.yaw_std)
     self.pose.low_std = model_std_max < _POSESTD_THRESHOLD
-    self.blink.left_blink = driver_state.leftBlinkProb * (driver_state.leftEyeProb > _EYE_THRESHOLD)
-    self.blink.right_blink = driver_state.rightBlinkProb * (driver_state.rightEyeProb > _EYE_THRESHOLD)
+    self.blink.left_blink = driver_state.leftBlinkProb * (driver_state.leftEyeProb > _EYE_THRESHOLD) * (driver_state.sgProb < _SG_THRESHOLD)
+    self.blink.right_blink = driver_state.rightBlinkProb * (driver_state.rightEyeProb > _EYE_THRESHOLD) * (driver_state.sgProb < _SG_THRESHOLD)
     self.face_detected = driver_state.faceProb > _FACE_THRESHOLD and \
                           abs(driver_state.facePosition[0]) <= 0.4 and abs(driver_state.facePosition[1]) <= 0.45
 

selfdrive/monitoring/test_monitoring.py

@@ -31,6 +31,7 @@ class fake_DM_msg():
     self.rightBlinkProb = 1. * is_distracted
     self.faceOrientationStd = [1.*is_model_uncertain, 1.*is_model_uncertain, 1.*is_model_uncertain]
     self.facePositionStd = [1.*is_model_uncertain, 1.*is_model_uncertain]
+    self.sgProb = 0.
 
 
 # driver state from neural net, 10Hz

selfdrive/test/process_replay/process_replay.py

@@ -262,7 +262,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid"],
     init_callback=get_car_params,
     should_recv_callback=None,
-    tolerance=None,
+    tolerance=1e-7,
   ),
   ProcessConfig(
     proc_name="locationd",

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-b4cf3ec3c29ef22741d5a85d50abc0937b5d40bd
+f3261f36402ad9bc3003feb9de61a04a634f30e1