QT UI: sounds (#2078)

* move android into own dir

* fix name

* maybe this works? qt ui doesn't work on mac

* fix that

* pc sound works

* fix pc build

* lowercase

* that needs to be real_arch

* split into classes

* fix typo in lib

* Fix cycle alerts

* Add qt multimedia libs to install scripts

* Add ui/android folder

* Fix android build

* Raise exception if sound init fails

* add missing return

Co-authored-by: Willem Melching <willem.melching@gmail.com>
Co-authored-by: Comma Device <device@comma.ai>
old-commit-hash: acd1bde496904fbc0d97f4d89e630358c4d11f7f

Dockerfile.openpilot_base

@@ -38,6 +38,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     python-dev \
     python-pip \
     qt5-default \
+    qtmultimedia5-dev \
     sudo \
     wget \
   && rm -rf /var/lib/apt/lists/*

SConstruct

@@ -190,6 +190,7 @@ if arch in ["x86_64", "Darwin", "larch64"]:
       QT_BASE + "include/QtGui",
       QT_BASE + "include/QtCore",
       QT_BASE + "include/QtDBus",
+      QT_BASE + "include/QtMultimedia",
     ]
     qt_env["LINKFLAGS"] += ["-F" + QT_BASE + "lib"]
   else:
@@ -199,6 +200,7 @@ if arch in ["x86_64", "Darwin", "larch64"]:
       f"/usr/include/{real_arch}-linux-gnu/qt5/QtGui",
       f"/usr/include/{real_arch}-linux-gnu/qt5/QtCore",
       f"/usr/include/{real_arch}-linux-gnu/qt5/QtDBus",
+      f"/usr/include/{real_arch}-linux-gnu/qt5/QtMultimedia",
     ]
 
   qt_env.Tool('qt')

release/files_common

@@ -350,6 +350,9 @@ selfdrive/ui/text/text.c
 selfdrive/ui/qt/*.cc
 selfdrive/ui/qt/*.hpp
 
+selfdrive/ui/android/*.cc
+selfdrive/ui/android/*.hpp
+
 selfdrive/camerad/SConscript
 selfdrive/camerad/main.cc
 selfdrive/camerad/bufs.h

selfdrive/debug/cycle_alerts.py

@@ -7,52 +7,63 @@ import argparse
 import time
 
 import cereal.messaging as messaging
-from selfdrive.controls.lib.events import EVENTS, Alert
+from selfdrive.car.honda.interface import CarInterface
+from selfdrive.controls.lib.events import ET, EVENTS, Alert, Events
+from selfdrive.controls.lib.alertmanager import AlertManager
 
-def now_millis(): return time.time() * 1000
 
-ALERTS = [a for _, et in EVENTS.items() for _, a in et.items() if isinstance(a, Alert)]
+def cycle_alerts(duration=200, is_metric=False):
+  alerts = list(EVENTS.keys())
+  print(alerts)
 
-#from cereal import car
-#ALERTS = [a for a in ALERTS if a.audible_alert == car.CarControl.HUDControl.AudibleAlert.chimeWarningRepeat]
-
-default_alerts = sorted(ALERTS, key=lambda alert: (alert.alert_size, len(alert.alert_text_2)))
-
-def cycle_alerts(duration_millis, alerts=None):
-  if alerts is None:
-    alerts = default_alerts
+  CP = CarInterface.get_params("HONDA CIVIC 2016 TOURING")
+  sm = messaging.SubMaster(['thermal', 'health', 'frame', 'model', 'liveCalibration',
+                            'dMonitoringState', 'plan', 'pathPlan', 'liveLocationKalman'])
 
   controls_state = messaging.pub_sock('controlsState')
+  thermal = messaging.pub_sock('thermal')
 
   idx, last_alert_millis = 0, 0
   alert = alerts[0]
+
+  events = Events()
+  AM = AlertManager()
+
+  frame = 0
+
   while 1:
-    if (now_millis() - last_alert_millis) > duration_millis:
-      alert = alerts[idx]
+    if frame % duration == 0:
       idx = (idx + 1) % len(alerts)
-      last_alert_millis = now_millis()
-      print('sending {}'.format(str(alert)))
+      events.clear()
+      events.add(alerts[idx])
+
+
+    current_alert_types = [ET.PERMANENT, ET.USER_DISABLE, ET.IMMEDIATE_DISABLE,
+                           ET.SOFT_DISABLE, ET.PRE_ENABLE, ET.NO_ENTRY,
+                           ET.ENABLE, ET.WARNING]
+    a = events.create_alerts(current_alert_types, [CP, sm, is_metric])
+    AM.add_many(frame, a)
+    AM.process_alerts(frame)
 
     dat = messaging.new_message()
     dat.init('controlsState')
 
-    dat.controlsState.alertType = alert.alert_type
-    dat.controlsState.alertText1 = alert.alert_text_1
-    dat.controlsState.alertText2 = alert.alert_text_2
-    dat.controlsState.alertSize = alert.alert_size
-    #dat.controlsState.alertStatus = alert.alert_status
-    dat.controlsState.alertSound = alert.audible_alert
+    dat.controlsState.alertText1 = AM.alert_text_1
+    dat.controlsState.alertText2 = AM.alert_text_2
+    dat.controlsState.alertSize = AM.alert_size
+    dat.controlsState.alertStatus = AM.alert_status
+    dat.controlsState.alertBlinkingRate = AM.alert_rate
+    dat.controlsState.alertType = AM.alert_type
+    dat.controlsState.alertSound = AM.audible_alert
     controls_state.send(dat.to_bytes())
 
+    dat = messaging.new_message()
+    dat.init('thermal')
+    dat.thermal.started = True
+    thermal.send(dat.to_bytes())
+
+    frame += 1
     time.sleep(0.01)
 
 if __name__ == '__main__':
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--duration', type=int, default=1000)
-  parser.add_argument('--alert-types', nargs='+')
-  args = parser.parse_args()
-  alerts = None
-  if args.alert_types:
-    alerts = [next(a for a in ALERTS if a.alert_type==alert_type) for alert_type in args.alert_types]
-
-  cycle_alerts(args.duration, alerts=alerts)
+  cycle_alerts()

selfdrive/ui/SConscript

@@ -5,11 +5,10 @@ libs = [common, 'zmq', 'czmq', 'capnp', 'kj', 'm', cereal, messaging, gpucommon,
 
 
 if qt_env is None:
-  src += ['sound.cc']
   libs += ['EGL', 'GLESv3', 'gnustl_shared', 'log', 'utils', 'gui', 'hardware', 'ui', 'CB', 'gsl', 'adreno_utils', 'OpenSLES', 'cutils', 'uuid', 'OpenCL']
   linkflags = ['-Wl,-rpath=/system/lib64,-rpath=/system/comma/usr/lib']
 
-  src = ["android_ui.cc"] + src
+  src += ["android/ui.cc", "android/sl_sound.cc"]
   env.Program('_ui', src,
     LINKFLAGS=linkflags,
     LIBS=libs)
@@ -18,14 +17,14 @@ else:
   qt_libs = ["pthread"]
 
   if arch == "Darwin":
-    qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus"]
+    qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus", "QtMultimedia"]
   else:
-    qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus"]
+    qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus", "Qt5Multimedia"]
 
   if arch == "larch64":
     qt_libs += ["GLESv2"]
   else:
     qt_libs += ["GL"]
 
-  qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc"] + src
+  qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc", "qt/qt_sound.cc"] + src
   qt_env.Program("_ui", qt_src, LIBS=qt_libs + libs)

selfdrive/ui/android/sl_sound.cc

@@ -1,35 +1,31 @@
-
-#include "sound.hpp"
 #include <math.h>
 #include <stdlib.h>
 #include <atomic>
 #include "common/swaglog.h"
 #include "common/timing.h"
 
+#include "android/sl_sound.hpp"
+
 #define LogOnError(func, msg) \
   if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); }
 
 #define ReturnOnError(func, msg) \
   if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); return false; }
 
-static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
-    {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}},
-    {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}},
-    {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}},
-    {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}},
-    {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}},
-    {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}},
-    {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}},
-    {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}};
-
-struct Sound::Player {
+struct SLSound::Player {
   SLObjectItf player;
   SLPlayItf playItf;
   // slplay_callback runs on a background thread,use atomic to ensure thread safe.
   std::atomic<int> repeat;
 };
 
-bool Sound::init(int volume) {
+SLSound::SLSound() {
+  if (!init()){
+    throw std::runtime_error("Failed to initialize sound");
+  }
+}
+
+bool SLSound::init() {
   SLEngineOption engineOptions[] = {{SL_ENGINEOPTION_THREADSAFE, SL_BOOLEAN_TRUE}};
   const SLInterfaceID ids[1] = {SL_IID_VOLUME};
   const SLboolean req[1] = {SL_BOOLEAN_FALSE};
@@ -54,15 +50,13 @@ bool Sound::init(int volume) {
     ReturnOnError((*player)->GetInterface(player, SL_IID_PLAY, &playItf), "Failed to get player interface");
     ReturnOnError((*playItf)->SetPlayState(playItf, SL_PLAYSTATE_PAUSED), "Failed to initialize playstate to SL_PLAYSTATE_PAUSED");
 
-    player_[kv.first] = new Sound::Player{player, playItf};
+    player_[kv.first] = new SLSound::Player{player, playItf};
   }
-
-  setVolume(volume);
   return true;
 }
 
 void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event) {
-  Sound::Player *s = reinterpret_cast<Sound::Player *>(context);
+  SLSound::Player *s = reinterpret_cast<SLSound::Player *>(context);
   if (event == SL_PLAYEVENT_HEADATEND && s->repeat > 1) {
     --s->repeat;
     (*playItf)->SetPlayState(playItf, SL_PLAYSTATE_STOPPED);
@@ -71,7 +65,7 @@ void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event
   }
 }
 
-bool Sound::play(AudibleAlert alert) {
+bool SLSound::play(AudibleAlert alert) {
   if (currentSound_ != AudibleAlert::NONE) {
     stop();
   }
@@ -93,7 +87,7 @@ bool Sound::play(AudibleAlert alert) {
   return true;
 }
 
-void Sound::stop() {
+void SLSound::stop() {
   if (currentSound_ != AudibleAlert::NONE) {
     auto player = player_.at(currentSound_);
     player->repeat = 0;
@@ -102,9 +96,9 @@ void Sound::stop() {
   }
 }
 
-void Sound::setVolume(int volume) {
+void SLSound::setVolume(int volume) {
   if (last_volume_ == volume) return;
-  
+
   double current_time = nanos_since_boot();
   if ((current_time - last_set_volume_time_) > (5 * (1e+9))) { // 5s timeout on updating the volume
     char volume_change_cmd[64];
@@ -115,11 +109,15 @@ void Sound::setVolume(int volume) {
   }
 }
 
-Sound::~Sound() {
+SLSound::~SLSound() {
   for (auto &kv : player_) {
     (*(kv.second->player))->Destroy(kv.second->player);
     delete kv.second;
   }
-  if (outputMix_) (*outputMix_)->Destroy(outputMix_);
-  if (engine_) (*engine_)->Destroy(engine_);
+  if (outputMix_) {
+    (*outputMix_)->Destroy(outputMix_);
+  }
+  if (engine_) {
+    (*engine_)->Destroy(engine_);
+  }
 }

selfdrive/ui/android/sl_sound.hpp

@@ -0,0 +1,26 @@
+#pragma once
+#include <SLES/OpenSLES.h>
+#include <SLES/OpenSLES_Android.h>
+
+#include "sound.hpp"
+
+
+class SLSound : public Sound {
+public:
+  SLSound();
+  ~SLSound();
+  bool play(AudibleAlert alert);
+  void stop();
+  void setVolume(int volume);
+
+private:
+  bool init();
+  SLObjectItf engine_ = nullptr;
+  SLObjectItf outputMix_ = nullptr;
+  int last_volume_ = 0;
+  double last_set_volume_time_ = 0.;
+  AudibleAlert currentSound_ = AudibleAlert::NONE;
+  struct Player;
+  std::map<AudibleAlert, Player *> player_;
+  friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event);
+};

selfdrive/ui/android/ui.cc

@@ -12,6 +12,7 @@
 
 #include "ui.hpp"
 #include "paint.hpp"
+#include "android/sl_sound.hpp"
 
 // Includes for light sensor
 #include <cutils/properties.h>
@@ -169,10 +170,13 @@ int main(int argc, char* argv[]) {
   setpriority(PRIO_PROCESS, 0, -14);
 
   signal(SIGINT, (sighandler_t)set_do_exit);
+  SLSound sound;
 
   UIState uistate = {};
   UIState *s = &uistate;
   ui_init(s);
+  s->sound = &sound;
+
   set_awake(s, true);
   enable_event_processing(true);
 
@@ -201,7 +205,7 @@ int main(int argc, char* argv[]) {
 
   const int MIN_VOLUME = LEON ? 12 : 9;
   const int MAX_VOLUME = LEON ? 15 : 12;
-  assert(s->sound.init(MIN_VOLUME));
+  s->sound->setVolume(MIN_VOLUME);
 
   while (!do_exit) {
     if (!s->started || !s->vision_connected) {
@@ -238,7 +242,7 @@ int main(int argc, char* argv[]) {
     }
 
     // up one notch every 5 m/s
-    s->sound.setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5));
+    s->sound->setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5));
 
     // set brightness
     float clipped_brightness = fmin(512, (s->light_sensor*brightness_m) + brightness_b);

selfdrive/ui/qt/qt_sound.cc

@@ -0,0 +1,29 @@
+#include <QUrl>
+#include "qt/qt_sound.hpp"
+
+QtSound::QtSound() {
+  for (auto &kv : sound_map) {
+    auto path = QUrl::fromLocalFile(kv.second.first);
+    sounds[kv.first].setSource(path);
+  }
+}
+
+bool QtSound::play(AudibleAlert alert) {
+  sounds[alert].setLoopCount(sound_map[alert].second);
+  sounds[alert].play();
+  return true;
+}
+
+void QtSound::stop() {
+  for (auto &kv : sounds) {
+    kv.second.stop();
+  }
+}
+
+void QtSound::setVolume(int volume) {
+  // TODO: implement this
+}
+
+QtSound::~QtSound() {
+
+}

selfdrive/ui/qt/qt_sound.hpp

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <QSoundEffect>
+#include "sound.hpp"
+
+class QtSound : public Sound {
+public:
+  QtSound();
+  ~QtSound();
+  bool play(AudibleAlert alert);
+  void stop();
+  void setVolume(int volume);
+
+private:
+  std::map<AudibleAlert, QSoundEffect> sounds;
+};

selfdrive/ui/qt/window.cc

@@ -13,7 +13,6 @@
 #include "settings.hpp"
 
 #include "paint.hpp"
-#include "sound.hpp"
 
 volatile sig_atomic_t do_exit = 0;
 
@@ -40,11 +39,11 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
   )");
 }
 
-void MainWindow::openSettings(){
+void MainWindow::openSettings() {
   main_layout->setCurrentIndex(1);
 }
 
-void MainWindow::closeSettings(){
+void MainWindow::closeSettings() {
   main_layout->setCurrentIndex(0);
 }
 
@@ -52,6 +51,7 @@ void MainWindow::closeSettings(){
 GLWindow::GLWindow(QWidget *parent) : QOpenGLWidget(parent) {
   timer = new QTimer(this);
   QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate()));
+
 }
 
 GLWindow::~GLWindow() {
@@ -68,6 +68,7 @@ void GLWindow::initializeGL() {
 
   ui_state = new UIState();
   ui_init(ui_state);
+  ui_state->sound = &sound;
   ui_state->fb_w = vwp_w;
   ui_state->fb_h = vwp_h;
 
@@ -103,13 +104,6 @@ void GLWindow::mousePressEvent(QMouseEvent *e) {
 }
 
 
-/* HACKS */
-bool Sound::init(int volume) { return true; }
-bool Sound::play(AudibleAlert alert) { printf("play sound: %d\n", (int)alert); return true; }
-void Sound::stop() {}
-void Sound::setVolume(int volume) {}
-Sound::~Sound() {}
-
 GLuint visionimg_to_gl(const VisionImg *img, EGLImageKHR *pkhr, void **pph) {
   unsigned int texture;
   glGenTextures(1, &texture);

selfdrive/ui/qt/window.hpp

@@ -7,6 +7,7 @@
 #include <QOpenGLFunctions>
 #include <QStackedLayout>
 
+#include "qt/qt_sound.hpp"
 #include "ui/ui.hpp"
 
 class MainWindow : public QWidget
@@ -45,6 +46,7 @@ protected:
 private:
   QTimer * timer;
   UIState * ui_state;
+  QtSound sound;
 
 public slots:
   void timerUpdate();

selfdrive/ui/sound.hpp

@@ -2,31 +2,23 @@
 #include <map>
 #include "cereal/gen/cpp/log.capnp.h"
 
-#if defined(QCOM)
-#include <SLES/OpenSLES.h>
-#include <SLES/OpenSLES_Android.h>
-#endif
-
 typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert;
 
-class Sound {
- public:
-  Sound() = default;
-  bool init(int volume);
-  bool play(AudibleAlert alert);
-  void stop();
-  void setVolume(int volume);
-  ~Sound();
-
-#if defined(QCOM)
- private:
-  SLObjectItf engine_ = nullptr;
-  SLObjectItf outputMix_ = nullptr;
-  int last_volume_ = 0;
-  double last_set_volume_time_ = 0.;
-  AudibleAlert currentSound_ = AudibleAlert::NONE;
-  struct Player;
-  std::map<AudibleAlert, Player *> player_;
-  friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event);
-#endif
+static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
+  // AudibleAlert, (file path, loop count)
+  {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}},
+  {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}},
+  {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}},
+  {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}},
+  {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}},
+  {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}},
+  {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}},
+  {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}
+};
+
+class Sound {
+public:
+  virtual bool play(AudibleAlert alert) = 0;
+  virtual void stop() = 0;
+  virtual void setVolume(int volume) = 0;
 };

selfdrive/ui/ui.cc

@@ -130,9 +130,9 @@ void update_sockets(UIState *s) {
     auto alert_sound = scene.controls_state.getAlertSound();
     if (scene.alert_type.compare(scene.controls_state.getAlertType()) != 0) {
       if (alert_sound == AudibleAlert::NONE) {
-        s->sound.stop();
+        s->sound->stop();
       } else {
-        s->sound.play(alert_sound);
+        s->sound->play(alert_sound);
       }
     }
     scene.alert_text1 = scene.controls_state.getAlertText1();
@@ -254,7 +254,7 @@ void ui_update(UIState *s) {
     } else {
       // car is started, but controls is lagging or died
       if (s->scene.alert_text2 != "Controls Unresponsive") {
-        s->sound.play(AudibleAlert::CHIME_WARNING_REPEAT);
+        s->sound->play(AudibleAlert::CHIME_WARNING_REPEAT);
         LOGE("Controls unresponsive");
       }
 

selfdrive/ui/ui.hpp

@@ -174,7 +174,7 @@ typedef struct UIState {
 
   SubMaster *sm;
 
-  Sound sound;
+  Sound *sound;
   UIStatus status;
   UIScene scene;
   cereal::UiLayoutState::App active_app;

tools/ubuntu_setup.sh

@@ -43,6 +43,7 @@ sudo apt-get update && sudo apt-get install -y \
     python-dev \
     python-pip \
     qt5-default \
+    qtmultimedia5-dev \
     screen \
     sudo \
     vim \