Revert "ui: single-threaded CameraView (#32291)"

This reverts commit 6adb53abf4d483e70bbe36725588a05cf75681fb.

old-commit-hash: c9531b463a
This commit is contained in:
Adeeb Shihadeh 2024-05-18 16:34:03 -07:00
parent bf22381107
commit 72b6504595
9 changed files with 196 additions and 136 deletions

View File

@ -7,7 +7,7 @@
const int FACE_IMG_SIZE = 130;
DriverViewWindow::DriverViewWindow(QWidget* parent) : CameraView("camerad", VISION_STREAM_DRIVER, true, parent) {
DriverViewWindow::DriverViewWindow(QWidget* parent) : CameraWidget("camerad", VISION_STREAM_DRIVER, true, parent) {
face_img = loadPixmap("../assets/img_driver_face_static.png", {FACE_IMG_SIZE, FACE_IMG_SIZE});
QObject::connect(this, &CameraWidget::clicked, this, &DriverViewWindow::done);
QObject::connect(device(), &Device::interactiveTimeout, this, [this]() {
@ -20,20 +20,22 @@ DriverViewWindow::DriverViewWindow(QWidget* parent) : CameraView("camerad", VISI
void DriverViewWindow::showEvent(QShowEvent* event) {
params.putBool("IsDriverViewEnabled", true);
device()->resetInteractiveTimeout(60);
CameraView::showEvent(event);
CameraWidget::showEvent(event);
}
void DriverViewWindow::hideEvent(QHideEvent* event) {
params.putBool("IsDriverViewEnabled", false);
CameraView::hideEvent(event);
stopVipcThread();
CameraWidget::hideEvent(event);
}
void DriverViewWindow::paintGL() {
CameraView::paintGL();
CameraWidget::paintGL();
std::lock_guard lk(frame_lock);
QPainter p(this);
// startup msg
if (recent_frames.empty()) {
if (frames.empty()) {
p.setPen(Qt::white);
p.setRenderHint(QPainter::TextAntialiasing);
p.setFont(InterFont(100, QFont::Bold));
@ -45,6 +47,7 @@ void DriverViewWindow::paintGL() {
cereal::DriverStateV2::Reader driver_state = sm["driverStateV2"].getDriverStateV2();
bool is_rhd = driver_state.getWheelOnRightProb() > 0.5;
auto driver_data = is_rhd ? driver_state.getRightDriverData() : driver_state.getLeftDriverData();
bool face_detected = driver_data.getFaceProb() > 0.7;
if (face_detected) {
auto fxy_list = driver_data.getFacePosition();

View File

@ -2,7 +2,7 @@
#include "selfdrive/ui/qt/widgets/cameraview.h"
class DriverViewWindow : public CameraView {
class DriverViewWindow : public CameraWidget {
Q_OBJECT
public:

View File

@ -6,11 +6,11 @@
#include <cmath>
#include "common/swaglog.h"
#include "selfdrive/ui/qt/onroad/buttons.h"
#include "selfdrive/ui/qt/util.h"
// Window that shows camera view and variety of info drawn on top
AnnotatedCameraWidget::AnnotatedCameraWidget(VisionStreamType type, QWidget *parent)
: fps_filter(UI_FREQ, 3, 1. / UI_FREQ), CameraWidget("camerad", type, true, parent) {
AnnotatedCameraWidget::AnnotatedCameraWidget(VisionStreamType type, QWidget* parent) : fps_filter(UI_FREQ, 3, 1. / UI_FREQ), CameraWidget("camerad", type, true, parent) {
pm = std::make_unique<PubMaster, const std::initializer_list<const char *>>({"uiDebug"});
main_layout = new QVBoxLayout(this);
@ -76,8 +76,6 @@ void AnnotatedCameraWidget::updateState(const UIState &s) {
map_settings_btn->setVisible(!hideBottomIcons);
main_layout->setAlignment(map_settings_btn, (rightHandDM ? Qt::AlignLeft : Qt::AlignRight) | Qt::AlignBottom);
}
update();
}
void AnnotatedCameraWidget::drawHud(QPainter &p) {
@ -355,8 +353,25 @@ void AnnotatedCameraWidget::paintGL() {
UIState *s = uiState();
SubMaster &sm = *(s->sm);
const double start_draw_t = millis_since_boot();
const cereal::ModelDataV2::Reader &model = sm["modelV2"].getModelV2();
// draw camera frame
{
std::lock_guard lk(frame_lock);
if (frames.empty()) {
if (skip_frame_count > 0) {
skip_frame_count--;
qDebug() << "skipping frame, not ready";
return;
}
} else {
// skip drawing up to this many frames if we're
// missing camera frames. this smooths out the
// transitions from the narrow and wide cameras
skip_frame_count = 5;
}
// Wide or narrow cam dependent on speed
bool has_wide_cam = available_streams.count(VISION_STREAM_WIDE_ROAD);
if (has_wide_cam) {
@ -372,16 +387,14 @@ void AnnotatedCameraWidget::paintGL() {
}
CameraWidget::setStreamType(wide_cam_requested ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD);
s->scene.wide_cam = CameraWidget::streamType() == VISION_STREAM_WIDE_ROAD;
s->scene.wide_cam = CameraWidget::getStreamType() == VISION_STREAM_WIDE_ROAD;
if (s->scene.calibration_valid) {
auto calib = s->scene.wide_cam ? s->scene.view_from_wide_calib : s->scene.view_from_calib;
CameraWidget::updateCalibration(calib);
} else {
CameraWidget::updateCalibration(DEFAULT_CALIBRATION);
}
// Draw the frame based on the UI plan's frame ID
CameraWidget::setFrameId(sm["uiPlan"].getUiPlan().getFrameId());
CameraWidget::setFrameId(model.getFrameId());
CameraWidget::paintGL();
}
@ -390,7 +403,6 @@ void AnnotatedCameraWidget::paintGL() {
painter.setPen(Qt::NoPen);
if (s->scene.world_objects_visible) {
const cereal::ModelDataV2::Reader &model = sm["modelV2"].getModelV2();
update_model(s, model, sm["uiPlan"].getUiPlan());
drawLaneLines(painter, s);

View File

@ -37,6 +37,7 @@ private:
int status = STATUS_DISENGAGED;
std::unique_ptr<PubMaster> pm;
int skip_frame_count = 0;
bool wide_cam_requested = false;
protected:

View File

@ -97,22 +97,37 @@ mat4 get_fit_view_transform(float widget_aspect_ratio, float frame_aspect_ratio)
} // namespace
CameraWidget::CameraWidget(std::string stream_name, VisionStreamType type, bool zoom, QWidget *parent)
: stream_name(stream_name), stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) {
CameraWidget::CameraWidget(std::string stream_name, VisionStreamType type, bool zoom, QWidget* parent) :
stream_name(stream_name), active_stream_type(type), requested_stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) {
setAttribute(Qt::WA_OpaquePaintEvent);
qRegisterMetaType<std::set<VisionStreamType>>("availableStreams");
QObject::connect(this, &CameraWidget::vipcThreadConnected, this, &CameraWidget::vipcConnected, Qt::BlockingQueuedConnection);
QObject::connect(this, &CameraWidget::vipcThreadFrameReceived, this, &CameraWidget::vipcFrameReceived, Qt::QueuedConnection);
QObject::connect(this, &CameraWidget::vipcAvailableStreamsUpdated, this, &CameraWidget::availableStreamsUpdated, Qt::QueuedConnection);
}
CameraWidget::~CameraWidget() {
makeCurrent();
stopVipcThread();
if (isValid()) {
glDeleteVertexArrays(1, &frame_vao);
glDeleteBuffers(1, &frame_vbo);
glDeleteBuffers(1, &frame_ibo);
glDeleteBuffers(2, textures);
}
clearEGLImages();
doneCurrent();
}
// Qt uses device-independent pixels, depending on platform this may be
// different to what OpenGL uses
int CameraWidget::glWidth() {
return width() * devicePixelRatio();
}
int CameraWidget::glHeight() {
return height() * devicePixelRatio();
}
void CameraWidget::initializeGL() {
initializeOpenGLFunctions();
@ -126,7 +141,7 @@ void CameraWidget::initializeGL() {
GLint frame_pos_loc = program->attributeLocation("aPosition");
GLint frame_texcoord_loc = program->attributeLocation("aTexCoord");
auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
auto [x1, x2, y1, y2] = requested_stream_type == VISION_STREAM_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
const float frame_coords[4][4] = {
{-1.0, -1.0, x2, y1}, // bl
@ -163,11 +178,45 @@ void CameraWidget::initializeGL() {
#endif
}
void CameraWidget::showEvent(QShowEvent *event) {
if (!vipc_thread) {
clearFrames();
vipc_thread = new QThread();
connect(vipc_thread, &QThread::started, [=]() { vipcThread(); });
connect(vipc_thread, &QThread::finished, vipc_thread, &QObject::deleteLater);
vipc_thread->start();
}
}
void CameraWidget::stopVipcThread() {
makeCurrent();
if (vipc_thread) {
vipc_thread->requestInterruption();
vipc_thread->quit();
vipc_thread->wait();
vipc_thread = nullptr;
}
#ifdef QCOM2
EGLDisplay egl_display = eglGetCurrentDisplay();
assert(egl_display != EGL_NO_DISPLAY);
for (auto &pair : egl_images) {
eglDestroyImageKHR(egl_display, pair.second);
assert(eglGetError() == EGL_SUCCESS);
}
egl_images.clear();
#endif
}
void CameraWidget::availableStreamsUpdated(std::set<VisionStreamType> streams) {
available_streams = streams;
}
void CameraWidget::updateFrameMat() {
int w = glWidth(), h = glHeight();
if (zoomed_view) {
if (streamType() == VISION_STREAM_DRIVER) {
if (active_stream_type == VISION_STREAM_DRIVER) {
if (stream_width > 0 && stream_height > 0) {
frame_mat = get_driver_view_transform(w, h, stream_width, stream_height);
}
@ -176,7 +225,7 @@ void CameraWidget::updateFrameMat() {
// to ensure this ends up in the middle of the screen
// for narrow come and a little lower for wide cam.
// TODO: use proper perspective transform?
if (streamType() == VISION_STREAM_WIDE_ROAD) {
if (active_stream_type == VISION_STREAM_WIDE_ROAD) {
intrinsic_matrix = ECAM_INTRINSIC_MATRIX;
zoom = 2.0;
} else {
@ -198,12 +247,13 @@ void CameraWidget::updateFrameMat() {
float zx = zoom * 2 * intrinsic_matrix.v[2] / w;
float zy = zoom * 2 * intrinsic_matrix.v[5] / h;
frame_mat = {{
const mat4 frame_transform = {{
zx, 0.0, 0.0, -x_offset / w * 2,
0.0, zy, 0.0, y_offset / h * 2,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
frame_mat = frame_transform;
}
} else if (stream_width > 0 && stream_height > 0) {
// fit frame to widget size
@ -221,15 +271,25 @@ void CameraWidget::paintGL() {
glClearColor(bg.redF(), bg.greenF(), bg.blueF(), bg.alphaF());
glClear(GL_STENCIL_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
VisionBuf *frame = receiveFrame(draw_frame_id);
if (!frame) return;
std::lock_guard lk(frame_lock);
if (frames.empty()) return;
int frame_idx = frames.size() - 1;
// Always draw latest frame until sync logic is more stable
// for (frame_idx = 0; frame_idx < frames.size() - 1; frame_idx++) {
// if (frames[frame_idx].first == draw_frame_id) break;
// }
// Log duplicate/dropped frames
uint32_t frame_id = frame->get_frame_id();
if (prev_frame_id != INVALID_FRAME_ID && frame_id != prev_frame_id + 1) {
qDebug() << (frame_id == prev_frame_id ? "Drawing same frame twice" : "Skip frame") << frame_id;
if (frames[frame_idx].first == prev_frame_id) {
qDebug() << "Drawing same frame twice" << frames[frame_idx].first;
} else if (frames[frame_idx].first != prev_frame_id + 1) {
qDebug() << "Skipped frame" << frames[frame_idx].first;
}
prev_frame_id = frame_id;
prev_frame_id = frames[frame_idx].first;
VisionBuf *frame = frames[frame_idx].second;
assert(frame != nullptr);
updateFrameMat();
@ -269,14 +329,20 @@ void CameraWidget::paintGL() {
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
}
void CameraWidget::vipcConnected() {
void CameraWidget::vipcConnected(VisionIpcClient *vipc_client) {
makeCurrent();
stream_width = vipc_client->buffers[0].width;
stream_height = vipc_client->buffers[0].height;
stream_stride = vipc_client->buffers[0].stride;
#ifdef QCOM2
clearEGLImages();
EGLDisplay egl_display = eglGetCurrentDisplay();
assert(egl_display != EGL_NO_DISPLAY);
for (auto &pair : egl_images) {
eglDestroyImageKHR(egl_display, pair.second);
}
egl_images.clear();
for (int i = 0; i < vipc_client->num_buffers; i++) { // import buffers into OpenGL
int fd = dup(vipc_client->buffers[i].fd); // eglDestroyImageKHR will close, so duplicate
EGLint img_attrs[] = {
@ -313,78 +379,59 @@ void CameraWidget::vipcConnected() {
#endif
}
VisionBuf *CameraWidget::receiveFrame(uint64_t request_frame_id) {
if (!ensureConnection()) {
return nullptr;
}
// Receive frames and store them in recent_frames
while (auto buf = vipc_client->recv(nullptr, 0)) {
recent_frames.emplace_back(buf);
if (recent_frames.size() > FRAME_BUFFER_SIZE) {
recent_frames.pop_front();
}
}
if (!vipc_client->connected || recent_frames.empty()) {
return nullptr;
}
// Find the requested frame
auto it = std::find_if(recent_frames.rbegin(), recent_frames.rend(),
[request_frame_id](VisionBuf *buf) { return buf->get_frame_id() == request_frame_id; });
return it != recent_frames.rend() ? *it : recent_frames.back();
void CameraWidget::vipcFrameReceived() {
update();
}
bool CameraWidget::ensureConnection() {
// Reconnect if the client is not initialized or the stream type has changed
if (!vipc_client || vipc_client->type != stream_type) {
qDebug() << "connecting to stream" << stream_type;
vipc_client.reset(new VisionIpcClient(stream_name, stream_type, false));
}
void CameraWidget::vipcThread() {
VisionStreamType cur_stream = requested_stream_type;
std::unique_ptr<VisionIpcClient> vipc_client;
VisionIpcBufExtra meta_main = {0};
// Re-establish connection if not connected
if (!vipc_client->connected) {
clearFrames();
available_streams = VisionIpcClient::getAvailableStreams(stream_name, false);
if (available_streams.empty() || !vipc_client->connect(false)) {
return false;
while (!QThread::currentThread()->isInterruptionRequested()) {
if (!vipc_client || cur_stream != requested_stream_type) {
clearFrames();
qDebug().nospace() << "connecting to stream " << requested_stream_type << ", was connected to " << cur_stream;
cur_stream = requested_stream_type;
vipc_client.reset(new VisionIpcClient(stream_name, cur_stream, false));
}
active_stream_type = cur_stream;
if (!vipc_client->connected) {
clearFrames();
auto streams = VisionIpcClient::getAvailableStreams(stream_name, false);
if (streams.empty()) {
QThread::msleep(100);
continue;
}
emit vipcAvailableStreamsUpdated(streams);
if (!vipc_client->connect(false)) {
QThread::msleep(100);
continue;
}
emit vipcThreadConnected(vipc_client.get());
}
if (VisionBuf *buf = vipc_client->recv(&meta_main, 1000)) {
{
std::lock_guard lk(frame_lock);
frames.push_back(std::make_pair(meta_main.frame_id, buf));
while (frames.size() > FRAME_BUFFER_SIZE) {
frames.pop_front();
}
}
emit vipcThreadFrameReceived();
} else {
if (!isVisible()) {
vipc_client->connected = false;
}
}
emit vipcAvailableStreamsUpdated();
vipcConnected();
}
return true;
}
void CameraWidget::clearFrames() {
recent_frames.clear();
draw_frame_id = INVALID_FRAME_ID;
prev_frame_id = INVALID_FRAME_ID;
}
void CameraWidget::clearEGLImages() {
#ifdef QCOM2
EGLDisplay egl_display = eglGetCurrentDisplay();
assert(egl_display != EGL_NO_DISPLAY);
for (auto &pair : egl_images) {
eglDestroyImageKHR(egl_display, pair.second);
}
egl_images.clear();
#endif
}
// Cameraview
CameraView::CameraView(const std::string &name, VisionStreamType stream_type, bool zoom, QWidget *parent)
: CameraWidget(name, stream_type, zoom, parent) {
timer = new QTimer(this);
timer->setInterval(1000.0 / UI_FREQ);
timer->callOnTimeout(this, [this]() { update(); });
}
void CameraView::showEvent(QShowEvent *event) {
timer->start();
}
void CameraView::hideEvent(QHideEvent *event) {
timer->stop();
std::lock_guard lk(frame_lock);
frames.clear();
available_streams.clear();
}

View File

@ -3,6 +3,7 @@
#include <deque>
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <string>
#include <utility>
@ -10,7 +11,7 @@
#include <QOpenGLFunctions>
#include <QOpenGLShaderProgram>
#include <QOpenGLWidget>
#include <QTimer>
#include <QThread>
#ifdef QCOM2
#define EGL_EGLEXT_PROTOTYPES
@ -26,40 +27,40 @@
#include "selfdrive/ui/ui.h"
const int FRAME_BUFFER_SIZE = 5;
const uint32_t INVALID_FRAME_ID = -1;
static_assert(FRAME_BUFFER_SIZE <= YUV_BUFFER_COUNT);
class CameraWidget : public QOpenGLWidget, protected QOpenGLFunctions {
Q_OBJECT
public:
using QOpenGLWidget::QOpenGLWidget;
explicit CameraWidget(std::string stream_name, VisionStreamType stream_type, bool zoom, QWidget* parent = nullptr);
~CameraWidget();
void setBackgroundColor(const QColor &color) { bg = color; }
void setFrameId(int frame_id) { draw_frame_id = frame_id; }
void setStreamType(VisionStreamType type) { stream_type = type; }
inline VisionStreamType streamType() const { return stream_type; }
inline const std::set<VisionStreamType> &availableStreams() const { return available_streams; }
VisionBuf *receiveFrame(uint64_t request_frame_id = INVALID_FRAME_ID);
void setStreamType(VisionStreamType type) { requested_stream_type = type; }
VisionStreamType getStreamType() { return active_stream_type; }
void stopVipcThread();
signals:
void vipcAvailableStreamsUpdated();
void clicked();
void vipcThreadConnected(VisionIpcClient *);
void vipcThreadFrameReceived();
void vipcAvailableStreamsUpdated(std::set<VisionStreamType>);
protected:
bool ensureConnection();
void paintGL() override;
void initializeGL() override;
void vipcConnected();
void clearFrames();
void clearEGLImages();
void resizeGL(int w, int h) override { updateFrameMat(); }
void showEvent(QShowEvent *event) override;
void mouseReleaseEvent(QMouseEvent *event) override { emit clicked(); }
virtual void updateFrameMat();
void updateCalibration(const mat3 &calib);
int glWidth() const { return width() * devicePixelRatio(); }
int glHeight() const { return height() * devicePixelRatio(); }
void vipcThread();
void clearFrames();
int glWidth();
int glHeight();
bool zoomed_view;
GLuint frame_vao, frame_vbo, frame_ibo;
@ -76,7 +77,10 @@ protected:
int stream_width = 0;
int stream_height = 0;
int stream_stride = 0;
VisionStreamType stream_type;
std::atomic<VisionStreamType> active_stream_type;
std::atomic<VisionStreamType> requested_stream_type;
std::set<VisionStreamType> available_streams;
QThread *vipc_thread = nullptr;
// Calibration
float x_offset = 0;
@ -85,21 +89,15 @@ protected:
mat3 calibration = DEFAULT_CALIBRATION;
mat3 intrinsic_matrix = FCAM_INTRINSIC_MATRIX;
std::set<VisionStreamType> available_streams;
std::unique_ptr<VisionIpcClient> vipc_client;
std::deque<VisionBuf*> recent_frames;
uint32_t draw_frame_id = INVALID_FRAME_ID;
uint32_t prev_frame_id = INVALID_FRAME_ID;
std::recursive_mutex frame_lock;
std::deque<std::pair<uint32_t, VisionBuf*>> frames;
uint32_t draw_frame_id = 0;
uint32_t prev_frame_id = 0;
protected slots:
void vipcConnected(VisionIpcClient *vipc_client);
void vipcFrameReceived();
void availableStreamsUpdated(std::set<VisionStreamType> streams);
};
// update frames based on timer
class CameraView : public CameraWidget {
Q_OBJECT
public:
CameraView(const std::string &name, VisionStreamType stream_type, bool zoom, QWidget *parent = nullptr);
void showEvent(QShowEvent *event) override;
void hideEvent(QHideEvent *event) override;
private:
QTimer *timer;
};
Q_DECLARE_METATYPE(std::set<VisionStreamType>);

View File

@ -19,15 +19,15 @@ int main(int argc, char *argv[]) {
{
QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout);
hlayout->addWidget(new CameraView("navd", VISION_STREAM_MAP, false));
hlayout->addWidget(new CameraView("camerad", VISION_STREAM_ROAD, false));
hlayout->addWidget(new CameraWidget("navd", VISION_STREAM_MAP, false));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_ROAD, false));
}
{
QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout);
hlayout->addWidget(new CameraView("camerad", VISION_STREAM_DRIVER, false));
hlayout->addWidget(new CameraView("camerad", VISION_STREAM_WIDE_ROAD, false));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_DRIVER, false));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_WIDE_ROAD, false));
}
return a.exec();

View File

@ -139,7 +139,7 @@ QWidget *VideoWidget::createCameraWidget() {
QStackedLayout *stacked = new QStackedLayout();
stacked->setStackingMode(QStackedLayout::StackAll);
stacked->addWidget(cam_widget = new CameraView("camerad", VISION_STREAM_ROAD, false));
stacked->addWidget(cam_widget = new CameraWidget("camerad", VISION_STREAM_ROAD, false));
cam_widget->setMinimumHeight(MIN_VIDEO_HEIGHT);
cam_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
stacked->addWidget(alert_label = new InfoLabel(this));
@ -161,13 +161,12 @@ QWidget *VideoWidget::createCameraWidget() {
return w;
}
void VideoWidget::vipcAvailableStreamsUpdated() {
void VideoWidget::vipcAvailableStreamsUpdated(std::set<VisionStreamType> streams) {
static const QString stream_names[] = {
[VISION_STREAM_ROAD] = "Road camera",
[VISION_STREAM_WIDE_ROAD] = "Wide road camera",
[VISION_STREAM_DRIVER] = "Driver camera"};
const auto &streams = cam_widget->availableStreams();
for (int i = 0; i < streams.size(); ++i) {
if (camera_tab->count() <= i) {
camera_tab->addTab(QString());

View File

@ -73,9 +73,9 @@ protected:
QWidget *createCameraWidget();
QHBoxLayout *createPlaybackController();
void loopPlaybackClicked();
void vipcAvailableStreamsUpdated();
void vipcAvailableStreamsUpdated(std::set<VisionStreamType> streams);
CameraView *cam_widget;
CameraWidget *cam_widget;
double maximum_time = 0;
QToolButton *time_btn = nullptr;
ToolButton *seek_backward_btn = nullptr;