From 342ff24510542d5ba2d4bb9c193308db0c9ad462 Mon Sep 17 00:00:00 2001 From: royjr Date: Tue, 26 Aug 2025 11:49:55 -0400 Subject: [PATCH] feature: external storage (#979) * external storage * fix mountStorage * fix perms * works for now * better * lagless * move to sp qt * orderish * fix ui crash * cleanup * fix format * offroad only * debug external storage * dont care about delete * just use cloudlog * show logs if using external storage * better text * wipe entire drive * allow partitionless drive to be formatted * label while formatting * this works * better * cleaner * cleaner logs * keep upstream happy --------- Co-authored-by: DevTekVE --- selfdrive/ui/sunnypilot/SConscript | 1 + .../qt/offroad/settings/developer_panel.cc | 5 + .../sunnypilot/qt/widgets/external_storage.cc | 170 ++++++++++++++++++ .../sunnypilot/qt/widgets/external_storage.h | 34 ++++ system/athena/athenad.py | 24 ++- system/hardware/hw.py | 4 + system/loggerd/config.py | 17 +- system/loggerd/deleter.py | 37 ++++ 8 files changed, 280 insertions(+), 12 deletions(-) create mode 100644 selfdrive/ui/sunnypilot/qt/widgets/external_storage.cc create mode 100644 selfdrive/ui/sunnypilot/qt/widgets/external_storage.h diff --git a/selfdrive/ui/sunnypilot/SConscript b/selfdrive/ui/sunnypilot/SConscript index 810338aae..2f3c8ddd8 100644 --- a/selfdrive/ui/sunnypilot/SConscript +++ b/selfdrive/ui/sunnypilot/SConscript @@ -4,6 +4,7 @@ widgets_src = [ "sunnypilot/qt/widgets/controls.cc", "sunnypilot/qt/widgets/drive_stats.cc", "sunnypilot/qt/widgets/expandable_row.cc", + "sunnypilot/qt/widgets/external_storage.cc", "sunnypilot/qt/widgets/prime.cc", "sunnypilot/qt/widgets/scrollview.cc", "sunnypilot/qt/network/networking.cc", diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.cc index 58193f9fe..a4a6bf481 100644 --- a/selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.cc +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.cc @@ -5,9 +5,14 @@ * See the LICENSE.md file in the root directory for more details. */ #include "selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.h" +#include "selfdrive/ui/sunnypilot/qt/widgets/external_storage.h" DeveloperPanelSP::DeveloperPanelSP(SettingsWindow *parent) : DeveloperPanel(parent) { + #ifndef __APPLE__ + addItem(new ExternalStorageControl()); + #endif + // Advanced Controls Toggle showAdvancedControls = new ParamControlSP("ShowAdvancedControls", tr("Show Advanced Controls"), tr("Toggle visibility of advanced sunnypilot controls.\nThis only toggles the visibility of the controls; it does not toggle the actual control enabled/disabled state."), ""); addItem(showAdvancedControls); diff --git a/selfdrive/ui/sunnypilot/qt/widgets/external_storage.cc b/selfdrive/ui/sunnypilot/qt/widgets/external_storage.cc new file mode 100644 index 000000000..f95118751 --- /dev/null +++ b/selfdrive/ui/sunnypilot/qt/widgets/external_storage.cc @@ -0,0 +1,170 @@ +/** + * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + * + * This file is part of sunnypilot and is licensed under the MIT License. + * See the LICENSE.md file in the root directory for more details. + */ + +#include "selfdrive/ui/sunnypilot/qt/widgets/external_storage.h" + +#include +#include +#include +#include +#include + +#include "common/params.h" +#include "selfdrive/ui/qt/api.h" +#include "selfdrive/ui/qt/widgets/input.h" +#include "selfdrive/ui/sunnypilot/ui.h" + +ExternalStorageControl::ExternalStorageControl() : + ButtonControl(tr("External Storage"), "", tr("Extend your comma device's storage by inserting a USB drive into the aux port.")) { + + QObject::connect(this, &ButtonControl::clicked, [=]() { + if (text() == tr("CHECK") || text() == tr("MOUNT")) { + mountStorage(); + } else if (text() == tr("UNMOUNT")) { + unmountStorage(); + } else if (text() == tr("FORMAT")) { + if (ConfirmationDialog::confirm(tr("Are you sure you want to format this drive? This will erase all data."), tr("Format"), this)) { + formatStorage(); + } + } + }); + + QObject::connect(uiState(), &UIState::offroadTransition, this, &ExternalStorageControl::updateState); + updateState(!uiState()->scene.started); + + refresh(); +} + +void ExternalStorageControl::updateState(bool offroad) { + setEnabled(offroad); +} + +void ExternalStorageControl::debouncedRefresh() { + if (refreshPending) return; + refreshPending = true; + + QTimer::singleShot(250, this, [=]() { + refreshPending = false; + refresh(); + }); +} + +void ExternalStorageControl::refresh() { + QtConcurrent::run([=]() { + auto run = [](const QString &cmd) { + QProcess p; + p.start("sh", QStringList() << "-c" << cmd); + p.waitForFinished(); + return p.exitCode() == 0; + }; + + bool isMounted = run("findmnt -n /mnt/external_realdata"); + bool hasDrive = run("lsblk -f /dev/sdg"); + bool hasFs = run("lsblk -f /dev/sdg1 | grep -q ext4"); + bool hasLabel = run("sudo blkid /dev/sdg1 | grep -q 'LABEL=\"openpilot\"'"); + + QString info; + if (isMounted && hasLabel) { + QProcess df; + df.start("sh", QStringList() << "-c" << "df -h /mnt/external_realdata | awk 'NR==2 {print $3 \"/\" $2}'"); + df.waitForFinished(); + info = df.readAllStandardOutput().trimmed(); + } + + QMetaObject::invokeMethod(this, [=]() { + if (formatting) { + setValue(tr("formatting")); + setText(tr("FORMAT")); + setEnabled(false); + } else { + if (!hasDrive) { + setValue(tr("insert drive")); + setText(tr("CHECK")); + } else if (!hasFs || !hasLabel) { + setValue(tr("needs format")); + setText(tr("FORMAT")); + } else if (isMounted) { + setValue(info); + setText(tr("UNMOUNT")); + } else { + setValue("drive detected"); + setText(tr("MOUNT")); + } + updateState(!uiState()->scene.started); + } + }, Qt::QueuedConnection); + }); +} + +void ExternalStorageControl::mountStorage() { + setValue(tr("mounting")); + setEnabled(false); + + QtConcurrent::run([=]() { + QProcess process; + process.start("sh", QStringList() << "-c" << + "sudo mount -o remount,rw / && " + "sudo mkdir -p /mnt/external_realdata && " + "grep -q '/dev/sdg1 /mnt/external_realdata' /etc/fstab || " + "echo '/dev/sdg1 /mnt/external_realdata ext4 defaults,nofail 0 2' | sudo tee -a /etc/fstab && " + "sudo systemctl daemon-reexec && " + "sudo mount /mnt/external_realdata && " + "sudo chown -R comma:comma /mnt/external_realdata && " + "sudo chmod -R 775 /mnt/external_realdata && " + "sudo mount -o remount,ro /"); + process.waitForFinished(); + + QMetaObject::invokeMethod(this, [=]() { + debouncedRefresh(); + }, Qt::QueuedConnection); + }); +} + +void ExternalStorageControl::unmountStorage() { + setValue(tr("unmounting")); + setEnabled(false); + + QtConcurrent::run([=]() { + QProcess process; + process.start("sh", QStringList() << "-c" << "sudo umount /mnt/external_realdata"); + process.waitForFinished(); + + QMetaObject::invokeMethod(this, [=]() { + debouncedRefresh(); + }, Qt::QueuedConnection); + }); +} + +void ExternalStorageControl::formatStorage() { + unmountStorage(); + formatting = true; + setValue(tr("formatting")); + setEnabled(false); + + QProcess *process = new QProcess(this); + connect(process, static_cast(&QProcess::finished), + this, [=](int exitCode, QProcess::ExitStatus status) { + process->deleteLater(); + formatting = false; + if (exitCode == 0 && status == QProcess::NormalExit) { + mountStorage(); + } else { + setValue(tr("needs format")); + updateState(!uiState()->scene.started); + } + }); + process->start("sh", QStringList() << "-c" << + "sudo wipefs -a /dev/sdg && " + "sudo parted -s /dev/sdg mklabel gpt mkpart primary ext4 0% 100% && " + "sudo mkfs.ext4 -F -L openpilot /dev/sdg1" + ); +} + +void ExternalStorageControl::showEvent(QShowEvent *event) { + ButtonControl::showEvent(event); + QTimer::singleShot(100, this, &ExternalStorageControl::debouncedRefresh); +} diff --git a/selfdrive/ui/sunnypilot/qt/widgets/external_storage.h b/selfdrive/ui/sunnypilot/qt/widgets/external_storage.h new file mode 100644 index 000000000..d26eefd18 --- /dev/null +++ b/selfdrive/ui/sunnypilot/qt/widgets/external_storage.h @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + * + * This file is part of sunnypilot and is licensed under the MIT License. + * See the LICENSE.md file in the root directory for more details. + */ + +#pragma once + +#include "system/hardware/hw.h" +#include "selfdrive/ui/sunnypilot/qt/widgets/controls.h" +#define ButtonControl ButtonControlSP + +class ExternalStorageControl : public ButtonControl { + Q_OBJECT + +public: + ExternalStorageControl(); + +protected: + void showEvent(QShowEvent *event) override; + +private: + Params params; + + bool refreshPending = false; + bool formatting = false; + void updateState(bool offroad); + void refresh(); + void debouncedRefresh(); + void mountStorage(); + void unmountStorage(); + void formatStorage(); +}; diff --git a/system/athena/athenad.py b/system/athena/athenad.py index f97b8e55b..42c9cf8a1 100755 --- a/system/athena/athenad.py +++ b/system/athena/athenad.py @@ -381,20 +381,22 @@ def setNavDestination(latitude: int = 0, longitude: int = 0, place_name: str = N return {"success": 1} -def scan_dir(path: str, prefix: str) -> list[str]: +def scan_dir(path: str, prefix: str, base: str | None = None) -> list[str]: + if base is None: + base = path files = [] # only walk directories that match the prefix # (glob and friends traverse entire dir tree) with os.scandir(path) as i: for e in i: - rel_path = os.path.relpath(e.path, Paths.log_root()) + rel_path = os.path.relpath(e.path, base) if e.is_dir(follow_symlinks=False): # add trailing slash rel_path = os.path.join(rel_path, '') # if prefix is a partial dir name, current dir will start with prefix # if prefix is a partial file name, prefix with start with dir name if rel_path.startswith(prefix) or prefix.startswith(rel_path): - files.extend(scan_dir(e.path, prefix)) + files.extend(scan_dir(e.path, prefix, base)) else: if rel_path.startswith(prefix): files.append(rel_path) @@ -402,7 +404,12 @@ def scan_dir(path: str, prefix: str) -> list[str]: @dispatcher.add_method def listDataDirectory(prefix='') -> list[str]: - return scan_dir(Paths.log_root(), prefix) + internal_files = scan_dir(Paths.log_root(), prefix, Paths.log_root()) + try: + external_files = scan_dir(Paths.log_root_external(), prefix, Paths.log_root_external()) + except FileNotFoundError: + external_files = [] + return sorted(set(internal_files + external_files)) @dispatcher.add_method @@ -427,8 +434,13 @@ def uploadFilesToUrls(files_data: list[UploadFileDict]) -> UploadFilesToUrlRespo failed.append(file.fn) continue - path = os.path.join(Paths.log_root(), file.fn) - if not os.path.exists(path) and not os.path.exists(strip_zst_extension(path)): + path_internal = os.path.join(Paths.log_root(), file.fn) + path_external = os.path.join(Paths.log_root_external(), file.fn) + if os.path.exists(path_internal) or os.path.exists(strip_zst_extension(path_internal)): + path = path_internal + elif os.path.exists(path_external) or os.path.exists(strip_zst_extension(path_external)): + path = path_external + else: failed.append(file.fn) continue diff --git a/system/hardware/hw.py b/system/hardware/hw.py index 5e40fff13..d24857e8b 100644 --- a/system/hardware/hw.py +++ b/system/hardware/hw.py @@ -20,6 +20,10 @@ class Paths: else: return '/data/media/0/realdata/' + @staticmethod + def log_root_external() -> str: + return '/mnt/external_realdata/' + @staticmethod def swaglog_root() -> str: if PC: diff --git a/system/loggerd/config.py b/system/loggerd/config.py index e1c47c768..d9befb561 100644 --- a/system/loggerd/config.py +++ b/system/loggerd/config.py @@ -9,21 +9,26 @@ STATS_DIR_FILE_LIMIT = 10000 STATS_SOCKET = "ipc:///tmp/stats" STATS_FLUSH_TIME_S = 60 -def get_available_percent(default: float) -> float: +PATH_DICT = { + "internal": Paths.log_root(), + "external": Paths.log_root_external() +} + +def get_available_percent(default: float, path_type="internal") -> float: try: - statvfs = os.statvfs(Paths.log_root()) + statvfs = os.statvfs(PATH_DICT[path_type]) available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks - except OSError: + except (OSError, KeyError): available_percent = default return available_percent -def get_available_bytes(default: int) -> int: +def get_available_bytes(default: int, path_type="internal") -> int: try: - statvfs = os.statvfs(Paths.log_root()) + statvfs = os.statvfs(PATH_DICT[path_type]) available_bytes = statvfs.f_bavail * statvfs.f_frsize - except OSError: + except (OSError, KeyError): available_bytes = default return available_bytes diff --git a/system/loggerd/deleter.py b/system/loggerd/deleter.py index eb8fd35f2..058f5c301 100755 --- a/system/loggerd/deleter.py +++ b/system/loggerd/deleter.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import os +import time import shutil import threading +from pathlib import Path from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.loggerd.config import get_available_bytes, get_available_percent @@ -61,6 +63,41 @@ def deleter_thread(exit_event: threading.Event): if any(name.endswith(".lock") for name in os.listdir(delete_path)): continue + if Path(Paths.log_root_external()).is_mount(): + out_of_bytes_external = get_available_bytes(default=MIN_BYTES + 1, path_type="external") < MIN_BYTES + out_of_percent_external = get_available_percent(default=MIN_PERCENT + 1, path_type="external") < MIN_PERCENT + + if out_of_percent_external or out_of_bytes_external: + dirs_external = listdir_by_creation(Paths.log_root_external()) + + # remove the earliest external directory we can + for delete_dir_external in sorted(dirs_external): + delete_path_external = os.path.join(Paths.log_root_external(), delete_dir_external) + try: + cloudlog.warning(f"deleting {delete_path_external}") + shutil.rmtree(delete_path_external) + break + except OSError: + cloudlog.exception(f"issue deleting {delete_path_external}") + + # move directory from internal to external + path_external = os.path.join(Paths.log_root_external(), delete_dir) + try: + cloudlog.warning(f"moving {delete_path} to {path_external}") + start = time.monotonic() + shutil.move(delete_path, path_external) + cloudlog.warning(f"moved {delete_path} to {path_external} in {time.monotonic() - start:.2f}s") + break + except Exception: + cloudlog.error(f"issue moving {delete_path} to {path_external}") + try: + cloudlog.warning(f"deleting {delete_path}") + shutil.rmtree(delete_path) + break + except OSError: + cloudlog.exception(f"issue deleting {delete_path}") + continue + try: cloudlog.info(f"deleting {delete_path}") shutil.rmtree(delete_path)