feature: external storage (#979)

* external storage

* fix mountStorage

* fix perms

* works for now

* better

* lagless

* move to sp qt

* orderish

* fix ui crash

* cleanup

* fix format

* offroad only

* debug external storage

* dont care about delete

* just use cloudlog

* show logs if using external storage

* better text

* wipe entire drive

* allow partitionless drive to be formatted

* label while formatting

* this works

* better

* cleaner

* cleaner logs

* keep upstream happy

---------

Co-authored-by: DevTekVE <devtekve@gmail.com>

selfdrive/ui/sunnypilot/SConscript

@@ -4,6 +4,7 @@ widgets_src = [
   "sunnypilot/qt/widgets/controls.cc",
   "sunnypilot/qt/widgets/drive_stats.cc",
   "sunnypilot/qt/widgets/expandable_row.cc",
+  "sunnypilot/qt/widgets/external_storage.cc",
   "sunnypilot/qt/widgets/prime.cc",
   "sunnypilot/qt/widgets/scrollview.cc",
   "sunnypilot/qt/network/networking.cc",

selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.cc

@@ -5,9 +5,14 @@
  * See the LICENSE.md file in the root directory for more details.
  */
 #include "selfdrive/ui/sunnypilot/qt/offroad/settings/developer_panel.h"
+#include "selfdrive/ui/sunnypilot/qt/widgets/external_storage.h"
 
 DeveloperPanelSP::DeveloperPanelSP(SettingsWindow *parent) : DeveloperPanel(parent) {
 
+  #ifndef __APPLE__
+  addItem(new ExternalStorageControl());
+  #endif
+
   // Advanced Controls Toggle
   showAdvancedControls = new ParamControlSP("ShowAdvancedControls", tr("Show Advanced Controls"), tr("Toggle visibility of advanced sunnypilot controls.\nThis only toggles the visibility of the controls; it does not toggle the actual control enabled/disabled state."), "");
   addItem(showAdvancedControls);

selfdrive/ui/sunnypilot/qt/widgets/external_storage.cc

@@ -0,0 +1,170 @@
+/**
+ * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+ *
+ * This file is part of sunnypilot and is licensed under the MIT License.
+ * See the LICENSE.md file in the root directory for more details.
+ */
+
+#include "selfdrive/ui/sunnypilot/qt/widgets/external_storage.h"
+
+#include <QProcess>
+#include <QCoreApplication>
+#include <QShowEvent>
+#include <QTimer>
+#include <QtConcurrent>
+
+#include "common/params.h"
+#include "selfdrive/ui/qt/api.h"
+#include "selfdrive/ui/qt/widgets/input.h"
+#include "selfdrive/ui/sunnypilot/ui.h"
+
+ExternalStorageControl::ExternalStorageControl() :
+  ButtonControl(tr("External Storage"), "", tr("Extend your comma device's storage by inserting a USB drive into the aux port.")) {
+
+  QObject::connect(this, &ButtonControl::clicked, [=]() {
+    if (text() == tr("CHECK") || text() == tr("MOUNT")) {
+      mountStorage();
+    } else if (text() == tr("UNMOUNT")) {
+      unmountStorage();
+    } else if (text() == tr("FORMAT")) {
+      if (ConfirmationDialog::confirm(tr("Are you sure you want to format this drive? This will erase all data."), tr("Format"), this)) {
+        formatStorage();
+      }
+    }
+  });
+
+  QObject::connect(uiState(), &UIState::offroadTransition, this, &ExternalStorageControl::updateState);
+  updateState(!uiState()->scene.started);
+
+  refresh();
+}
+
+void ExternalStorageControl::updateState(bool offroad) {
+  setEnabled(offroad);
+}
+
+void ExternalStorageControl::debouncedRefresh() {
+  if (refreshPending) return;
+  refreshPending = true;
+
+  QTimer::singleShot(250, this, [=]() {
+    refreshPending = false;
+    refresh();
+  });
+}
+
+void ExternalStorageControl::refresh() {
+  QtConcurrent::run([=]() {
+    auto run = [](const QString &cmd) {
+      QProcess p;
+      p.start("sh", QStringList() << "-c" << cmd);
+      p.waitForFinished();
+      return p.exitCode() == 0;
+    };
+
+    bool isMounted = run("findmnt -n /mnt/external_realdata");
+    bool hasDrive = run("lsblk -f /dev/sdg");
+    bool hasFs = run("lsblk -f /dev/sdg1 | grep -q ext4");
+    bool hasLabel = run("sudo blkid /dev/sdg1 | grep -q 'LABEL=\"openpilot\"'");
+
+    QString info;
+    if (isMounted && hasLabel) {
+      QProcess df;
+      df.start("sh", QStringList() << "-c" << "df -h /mnt/external_realdata | awk 'NR==2 {print $3 \"/\" $2}'");
+      df.waitForFinished();
+      info = df.readAllStandardOutput().trimmed();
+    }
+
+    QMetaObject::invokeMethod(this, [=]() {
+      if (formatting) {
+        setValue(tr("formatting"));
+        setText(tr("FORMAT"));
+        setEnabled(false);
+      } else {
+        if (!hasDrive) {
+          setValue(tr("insert drive"));
+          setText(tr("CHECK"));
+        } else if (!hasFs || !hasLabel) {
+          setValue(tr("needs format"));
+          setText(tr("FORMAT"));
+        } else if (isMounted) {
+          setValue(info);
+          setText(tr("UNMOUNT"));
+        } else {
+          setValue("drive detected");
+          setText(tr("MOUNT"));
+        }
+        updateState(!uiState()->scene.started);
+      }
+    }, Qt::QueuedConnection);
+  });
+}
+
+void ExternalStorageControl::mountStorage() {
+  setValue(tr("mounting"));
+  setEnabled(false);
+
+  QtConcurrent::run([=]() {
+    QProcess process;
+    process.start("sh", QStringList() << "-c" <<
+      "sudo mount -o remount,rw / && "
+      "sudo mkdir -p /mnt/external_realdata && "
+      "grep -q '/dev/sdg1 /mnt/external_realdata' /etc/fstab || "
+      "echo '/dev/sdg1 /mnt/external_realdata ext4 defaults,nofail 0 2' | sudo tee -a /etc/fstab && "
+      "sudo systemctl daemon-reexec && "
+      "sudo mount /mnt/external_realdata && "
+      "sudo chown -R comma:comma /mnt/external_realdata && "
+      "sudo chmod -R 775 /mnt/external_realdata && "
+      "sudo mount -o remount,ro /");
+    process.waitForFinished();
+
+    QMetaObject::invokeMethod(this, [=]() {
+      debouncedRefresh();
+    }, Qt::QueuedConnection);
+  });
+}
+
+void ExternalStorageControl::unmountStorage() {
+  setValue(tr("unmounting"));
+  setEnabled(false);
+
+  QtConcurrent::run([=]() {
+    QProcess process;
+    process.start("sh", QStringList() << "-c" << "sudo umount /mnt/external_realdata");
+    process.waitForFinished();
+
+    QMetaObject::invokeMethod(this, [=]() {
+      debouncedRefresh();
+    }, Qt::QueuedConnection);
+  });
+}
+
+void ExternalStorageControl::formatStorage() {
+  unmountStorage();
+  formatting = true;
+  setValue(tr("formatting"));
+  setEnabled(false);
+
+  QProcess *process = new QProcess(this);
+  connect(process, static_cast<void(QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
+          this, [=](int exitCode, QProcess::ExitStatus status) {
+            process->deleteLater();
+            formatting = false;
+            if (exitCode == 0 && status == QProcess::NormalExit) {
+              mountStorage();
+            } else {
+              setValue(tr("needs format"));
+              updateState(!uiState()->scene.started);
+            }
+          });
+  process->start("sh", QStringList() << "-c" <<
+    "sudo wipefs -a /dev/sdg && "
+    "sudo parted -s /dev/sdg mklabel gpt mkpart primary ext4 0% 100% && "
+    "sudo mkfs.ext4 -F -L openpilot /dev/sdg1"
+  );
+}
+
+void ExternalStorageControl::showEvent(QShowEvent *event) {
+  ButtonControl::showEvent(event);
+  QTimer::singleShot(100, this, &ExternalStorageControl::debouncedRefresh);
+}

selfdrive/ui/sunnypilot/qt/widgets/external_storage.h

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors.
+ *
+ * This file is part of sunnypilot and is licensed under the MIT License.
+ * See the LICENSE.md file in the root directory for more details.
+ */
+
+#pragma once
+
+#include "system/hardware/hw.h"
+#include "selfdrive/ui/sunnypilot/qt/widgets/controls.h"
+#define ButtonControl ButtonControlSP
+
+class ExternalStorageControl : public ButtonControl {
+  Q_OBJECT
+
+public:
+  ExternalStorageControl();
+
+protected:
+  void showEvent(QShowEvent *event) override;
+
+private:
+  Params params;
+
+  bool refreshPending = false;
+  bool formatting = false;
+  void updateState(bool offroad);
+  void refresh();
+  void debouncedRefresh();
+  void mountStorage();
+  void unmountStorage();
+  void formatStorage();
+};

system/athena/athenad.py

@@ -381,20 +381,22 @@ def setNavDestination(latitude: int = 0, longitude: int = 0, place_name: str = N
   return {"success": 1}
 
 
-def scan_dir(path: str, prefix: str) -> list[str]:
+def scan_dir(path: str, prefix: str, base: str | None = None) -> list[str]:
+  if base is None:
+    base = path
   files = []
   # only walk directories that match the prefix
   # (glob and friends traverse entire dir tree)
   with os.scandir(path) as i:
     for e in i:
-      rel_path = os.path.relpath(e.path, Paths.log_root())
+      rel_path = os.path.relpath(e.path, base)
       if e.is_dir(follow_symlinks=False):
         # add trailing slash
         rel_path = os.path.join(rel_path, '')
         # if prefix is a partial dir name, current dir will start with prefix
         # if prefix is a partial file name, prefix with start with dir name
         if rel_path.startswith(prefix) or prefix.startswith(rel_path):
-          files.extend(scan_dir(e.path, prefix))
+          files.extend(scan_dir(e.path, prefix, base))
       else:
         if rel_path.startswith(prefix):
           files.append(rel_path)
@@ -402,7 +404,12 @@ def scan_dir(path: str, prefix: str) -> list[str]:
 
 @dispatcher.add_method
 def listDataDirectory(prefix='') -> list[str]:
-  return scan_dir(Paths.log_root(), prefix)
+  internal_files = scan_dir(Paths.log_root(), prefix, Paths.log_root())
+  try:
+    external_files = scan_dir(Paths.log_root_external(), prefix, Paths.log_root_external())
+  except FileNotFoundError:
+    external_files = []
+  return sorted(set(internal_files + external_files))
 
 
 @dispatcher.add_method
@@ -427,8 +434,13 @@ def uploadFilesToUrls(files_data: list[UploadFileDict]) -> UploadFilesToUrlRespo
       failed.append(file.fn)
       continue
 
-    path = os.path.join(Paths.log_root(), file.fn)
-    if not os.path.exists(path) and not os.path.exists(strip_zst_extension(path)):
+    path_internal = os.path.join(Paths.log_root(), file.fn)
+    path_external = os.path.join(Paths.log_root_external(), file.fn)
+    if os.path.exists(path_internal) or os.path.exists(strip_zst_extension(path_internal)):
+      path = path_internal
+    elif os.path.exists(path_external) or os.path.exists(strip_zst_extension(path_external)):
+      path = path_external
+    else:
       failed.append(file.fn)
       continue
 

system/hardware/hw.py

@@ -20,6 +20,10 @@ class Paths:
     else:
       return '/data/media/0/realdata/'
 
+  @staticmethod
+  def log_root_external() -> str:
+    return '/mnt/external_realdata/'
+
   @staticmethod
   def swaglog_root() -> str:
     if PC:

system/loggerd/config.py

@@ -9,21 +9,26 @@ STATS_DIR_FILE_LIMIT = 10000
 STATS_SOCKET = "ipc:///tmp/stats"
 STATS_FLUSH_TIME_S = 60
 
-def get_available_percent(default: float) -> float:
+PATH_DICT = {
+  "internal": Paths.log_root(),
+  "external": Paths.log_root_external()
+}
+
+def get_available_percent(default: float, path_type="internal") -> float:
   try:
-    statvfs = os.statvfs(Paths.log_root())
+    statvfs = os.statvfs(PATH_DICT[path_type])
     available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
-  except OSError:
+  except (OSError, KeyError):
     available_percent = default
 
   return available_percent
 
 
-def get_available_bytes(default: int) -> int:
+def get_available_bytes(default: int, path_type="internal") -> int:
   try:
-    statvfs = os.statvfs(Paths.log_root())
+    statvfs = os.statvfs(PATH_DICT[path_type])
     available_bytes = statvfs.f_bavail * statvfs.f_frsize
-  except OSError:
+  except (OSError, KeyError):
     available_bytes = default
 
   return available_bytes

system/loggerd/deleter.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python3
 import os
+import time
 import shutil
 import threading
+from pathlib import Path
 from openpilot.system.hardware.hw import Paths
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
@@ -61,6 +63,41 @@ def deleter_thread(exit_event: threading.Event):
         if any(name.endswith(".lock") for name in os.listdir(delete_path)):
           continue
 
+        if Path(Paths.log_root_external()).is_mount():
+          out_of_bytes_external = get_available_bytes(default=MIN_BYTES + 1, path_type="external") < MIN_BYTES
+          out_of_percent_external = get_available_percent(default=MIN_PERCENT + 1, path_type="external") < MIN_PERCENT
+
+          if out_of_percent_external or out_of_bytes_external:
+            dirs_external = listdir_by_creation(Paths.log_root_external())
+
+            # remove the earliest external directory we can
+            for delete_dir_external in sorted(dirs_external):
+              delete_path_external = os.path.join(Paths.log_root_external(), delete_dir_external)
+              try:
+                cloudlog.warning(f"deleting {delete_path_external}")
+                shutil.rmtree(delete_path_external)
+                break
+              except OSError:
+                cloudlog.exception(f"issue deleting {delete_path_external}")
+
+          # move directory from internal to external
+          path_external = os.path.join(Paths.log_root_external(), delete_dir)
+          try:
+            cloudlog.warning(f"moving {delete_path} to {path_external}")
+            start = time.monotonic()
+            shutil.move(delete_path, path_external)
+            cloudlog.warning(f"moved {delete_path} to {path_external} in {time.monotonic() - start:.2f}s")
+            break
+          except Exception:
+            cloudlog.error(f"issue moving {delete_path} to {path_external}")
+            try:
+              cloudlog.warning(f"deleting {delete_path}")
+              shutil.rmtree(delete_path)
+              break
+            except OSError:
+              cloudlog.exception(f"issue deleting {delete_path}")
+          continue
+
         try:
           cloudlog.info(f"deleting {delete_path}")
           shutil.rmtree(delete_path)